Local_2_lexical_alignment.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. import data and packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import neccessary packages\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "import glob\n",
    "import os\n",
    "import re\n",
    "import random\n",
    "import itertools\n",
    "import collections\n",
    "from collections import Counter\n",
    "import regex as re\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from scipy.ndimage.filters import gaussian_filter1d\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.2 File import\n",
    "### You should have run SILLA first, and have the cleaned file ready"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ms = pd.read_pickle(\"your_processed_file_from_silla\")\n",
    "# only need these columns\n",
    "ms_df = df_ms[['Speaker', 'Text', 'group','group_turn_id','group_speaker_turn_id']]\n",
    "ms_df['w_target'] = ms_df['Text']\n",
    "ms_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Text preprocessing, get token list of words for each utterance \n",
    "lowercase, tokenize, punc, stopwords\n",
    "## 2.1 Convert text to lowercase\n",
    "Will convert text to lowercase so there is no issue with word comparisons."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "for i,j in enumerate(ms_df['w_target']):\n",
    "    ms_df['w_target'][i] = str(j).lower()\n",
    "\n",
    "print(ms_df['w_target'][-3:])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Remove the text in [], which includes things like \"[Student calls teacher to look at their assignment.]\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "for i,j in enumerate(ms_df['w_target']):\n",
    "    ms_df['Text'][i] = re.sub(\"[\\(\\[].*?[\\)\\]]\", '', str(j))\n",
    "\n",
    "# print(ms_df['w_target'][-10:])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.2 Tokenize and Remove punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "for i, j in enumerate(ms_df['w_target']):\n",
    "    # print('j:', j)\n",
    "    words = nltk.word_tokenize(str(j))\n",
    "    ms_df['w_target'][i] = [word for word in words if word.isalnum()]\n",
    "    # print(\"after:\",ms_df['Text'][i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(ms_df['w_target'][-10:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# ms_df.to_pickle('Middle_School/before_stop_remove')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.3 Remove Stop Words\n",
    "Be careful about removing stop words, because it will also remove some programming-related words like `if`, `for`, and `and`. If these words are important for your analysis, then skip this step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "stopword = stopwords.words('english')\n",
    "for i, j in enumerate(ms_df['Text']):\n",
    "    # print(j)\n",
    "    ms_df['w_target'][i] = [word for word in j if word not in stopword]\n",
    "    # print(ms_df['Text'][i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(ms_df['w_target'][-10:])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Set up target and prime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.1 target = word list of current utterance\n",
    "This has been setted up previously, the `w_target`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.2 get prime, previous n utterance from the other speaker (window size customizable)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set get_prime_w_list to be n turns prior to the target turn \n",
    "def get_prime_w_list(df_ms, n): # specify the number of turns  as n, return a dict\n",
    "    prime_dict = {}\n",
    "    for group,turn_id in zip(df_ms.group, df_ms.group_turn_id):\n",
    "        if turn_id == 1: # when it moves to a new group, store the group_name\n",
    "            group_name = group\n",
    "        #  get the index of the row \n",
    "        index1 = df_ms[(df_ms['group'] == group_name) & (df_ms['group_turn_id'] == turn_id)].index.item()\n",
    "        \n",
    "        if group == group_name: # prime has to be done within the same group\n",
    "            # if index < n, then do everything from 1 to index\n",
    "            prime_count = turn_id//2 # the number of rows priming: turn_id//2. 5//2 = 2\n",
    "            speaker = df_ms.iloc[index1]['Speaker'] # find the value in column 'Speaker' and row 'index1'\n",
    "            word_list = []\n",
    "            if prime_count < n: \n",
    "                word_list = df_ms[(df_ms['group'] == group_name) & \n",
    "                                    (df_ms['Speaker'] != speaker)].iloc[0: prime_count]['w_target'].tolist()\n",
    "            else: \n",
    "                word_list = df_ms[(df_ms['group'] == group_name) & \n",
    "                                    (df_ms['Speaker'] != speaker)].iloc[prime_count-n: prime_count]['w_target'].tolist()\n",
    "            word_list_reformat = []\n",
    "            for l in word_list: \n",
    "                word_list_reformat.extend(l) # add it to the reformatted list      \n",
    "\n",
    "            prime_dict[index1] = word_list_reformat # syntax list for prime                 \n",
    "    # print(list(prime_dict.items())[:2])\n",
    "    return prime_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add the prime subtree column (10-turn window)\n",
    "ms_df['w_prime_10'] = ms_df.index.map(get_prime_w_list(ms_df, 10)) \n",
    "# replace the first row prime NaN to [] so it can be treated universally\n",
    "ms_df['w_prime_10'] = [ [] if x is np.NaN else x for x in ms_df['w_prime_10'] ] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add the prime subtree column (5-turn window)\n",
    "ms_df['w_prime_5'] = ms_df.index.map(get_prime_w_list(ms_df, 5)) \n",
    "# replace the first row prime NaN to [] so it can be treated universally\n",
    "ms_df['w_prime_5'] = [ [] if x is np.NaN else x for x in ms_df['w_prime_5'] ] \n",
    "# df_ms.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.3 Calculate the LILLA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# find the overlap items in two lists (target and prime)\n",
    "# calculate the length of utterances\n",
    "def find_overlap(df_ms, prime_window):\n",
    "\n",
    "    prime_column = 'w_prime_'+ str(prime_window) # get the column name based on the prime_window chosed \n",
    "    len_prime_column = 'len_prime_' + str(prime_window)\n",
    "    \n",
    "    print(prime_column)\n",
    "    df_ms['overlap_count'] = 0\n",
    "    df_ms['len_target'] = 0\n",
    "    df_ms['len_prime'] = 0\n",
    "\n",
    "    for index, row in df_ms.iterrows():\n",
    "        n=0\n",
    "        list1 = row[prime_column]\n",
    "        list2 = row['w_target']\n",
    "        # print(type(list1), type(list2))\n",
    "        for i in list1:\n",
    "            \n",
    "            if i in list2: # if the item in list1 belongs to list 2, then n++\n",
    "                n+=1\n",
    "        # add columns (column manipulation: set use df.at, get use df.loc)\n",
    "        \n",
    "        df_ms.at[index,'overlap_count'] = n # set number of items that overlapped between prime and target\n",
    "        df_ms.at[index,'len_target'] = len(list2) # set length of target utterance\n",
    "        df_ms.at[index,'len_prime'] = len(list1) # set length of prime utterance\n",
    "    \n",
    "    return len_prime_column # keep the len_prime column name for future use "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# specify your prime window, currently is 10 \n",
    "find_overlap(ms_df, 10)\n",
    "ms_df[18:25]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the SILLA = p(target|prime)/p(target) = number of elements overlapped / (len_target*len_prime)\n",
    "ms_df['len_prime_target'] = ms_df['len_target'] * ms_df['len_prime']\n",
    "ms_df['lilla'] = ms_df['overlap_count']/ms_df['len_prime_target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_ms.to_csv(r'middle_school_SILLA_uncleaned.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_ms.to_csv(r'test_syntax.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.4 Calculate Normalized LLA (nLLA)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.4.1 compute $\\bar {LLA}$\n",
    "i.e., The average LLA for all pairs that have the same product of length, and for all possible product values n.\n",
    "Outcome of D1 should be a dictionary which the key to be product values (n), value to be the average LLA of all pairs which have that product values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reitter paper idea: normalize by the average LLA for the same product of length, which is len(Prime)*len(target)\n",
    "# \"product of length\" column: len_prime_target\n",
    "grouped_element_length = ms_df.groupby('len_prime_target') # group the df by utternace length \n",
    "print(len(grouped_element_length)) # total of 1657 unique len(Prime)*len(target) for ms, 946 for ug\n",
    "# 1539 unique len(Prime)*len(target) on lexicon level for ms\n",
    "\n",
    "avg_lla_list = {} # key = length, value = average_lla, use this to add the column to df later\n",
    "for l in grouped_element_length: # l is a tuple object, l[0] is the element length, l[1] is the subset dataframe\n",
    "    avg_lla = l[1][\"lilla\"].mean()\n",
    "    avg_lla_list[l[0]] = avg_lla\n",
    "print(list(avg_lla_list.items())[:5])   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add avg_lla dictionary to the df as a column, mapping by len(Prime)*len(target)\n",
    "ms_df['avg_lilla'] = ms_df.len_prime_target.map(avg_lla_list)\n",
    "ms_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.4.2 compute nLLA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the nLLA = LLA / avg_lla\n",
    "ms_df['nlilla'] = ms_df['lilla']/ms_df['avg_lilla'] \n",
    "ms_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ms_df.to_pickle(\"./LILLA_scores.pkl\") # change the file name if you want"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Visual inspection "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the data\n",
    "df_silla = pd.read_pickle(\"../data/2_middle school silla/ms_7812_nsilla_0414_dist10.pkl\")\n",
    "df_lilla = pd.read_pickle(\"../data/6_lilla/ms_7812_lilla_dist10.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.count_nonzero(ms_df['overlap_count'], axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.1 Distribution of SILLA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ms_df = ms_df.replace([np.inf, -np.inf], 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot the distribution of SILLA scores\n",
    "plt.hist(ms_df['lilla'], bins = 400)\n",
    "plt.xlim(-0.01, 0.1)\n",
    "plt.title('LILLA distribution - ms')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove zero\n",
    "plt.hist(ms_df[ms_df['lilla'] != 0]['lilla'], bins = 400)\n",
    "plt.xlim(-0.01, 0.15)\n",
    "plt.title('LILLA distribution - ms - nonzero only')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# count percentage of zero values of silla\n",
    "ms_df['lilla'].value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.2 Distribution of Normalized LLA (nLLA)\n",
    "The distribution shape changed a bit, the scale changed, general trends look similar."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot the distribution of normalized SILLA scores\n",
    "plt.hist(ms_df['nlilla'], bins = 120)\n",
    "plt.xlim(-0.2, 8)\n",
    "plt.title('nLILLA distribution - ms')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot the distribution of normalized SILLA scores\n",
    "plt.hist(ms_df[ms_df['nlilla'] != 0]['nlilla'], bins = 120)\n",
    "plt.xlim(-0.2, 8)\n",
    "plt.title('nLILLA distribution - ms - nonzero only')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.3 Distribution of LLA by groups"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.3.1  Distribution of LILLA by groups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grouped = ms_df.groupby('group')\n",
    "# print(list(grouped))\n",
    "\n",
    "for group in grouped:\n",
    "  # figure()\n",
    "  print(group[0])\n",
    "  group[1].lilla.plot.hist(bins = 200, xlim = (-0.01, 0.1), figsize=(3,3))\n",
    "  # plot.hist(group[1].N)\n",
    "  plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.3.2  Distribution of nLLA by groups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for group in grouped:\n",
    "  # figure()\n",
    "  print(group[0])\n",
    "  group[1].nlla.plot.hist(bins = 80, xlim = (-0.5, 6), figsize=(3,3))\n",
    "  # plot.hist(group[1].N)\n",
    "  plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.4 distribution of sentence length, relationship between sentence length and LLA "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot the distribution of sentence length\n",
    "plt.hist(ms_df['len_target'], bins = 200)\n",
    "plt.xlim(-1, 80)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# any patterns between sentence length and SILLA score\n",
    "x = ms_df['len_target']\n",
    "y = ms_df['lilla']\n",
    "plt.scatter(x,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# any patterns between sentence length and overlap count\n",
    "plt.scatter(ms_df['len_target'],ms_df['overlap_count'])\n",
    "m, b = np.polyfit(ms_df['len_target'], ms_df['overlap_count'], 1)\n",
    "plt.plot(ms_df['len_target'], m*ms_df['len_target'] + b,color = 'green')\n",
    "print('slop: ', m, 'intercept: ', b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# take a granular look \n",
    "plt.scatter(ms_df['len_target'],ms_df['overlap_count'])\n",
    "plt.xlim(0, 80)\n",
    "plt.ylim(-1, 120)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.5 Distribution of words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot the distribution of syntax subtrees\n",
    "\n",
    "# Create dictionary\n",
    "dict_freq = {}\n",
    "# Add syntax rules (subtrees) to dictionary\n",
    "for index, row in ms_df.iterrows():\n",
    "    list3 = row['w_target'] # list3 is subtress in one utterance\n",
    "    # print(list3)\n",
    "    # list is unhashable, convert list to tuple\n",
    "    for r in list3:   # r is each subtree\n",
    "        if r not in dict_freq:\n",
    "            dict_freq[r] = 0\n",
    "        dict_freq[r] += 1\n",
    "word_freq_list = [(v,k) for k,v in dict_freq.items()]\n",
    "freq_list_sorted = sorted(word_freq_list,reverse=True)\n",
    "print(freq_list_sorted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.book import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# take a look at the syntax subtree frequency dictionary \n",
    "print('unique words: ',len(dict_freq))\n",
    "print('total word freq (for all utterances):', sum(dict_freq.values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fdist = FreqDist(dict_freq)\n",
    "\n",
    "# print the top most common 50 words, using `.most_common(50)`\n",
    "# [Insert code here]\n",
    "fdist.most_common(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fdist.plot(40)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. Merge SILLA and LILLA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the data\n",
    "df_silla = pd.read_pickle(\"your silla file\")\n",
    "df_lilla = pd.read_pickle(\"your lilla file\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_lilla['w_len_target'] = df_lilla['len_target']\n",
    "df_lilla = df_lilla[['w_target', 'lilla', 'nlilla', 'w_len_target']]\n",
    "df_lilla[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_silla['syn_len_target'] = df_silla['len_target']\n",
    "df_silla['silla'] = df_silla['lla']\n",
    "df_silla['nsilla'] = df_silla['nlla']\n",
    "df_silla['group_speaker'] = df_silla['group'] + df_silla['Speaker']\n",
    "df_silla = df_silla[['Text', 'group', 'group_speaker', 'group_turn_id', 'syntax_tree_current', 'silla', 'nsilla', 'syn_len_target']]\n",
    "df_silla[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_merge = pd.merge(df_silla, df_lilla, left_index=True, right_index=True)\n",
    "df_merge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add the ratio \n",
    "df_merge['silla_lilla'] = df_merge['silla']/df_merge['lilla']\n",
    "df_merge['nsilla_nlilla'] = df_merge['nsilla']/df_merge['nlilla']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# output for other analysis\n",
    "df_merge.to_pickle(\"./ms_merged_silla_lilla.pkl\")\n",
    "df_merge.to_csv(r'ms_merged_silla_lilla.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## plot silla and lilla"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ms = pd.read_pickle(\"ms_merged_silla_lilla.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ms[['silla','lilla','nlilla','w_len_target']].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.scatter(df_merge['silla'],df_merge['lilla'])\n",
    "plt.xlim(-0.01, 0.2)\n",
    "plt.ylim(-0.01, 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# deal with NaN values\n",
    "grp = df_ms\n",
    "grp['lilla'].isna().sum() \n",
    "# replace NaN values to the median (1)\n",
    "grp['lilla'] = grp['lilla'].replace(np.nan, 0.01046)#  0.01046 for ms lilla\n",
    "# df_ms['nlla'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y=grp[['lilla']].to_numpy().flatten()\n",
    "ysmoothed = gaussian_filter1d(y, sigma=10)\n",
    "sns.lineplot(data=grp, x='group_turn_id', y=ysmoothed)\n",
    "plt.xlim(-10, 250)\n",
    "# plt.ylim(0.018, 0.033)\n",
    "plt.title('MS lilla over time for all groups')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot the distribution of ratio silla/lilla\n",
    "# for cases that lilla == 0, replace by -1\n",
    "ms_df = df_merge.replace([np.inf, -np.inf], -1)\n",
    "# plot\n",
    "plt.hist(ms_df['silla_lilla'], bins = 200)\n",
    "plt.xlim(-2, 15)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# after replacing the inf values to -1, here is the descr stats\n",
    "ms_df['silla_lilla'].describe()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}