similarity_tagging.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a9d4ffb0",
   "metadata": {},
   "source": [
    "### Install necessary packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "dfe89e53",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install texthero==1.0.5\n",
    "# !pip install nltk\n",
    "# !pip install spacy\n",
    "# !python -m spacy download en_core_web_lg"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "05550f78",
   "metadata": {},
   "source": [
    "### Import essential libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "da0de70a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import texthero as hero # cleaning pandas columns\n",
    "import time\n",
    "import nltk\n",
    "import spacy # check similarity of words precisely\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_colwidth', None)\n",
    "nlp = spacy.load('en_core_web_lg')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a54b43a",
   "metadata": {},
   "source": [
    "### Read `praise` and `tag` data into pandas dataframe and perform some cleansing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fc0faf2c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\fmagh\\anaconda3\\lib\\site-packages\\texthero\\preprocessing.py:46: FutureWarning: The default value of regex will change from True to False in a future version.\n",
      "  return input.str.replace(r\"^\\d+\\s|\\s\\d+\\s|\\s\\d+$\", \" \")\n",
      "C:\\Users\\fmagh\\anaconda3\\lib\\site-packages\\texthero\\preprocessing.py:81: FutureWarning: The default value of regex will change from True to False in a future version.\n",
      "  return input.str.replace(pat, '')\n"
     ]
    }
   ],
   "source": [
    "df_praise = pd.read_csv(\"data/sample_praise.csv\")\n",
    "df_tag = pd.read_csv(\"data/tags.csv\")\n",
    "df_praise.iloc[:,5] = hero.clean(df_praise.iloc[:,5])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f1ced525",
   "metadata": {},
   "source": [
    "Drop unnecessary columns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "505e221d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_praise = pd.concat([df_praise.iloc[:,0], df_praise.iloc[:,3:15]], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9b6d9be7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Category Code</th>\n",
       "      <th>To</th>\n",
       "      <th>From</th>\n",
       "      <th>Reason for dishing</th>\n",
       "      <th>Server</th>\n",
       "      <th>Date</th>\n",
       "      <th>Room</th>\n",
       "      <th>v1 norm</th>\n",
       "      <th>v2 norm</th>\n",
       "      <th>v3 norm</th>\n",
       "      <th>Avg %</th>\n",
       "      <th>IH per Praise</th>\n",
       "      <th>IH per person</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TEC15</td>\n",
       "      <td>liviade</td>\n",
       "      <td>cranders71</td>\n",
       "      <td>always caring finicky bot valued community members</td>\n",
       "      <td>Telegram</td>\n",
       "      <td>2021-01-13</td>\n",
       "      <td>TE Praise</td>\n",
       "      <td>30</td>\n",
       "      <td>10.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>6.549.137.185</td>\n",
       "      <td>4.911.852.889</td>\n",
       "      <td>6.767.403.323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TEC12</td>\n",
       "      <td>chuygarcia92</td>\n",
       "      <td>Tam2140</td>\n",
       "      <td>masana temples recommendation nice mix low high beats</td>\n",
       "      <td>Token Engineering Commons</td>\n",
       "      <td>2021-04-30</td>\n",
       "      <td>🙏praise</td>\n",
       "      <td>1000</td>\n",
       "      <td>100.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>723.394.157</td>\n",
       "      <td>8.680.729.884</td>\n",
       "      <td>718.127.319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TEC12</td>\n",
       "      <td>iviangita</td>\n",
       "      <td>iviangita</td>\n",
       "      <td>joining legal weekly sync</td>\n",
       "      <td>Token Engineering Commons</td>\n",
       "      <td>2021-04-30</td>\n",
       "      <td>🙏praise</td>\n",
       "      <td>1000</td>\n",
       "      <td>80.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>6.595.463.701</td>\n",
       "      <td>7.914.556.442</td>\n",
       "      <td>2.387.003.435</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TEC12</td>\n",
       "      <td>ygganderson</td>\n",
       "      <td>JessicaZartler</td>\n",
       "      <td>presence energy smiles comms working group today</td>\n",
       "      <td>Telegram</td>\n",
       "      <td>2020-11-24</td>\n",
       "      <td>TE Praise</td>\n",
       "      <td>20</td>\n",
       "      <td>60.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>1.283.215.058</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TEC12</td>\n",
       "      <td>metaverde</td>\n",
       "      <td>iviangita</td>\n",
       "      <td>mentioning retweeting te commons socials past week thank helping us grow token engineering commons community spreading message</td>\n",
       "      <td>Telegram</td>\n",
       "      <td>2021-01-22</td>\n",
       "      <td>TE Commons</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>198.159.943.382.873</td>\n",
       "      <td>3.170.559.094</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Category Code            To            From  \\\n",
       "0         TEC15       liviade      cranders71   \n",
       "1         TEC12  chuygarcia92         Tam2140   \n",
       "2         TEC12     iviangita       iviangita   \n",
       "3         TEC12   ygganderson  JessicaZartler   \n",
       "4         TEC12     metaverde       iviangita   \n",
       "\n",
       "                                                                                                               Reason for dishing  \\\n",
       "0                                                                              always caring finicky bot valued community members   \n",
       "1                                                                           masana temples recommendation nice mix low high beats   \n",
       "2                                                                                                       joining legal weekly sync   \n",
       "3                                                                                presence energy smiles comms working group today   \n",
       "4  mentioning retweeting te commons socials past week thank helping us grow token engineering commons community spreading message   \n",
       "\n",
       "                      Server        Date        Room v1 norm  v2 norm  \\\n",
       "0                   Telegram  2021-01-13   TE Praise      30     10.0   \n",
       "1  Token Engineering Commons  2021-04-30     🙏praise    1000    100.0   \n",
       "2  Token Engineering Commons  2021-04-30     🙏praise    1000     80.0   \n",
       "3                   Telegram  2020-11-24   TE Praise      20     60.0   \n",
       "4                   Telegram  2021-01-22  TE Commons     NaN      NaN   \n",
       "\n",
       "   v3 norm          Avg %        IH per Praise  IH per person  \n",
       "0     40.0  6.549.137.185        4.911.852.889  6.767.403.323  \n",
       "1    100.0    723.394.157        8.680.729.884    718.127.319  \n",
       "2    100.0  6.595.463.701        7.914.556.442  2.387.003.435  \n",
       "3     30.0  1.283.215.058                  NaN            NaN  \n",
       "4      NaN              0  198.159.943.382.873  3.170.559.094  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_praise.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b47e8b06",
   "metadata": {},
   "outputs": [],
   "source": [
    "# reading praise data\n",
    "df_praise = pd.read_csv(\"data/cleaned-non-quantifier-data.csv\", index_col=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "60dbf53b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\fmagh\\anaconda3\\lib\\site-packages\\texthero\\preprocessing.py:46: FutureWarning: The default value of regex will change from True to False in a future version.\n",
      "  return input.str.replace(r\"^\\d+\\s|\\s\\d+\\s|\\s\\d+$\", \" \")\n",
      "C:\\Users\\fmagh\\anaconda3\\lib\\site-packages\\texthero\\preprocessing.py:81: FutureWarning: The default value of regex will change from True to False in a future version.\n",
      "  return input.str.replace(pat, '')\n"
     ]
    }
   ],
   "source": [
    "praise_set = hero.clean(df_praise['Reason for dishing'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "be0165b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_praise_set = pd.DataFrame(praise_set).reset_index()\n",
    "df_praise_set.columns = [0,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "bd54df33",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>hosting kicking params party</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>hosting leading lot params parties</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>testing deploying bot record display meetings</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>huge success mvv process</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>awesome work recorder bot help documentation always getting recordings going pushing transparency wg good personality</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7620</th>\n",
       "      <td>7620</td>\n",
       "      <td>presence great contributions soft gov wg sync</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7621</th>\n",
       "      <td>7621</td>\n",
       "      <td>participating praisegiving call</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7622</th>\n",
       "      <td>7622</td>\n",
       "      <td>sharing gitcoin blog post lots great info https twitter com owocki status 1333842278578679809</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7623</th>\n",
       "      <td>7623</td>\n",
       "      <td>joining te praise channel tec lab today</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7624</th>\n",
       "      <td>7624</td>\n",
       "      <td>joining te praise channel tec lab today</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7625 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         0  \\\n",
       "0        0   \n",
       "1        1   \n",
       "2        2   \n",
       "3        3   \n",
       "4        4   \n",
       "...    ...   \n",
       "7620  7620   \n",
       "7621  7621   \n",
       "7622  7622   \n",
       "7623  7623   \n",
       "7624  7624   \n",
       "\n",
       "                                                                                                                          1  \n",
       "0                                                                                              hosting kicking params party  \n",
       "1                                                                                        hosting leading lot params parties  \n",
       "2                                                                             testing deploying bot record display meetings  \n",
       "3                                                                                                  huge success mvv process  \n",
       "4     awesome work recorder bot help documentation always getting recordings going pushing transparency wg good personality  \n",
       "...                                                                                                                     ...  \n",
       "7620                                                                          presence great contributions soft gov wg sync  \n",
       "7621                                                                                        participating praisegiving call  \n",
       "7622                          sharing gitcoin blog post lots great info https twitter com owocki status 1333842278578679809  \n",
       "7623                                                                                joining te praise channel tec lab today  \n",
       "7624                                                                                joining te praise channel tec lab today  \n",
       "\n",
       "[7625 rows x 2 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_praise_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a8306bc7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tag</th>\n",
       "      <th>keyword</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TEC1</td>\n",
       "      <td>comms article blog organized presentation graphic design website marketing seo social platform discord telegram forum discourse medium linkedin website article retweeting retweet retweets organized presentation communications</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TEC2</td>\n",
       "      <td>culture build soft gov survey vote voting debate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TEC3</td>\n",
       "      <td>params parameters param parties party</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TEC4</td>\n",
       "      <td>legal strategy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TEC5</td>\n",
       "      <td>commons swarm tech dev dapp tech dapp app bug</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>TEC6</td>\n",
       "      <td>hatch outreach onboarding onboard outreach</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>TEC7</td>\n",
       "      <td>omega survey philosophy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>TEC8</td>\n",
       "      <td>stewards github steward project management</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>TEC9</td>\n",
       "      <td>labs lab</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>TEC10</td>\n",
       "      <td>transparency youtube recording record</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>TEC12</td>\n",
       "      <td>participation retweets</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>TEC13</td>\n",
       "      <td>fundraising donating fundraising</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>TEC14</td>\n",
       "      <td>technical infrastructure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>TEC15</td>\n",
       "      <td>tec community building</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>TEC16</td>\n",
       "      <td>personal praise</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>TEC17</td>\n",
       "      <td>gravity conflict nonviolent communication</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      tag  \\\n",
       "0    TEC1   \n",
       "1    TEC2   \n",
       "2    TEC3   \n",
       "3    TEC4   \n",
       "4    TEC5   \n",
       "5    TEC6   \n",
       "6    TEC7   \n",
       "7    TEC8   \n",
       "8    TEC9   \n",
       "9   TEC10   \n",
       "10  TEC12   \n",
       "11  TEC13   \n",
       "12  TEC14   \n",
       "13  TEC15   \n",
       "14  TEC16   \n",
       "15  TEC17   \n",
       "\n",
       "                                                                                                                                                                                                                              keyword  \n",
       "0   comms article blog organized presentation graphic design website marketing seo social platform discord telegram forum discourse medium linkedin website article retweeting retweet retweets organized presentation communications  \n",
       "1                                                                                                                                                                                    culture build soft gov survey vote voting debate  \n",
       "2                                                                                                                                                                                               params parameters param parties party  \n",
       "3                                                                                                                                                                                                                      legal strategy  \n",
       "4                                                                                                                                                                                       commons swarm tech dev dapp tech dapp app bug  \n",
       "5                                                                                                                                                                                          hatch outreach onboarding onboard outreach  \n",
       "6                                                                                                                                                                                                             omega survey philosophy  \n",
       "7                                                                                                                                                                                          stewards github steward project management  \n",
       "8                                                                                                                                                                                                                            labs lab  \n",
       "9                                                                                                                                                                                               transparency youtube recording record  \n",
       "10                                                                                                                                                                                                             participation retweets  \n",
       "11                                                                                                                                                                                                   fundraising donating fundraising  \n",
       "12                                                                                                                                                                                                           technical infrastructure  \n",
       "13                                                                                                                                                                                                             tec community building  \n",
       "14                                                                                                                                                                                                                    personal praise  \n",
       "15                                                                                                                                                                                          gravity conflict nonviolent communication  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "59d6ca0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Callable\n",
    "\n",
    "def spacy_similarity(doc1, doc2):\n",
    "    return nlp(doc1).similarity(nlp(doc2))\n",
    "\n",
    "def max_intersection_similarity(doc1, doc2):\n",
    "    doc1 = set(doc1.split(\" \"))\n",
    "    doc2 = set(doc2.split(\" \"))\n",
    "    similarity = len(doc1.intersection(doc2))# / len(doc1.union(doc2))\n",
    "    return similarity\n",
    "\n",
    "def jaccard_similarity(doc1, doc2):\n",
    "    doc1 = set(doc1.split(\" \"))\n",
    "    doc2 = set(doc2.split(\" \"))\n",
    "    similarity = len(doc1.intersection(doc2)) / len(doc1.union(doc2))\n",
    "    return similarity\n",
    "\n",
    "def jaccard_label(praise_reason: str, sim_function: Callable):\n",
    "    df_tag['similarity'] = df_tag[\"keyword\"].apply(lambda x: sim_function(x, praise_reason))\n",
    "    max_tag = df_tag.sort_values('similarity',ascending=False).iloc[0]\n",
    "    sim_score = max_tag['similarity']\n",
    "    max_tag = max_tag[0]\n",
    "    if sim_score == 0:\n",
    "        max_tag = np.nan\n",
    "    return max_tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "69eda248",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_praise_set['max_intersection_similarity'] = df_praise_set[1].apply(lambda x: jaccard_label(x, max_intersection_similarity))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e7d19081",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_praise_set['jaccard_similarity'] = df_praise_set[1].apply(lambda x: jaccard_label(x, jaccard_similarity))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d1a5ca70",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-19-532b69d73a29>:4: UserWarning: [W008] Evaluating Doc.similarity based on empty vectors.\n",
      "  return nlp(doc1).similarity(nlp(doc2))\n"
     ]
    }
   ],
   "source": [
    "df_praise_set['spacy_similarity'] = df_praise_set[1].apply(lambda x: jaccard_label(x, spacy_similarity))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "77fcd3d8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>max_intersection_similarity</th>\n",
       "      <th>jaccard_similarity</th>\n",
       "      <th>spacy_similarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>hosting kicking params party</td>\n",
       "      <td>TEC3</td>\n",
       "      <td>TEC3</td>\n",
       "      <td>TEC3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>hosting leading lot params parties</td>\n",
       "      <td>TEC3</td>\n",
       "      <td>TEC3</td>\n",
       "      <td>TEC3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>testing deploying bot record display meetings</td>\n",
       "      <td>TEC10</td>\n",
       "      <td>TEC10</td>\n",
       "      <td>TEC10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>huge success mvv process</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TEC2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>awesome work recorder bot help documentation always getting recordings going pushing transparency wg good personality</td>\n",
       "      <td>TEC10</td>\n",
       "      <td>TEC10</td>\n",
       "      <td>TEC10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7620</th>\n",
       "      <td>7620</td>\n",
       "      <td>presence great contributions soft gov wg sync</td>\n",
       "      <td>TEC2</td>\n",
       "      <td>TEC2</td>\n",
       "      <td>TEC2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7621</th>\n",
       "      <td>7621</td>\n",
       "      <td>participating praisegiving call</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TEC2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7622</th>\n",
       "      <td>7622</td>\n",
       "      <td>sharing gitcoin blog post lots great info https twitter com owocki status 1333842278578679809</td>\n",
       "      <td>TEC1</td>\n",
       "      <td>TEC1</td>\n",
       "      <td>TEC1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7623</th>\n",
       "      <td>7623</td>\n",
       "      <td>joining te praise channel tec lab today</td>\n",
       "      <td>TEC9</td>\n",
       "      <td>TEC9</td>\n",
       "      <td>TEC15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7624</th>\n",
       "      <td>7624</td>\n",
       "      <td>joining te praise channel tec lab today</td>\n",
       "      <td>TEC9</td>\n",
       "      <td>TEC9</td>\n",
       "      <td>TEC15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7625 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         0  \\\n",
       "0        0   \n",
       "1        1   \n",
       "2        2   \n",
       "3        3   \n",
       "4        4   \n",
       "...    ...   \n",
       "7620  7620   \n",
       "7621  7621   \n",
       "7622  7622   \n",
       "7623  7623   \n",
       "7624  7624   \n",
       "\n",
       "                                                                                                                          1  \\\n",
       "0                                                                                              hosting kicking params party   \n",
       "1                                                                                        hosting leading lot params parties   \n",
       "2                                                                             testing deploying bot record display meetings   \n",
       "3                                                                                                  huge success mvv process   \n",
       "4     awesome work recorder bot help documentation always getting recordings going pushing transparency wg good personality   \n",
       "...                                                                                                                     ...   \n",
       "7620                                                                          presence great contributions soft gov wg sync   \n",
       "7621                                                                                        participating praisegiving call   \n",
       "7622                          sharing gitcoin blog post lots great info https twitter com owocki status 1333842278578679809   \n",
       "7623                                                                                joining te praise channel tec lab today   \n",
       "7624                                                                                joining te praise channel tec lab today   \n",
       "\n",
       "     max_intersection_similarity jaccard_similarity spacy_similarity  \n",
       "0                           TEC3               TEC3             TEC3  \n",
       "1                           TEC3               TEC3             TEC3  \n",
       "2                          TEC10              TEC10            TEC10  \n",
       "3                            NaN                NaN             TEC2  \n",
       "4                          TEC10              TEC10            TEC10  \n",
       "...                          ...                ...              ...  \n",
       "7620                        TEC2               TEC2             TEC2  \n",
       "7621                         NaN                NaN             TEC2  \n",
       "7622                        TEC1               TEC1             TEC1  \n",
       "7623                        TEC9               TEC9            TEC15  \n",
       "7624                        TEC9               TEC9            TEC15  \n",
       "\n",
       "[7625 rows x 5 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_praise_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "id": "b568c691",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_praise_set.to_csv('outputs/similarity_tagging.csv',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "10a0947d",
   "metadata": {},
   "source": [
    "### `check_freq` function returns most frequent tag for each row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "c74c3cb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_freq(tag_list):\n",
    "    final_tag_list = []\n",
    "    try:\n",
    "        str_ = ''\n",
    "        for item in tag_list:\n",
    "            if item is not np.nan:\n",
    "                str_ = str_ +\" \"+ item\n",
    "        moby_tokens = nltk.word_tokenize(str_)\n",
    "        text = nltk.Text(moby_tokens)\n",
    "\n",
    "        dist = nltk.FreqDist(text)\n",
    "        freq_lst = [k for k, v in sorted(dist.items(), key=lambda i: i[1], reverse=True)]\n",
    "        final_tag_list.append(freq_lst[0])\n",
    "        return final_tag_list[0]\n",
    "    except:\n",
    "        return []"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0a812a8",
   "metadata": {},
   "source": [
    "### Creates a single tag column with most frequent tag from all other 3 tags and creats a complete csv file for furture analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "d6cc687d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\fmagh\\anaconda3\\lib\\site-packages\\texthero\\preprocessing.py:46: FutureWarning: The default value of regex will change from True to False in a future version.\n",
      "  return input.str.replace(r\"^\\d+\\s|\\s\\d+\\s|\\s\\d+$\", \" \")\n",
      "C:\\Users\\fmagh\\anaconda3\\lib\\site-packages\\texthero\\preprocessing.py:81: FutureWarning: The default value of regex will change from True to False in a future version.\n",
      "  return input.str.replace(pat, '')\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>To</th>\n",
       "      <th>From</th>\n",
       "      <th>Reason for dishing</th>\n",
       "      <th>Server</th>\n",
       "      <th>Date</th>\n",
       "      <th>Room</th>\n",
       "      <th>v1 norm</th>\n",
       "      <th>v2 norm</th>\n",
       "      <th>v3 norm</th>\n",
       "      <th>Avg %</th>\n",
       "      <th>IH per Praise</th>\n",
       "      <th>IH per person</th>\n",
       "      <th>tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>zeptimusQ</td>\n",
       "      <td>Tam2140</td>\n",
       "      <td>hosting kicking params party</td>\n",
       "      <td>Token Engineering Commons</td>\n",
       "      <td>2021-05-07</td>\n",
       "      <td>praise</td>\n",
       "      <td>10000.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>200.0</td>\n",
       "      <td>0.001963</td>\n",
       "      <td>2.355252</td>\n",
       "      <td>39.04072</td>\n",
       "      <td>TEC3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>zeptimusQ</td>\n",
       "      <td>iviangita</td>\n",
       "      <td>hosting leading lot params parties</td>\n",
       "      <td>Token Engineering Commons</td>\n",
       "      <td>2021-05-07</td>\n",
       "      <td>praise</td>\n",
       "      <td>10000.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.001663</td>\n",
       "      <td>1.995539</td>\n",
       "      <td>39.04072</td>\n",
       "      <td>TEC3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>zeptimusQ</td>\n",
       "      <td>JuankBell</td>\n",
       "      <td>testing deploying bot record display meetings</td>\n",
       "      <td>Token Engineering Commons</td>\n",
       "      <td>2021-04-28</td>\n",
       "      <td>praise</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>200.0</td>\n",
       "      <td>200.0</td>\n",
       "      <td>0.001342</td>\n",
       "      <td>1.610872</td>\n",
       "      <td>39.04072</td>\n",
       "      <td>TEC10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>zeptimusQ</td>\n",
       "      <td>iviangita</td>\n",
       "      <td>huge success mvv process</td>\n",
       "      <td>Token Engineering Commons</td>\n",
       "      <td>2021-04-30</td>\n",
       "      <td>praise</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>200.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.001043</td>\n",
       "      <td>1.251160</td>\n",
       "      <td>39.04072</td>\n",
       "      <td>TEC2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>zeptimusQ</td>\n",
       "      <td>iviangita</td>\n",
       "      <td>awesome work recorder bot help documentation always getting recordings going pushing transparency wg good personality</td>\n",
       "      <td>Token Engineering Commons</td>\n",
       "      <td>2021-04-30</td>\n",
       "      <td>praise</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>200.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.001043</td>\n",
       "      <td>1.251160</td>\n",
       "      <td>39.04072</td>\n",
       "      <td>TEC10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7620</th>\n",
       "      <td>atacas</td>\n",
       "      <td>liviade</td>\n",
       "      <td>presence great contributions soft gov wg sync</td>\n",
       "      <td>Telegram</td>\n",
       "      <td>2020-12-01</td>\n",
       "      <td>te praise</td>\n",
       "      <td>30.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>0.001534</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TEC2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7621</th>\n",
       "      <td>Luukweber</td>\n",
       "      <td>iviangita</td>\n",
       "      <td>participating praisegiving call</td>\n",
       "      <td>Telegram</td>\n",
       "      <td>2020-11-28</td>\n",
       "      <td>te praise</td>\n",
       "      <td>40.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>0.001180</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TEC2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7622</th>\n",
       "      <td>krrisis</td>\n",
       "      <td>GriffGreen</td>\n",
       "      <td>sharing gitcoin blog post lots great info https twitter com owocki status 1333842278578679809</td>\n",
       "      <td>Telegram</td>\n",
       "      <td>2020-12-02</td>\n",
       "      <td>commons stack</td>\n",
       "      <td>10.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>0.001103</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TEC1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7623</th>\n",
       "      <td>sbilbao</td>\n",
       "      <td>JessicaZartler</td>\n",
       "      <td>joining te praise channel tec lab today</td>\n",
       "      <td>Telegram</td>\n",
       "      <td>2020-11-27</td>\n",
       "      <td>te praise</td>\n",
       "      <td>50.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.001005</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TEC9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7624</th>\n",
       "      <td>CryptononymousEconomist</td>\n",
       "      <td>JessicaZartler</td>\n",
       "      <td>joining te praise channel tec lab today</td>\n",
       "      <td>Telegram</td>\n",
       "      <td>2020-11-27</td>\n",
       "      <td>te praise</td>\n",
       "      <td>50.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.001005</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TEC9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7625 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                           To            From  \\\n",
       "0                   zeptimusQ         Tam2140   \n",
       "1                   zeptimusQ       iviangita   \n",
       "2                   zeptimusQ       JuankBell   \n",
       "3                   zeptimusQ       iviangita   \n",
       "4                   zeptimusQ       iviangita   \n",
       "...                       ...             ...   \n",
       "7620                   atacas         liviade   \n",
       "7621                Luukweber       iviangita   \n",
       "7622                  krrisis      GriffGreen   \n",
       "7623                  sbilbao  JessicaZartler   \n",
       "7624  CryptononymousEconomist  JessicaZartler   \n",
       "\n",
       "                                                                                                         Reason for dishing  \\\n",
       "0                                                                                              hosting kicking params party   \n",
       "1                                                                                        hosting leading lot params parties   \n",
       "2                                                                             testing deploying bot record display meetings   \n",
       "3                                                                                                  huge success mvv process   \n",
       "4     awesome work recorder bot help documentation always getting recordings going pushing transparency wg good personality   \n",
       "...                                                                                                                     ...   \n",
       "7620                                                                          presence great contributions soft gov wg sync   \n",
       "7621                                                                                        participating praisegiving call   \n",
       "7622                          sharing gitcoin blog post lots great info https twitter com owocki status 1333842278578679809   \n",
       "7623                                                                                joining te praise channel tec lab today   \n",
       "7624                                                                                joining te praise channel tec lab today   \n",
       "\n",
       "                         Server        Date           Room  v1 norm  v2 norm  \\\n",
       "0     Token Engineering Commons  2021-05-07         praise  10000.0    100.0   \n",
       "1     Token Engineering Commons  2021-05-07         praise  10000.0    100.0   \n",
       "2     Token Engineering Commons  2021-04-28         praise   1000.0    200.0   \n",
       "3     Token Engineering Commons  2021-04-30         praise   1000.0    200.0   \n",
       "4     Token Engineering Commons  2021-04-30         praise   1000.0    200.0   \n",
       "...                         ...         ...            ...      ...      ...   \n",
       "7620                   Telegram  2020-12-01      te praise     30.0     60.0   \n",
       "7621                   Telegram  2020-11-28      te praise     40.0     30.0   \n",
       "7622                   Telegram  2020-12-02  commons stack     10.0     40.0   \n",
       "7623                   Telegram  2020-11-27      te praise     50.0     30.0   \n",
       "7624                   Telegram  2020-11-27      te praise     50.0     30.0   \n",
       "\n",
       "      v3 norm     Avg %  IH per Praise  IH per person    tag  \n",
       "0       200.0  0.001963       2.355252       39.04072   TEC3  \n",
       "1       100.0  0.001663       1.995539       39.04072   TEC3  \n",
       "2       200.0  0.001342       1.610872       39.04072  TEC10  \n",
       "3       100.0  0.001043       1.251160       39.04072   TEC2  \n",
       "4       100.0  0.001043       1.251160       39.04072  TEC10  \n",
       "...       ...       ...            ...            ...    ...  \n",
       "7620     40.0  0.001534            NaN            NaN   TEC2  \n",
       "7621     30.0  0.001180            NaN            NaN   TEC2  \n",
       "7622     40.0  0.001103            NaN            NaN   TEC1  \n",
       "7623     10.0  0.001005            NaN            NaN   TEC9  \n",
       "7624     10.0  0.001005            NaN            NaN   TEC9  \n",
       "\n",
       "[7625 rows x 13 columns]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_praise['tag'] = [check_freq([*a]) for a in zip(df_praise_set.iloc[:,2], df_praise_set.iloc[:,3], df_praise_set.iloc[:,4])]\n",
    "df_tagged_praise = pd.concat([df_praise.iloc[:,:12], df_praise.iloc[:,-1]], axis=1)\n",
    "df_tagged_praise['Reason for dishing'] = hero.clean(df_tagged_praise['Reason for dishing'])\n",
    "df_tagged_praise['Room'] = hero.clean(df_tagged_praise['Room'])\n",
    "\n",
    "df_tagged_praise.to_csv(\"data/cleaned_tagged_praise.csv\")\n",
    "df_tagged_praise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a4e36f7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "e0704373",
   "metadata": {},
   "source": [
    "## Below this point is just workspace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b739f92",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "    \n",
    "\n",
    "praise = pd.DataFrame(praise_set)\n",
    "\n",
    "#praise['spacy_similarity'] = praise[1].apply(lambda x: jaccard_label(x, spacy_similarity))\n",
    "praise['max_intersection_similarity'] = praise[1].apply(lambda x: jaccard_label(x, max_intersection_similarity))\n",
    "#praise['jaccard_similarity'] = praise[1].apply(lambda x: jaccard_label(x, jaccard_similarity))\n",
    "\n",
    "praise"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "77d77086",
   "metadata": {},
   "source": [
    "### Iterating through all rows and tag data just for equal words(100% similarity based on tag dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e385aa07",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{('TEC1',\n",
       "  'comms article blog organized presentation graphic design website marketing seo social platform discord telegram forum discourse medium linkedin website article retweeting retweet retweets organized presentation communications'),\n",
       " ('TEC10', 'transparency youtube recording record'),\n",
       " ('TEC12', 'participation retweets'),\n",
       " ('TEC13', 'fundraising donating fundraising'),\n",
       " ('TEC14', 'technical infrastructure'),\n",
       " ('TEC15', 'tec community building'),\n",
       " ('TEC16', 'personal praise'),\n",
       " ('TEC17', 'gravity conflict nonviolent communication'),\n",
       " ('TEC2', 'culture build soft gov survey vote voting debate'),\n",
       " ('TEC3', 'params parameters param parties party'),\n",
       " ('TEC4', 'legal strategy'),\n",
       " ('TEC5', 'commons swarm tech dev dapp tech dapp app bug'),\n",
       " ('TEC6', 'hatch outreach onboarding onboard outreach'),\n",
       " ('TEC7', 'omega survey philosophy'),\n",
       " ('TEC8', 'stewards github steward project management'),\n",
       " ('TEC9', 'labs lab')}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tag_dict = set()\n",
    "for index_, row in df_tag.iterrows():\n",
    "    tag_dict.add((row[0], row[1]))\n",
    "    \n",
    "tag_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "170ed827",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{(0, 'always caring finicky bot valued community members'),\n",
       " (1, 'masana temples recommendation nice mix low high beats'),\n",
       " (2, 'joining legal weekly sync'),\n",
       " (3, 'presence energy smiles comms working group today'),\n",
       " (4,\n",
       "  'mentioning retweeting te commons socials past week thank helping us grow token engineering commons community spreading message'),\n",
       " (5,\n",
       "  'engaging discussion te commons forum https forum tecommons org past week thank helping token engineering commons community share learn'),\n",
       " (6,\n",
       "  'mentioning retweeting te commons socials past week thank helping us grow token engineering commons community spreading message'),\n",
       " (7, 'insightful well formed presentation gravity today'),\n",
       " (8, 'conducting giving feedback audit'),\n",
       " (9, 'last session gravity incredible whole ecosystem'),\n",
       " (10,\n",
       "  'joining hack session staying great discussion anarchy vs community governance'),\n",
       " (11, 'bringing board zenhub team'),\n",
       " (12, 'joining tec community call https www youtube com watch v dhnrqaow3ic'),\n",
       " (13, 'drafting audit proposal amazing'),\n",
       " (14, 'great energy thought provoking conversation legal wg weekly sync'),\n",
       " (15,\n",
       "  'amazing collaborations interest gravity project take care helping set strong basements high quality standards application alternative dispute resolution daos ethical professional responsibility'),\n",
       " (16, 'hours meeting gardens prep weekly recap'),\n",
       " (17,\n",
       "  'always available help work gravity living life graviton inspiring everyone incredible work gravity work gravity working around eth colombia community always really cool encouraging'),\n",
       " (18, 'awesome work gitbook contributions giving even new'),\n",
       " (19, 'griffgreen guiding scholarship request'),\n",
       " (20, 'joining book club last week'),\n",
       " (21,\n",
       "  'mentioning retweeting commons stack socials past week thank helping us grow commons stack community spreading message'),\n",
       " (22, 'awesome people interacted juan recently conversations amazing'),\n",
       " (23, 'test'),\n",
       " (24,\n",
       "  'delightful blogpost https medium com token engineering commons economy choice v2 4f7c9fb42d99'),\n",
       " (25, 'great conversations dan helping trusted seed'),\n",
       " (26,\n",
       "  'mentioning us retweeting us socials december thank helping us growing commons stack community spreading message'),\n",
       " (27, 'productive efficient onboarding wg weekly sync'),\n",
       " (28, 'dropping comms call'),\n",
       " (29, 'contributing much community'),\n",
       " (30, 'warm welcome gave tam'),\n",
       " (31, 'zeptimus gabi joining commons swarm weekly sync'),\n",
       " (32,\n",
       "  'willszal 6405 mentioning retweeting commons stack socials past week thank helping us grow commons stack community spreading message'),\n",
       " (33, 'joining stewards call'),\n",
       " (34, 'joining legal weekly sync'),\n",
       " (35, 'joining commons stack simulator sync continuing improve game'),\n",
       " (36, 'joining community stewards sync yesterday'),\n",
       " (37, 'great work part commons simulator deep dive'),\n",
       " (38, 'attending today impact hours deep dive looking forward parameter sets'),\n",
       " (39,\n",
       "  'donating commons simulator grant btw watch email week special invite first play game completely done'),\n",
       " (40,\n",
       "  \"joining legal wg discuss meeting attendance terms idea audit ' amazing put practice incredible work ' transparency going beyond scope\"),\n",
       " (41, 'great input todays gravity call'),\n",
       " (42, 'reviewing dapp critical legal eye'),\n",
       " (43,\n",
       "  'mentioning retweeting commons stack socials past week thank helping us grow commons stack community spreading message'),\n",
       " (44, 'helping suga taking block'),\n",
       " (45, 'joining community stewards sync yesterday'),\n",
       " (46, 'visible community work hatch super cool'),\n",
       " (47, 'starting lively dialogue around governance'),\n",
       " (48, 'contributing latest tec praise quant blog'),\n",
       " (49,\n",
       "  'creating praise system huge pillar gratitude culture implementing tec via contribution reward proposal early days cultural build lucky human based approach rewards going beyond concrete contributions also acknowledging subjective care work ones'),\n",
       " (50, 'helping push gitcoin things comms'),\n",
       " (51, 'tackling remaining issues swiss membership dapp'),\n",
       " (52,\n",
       "  'sharing decentralized learning journey much honesty humor open contributions graviton training'),\n",
       " (53, 'overall presence stability mind work gravity'),\n",
       " (54,\n",
       "  'coming recording gardens swarm call every time writing transparency wg forum post https forum tecommons org transparency working group 197'),\n",
       " (55,\n",
       "  'annamarieswan freedumbs00 natesuits juankbell mateodaza griff santigs suga zeptimus amwfund durgadas tam2140 chuygarcia92 joining soft gov always insightful spend time'),\n",
       " (56, 'joining tec weekly sync'),\n",
       " (57,\n",
       "  'joining commons stack simulator sync continuing push project finish line'),\n",
       " (58,\n",
       "  'willszal 6405 mentioning retweeting commons stack socials past week thank helping us grow commons stack community spreading message'),\n",
       " (59, 'joining book club last week'),\n",
       " (60, 'starting cadcad bootcamp study group'),\n",
       " (61, 'joining tec weekly sync'),\n",
       " (62,\n",
       "  'giving great talk ethdenver representing tec ethdenver onboarding lot people wednesday call participatory jake onboarding still focused ethdenver hosting ama call'),\n",
       " (63, 'joining call helping onboarding auryn'),\n",
       " (64, 'reaching help trusted seed membership dues going'),\n",
       " (65, 'understanding intentions moving space'),\n",
       " (66,\n",
       "  'mission vision value tokenlog waterfall griff land america sprint planning session'),\n",
       " (67, 'joining mission token log hack session'),\n",
       " (68, 'getting us hatch handling applications hatchers'),\n",
       " (69, 'fascinating informational deep dive dandelion voting parameters'),\n",
       " (70, 'presence great contributions soft gov wg sync'),\n",
       " (71,\n",
       "  'mentioning retweeting te commons socials past week thank helping us grow token engineering commons community spreading message'),\n",
       " (72, 'work decentralizing data markets'),\n",
       " (73,\n",
       "  'joining commons stack simulator sync continuing push project finish line'),\n",
       " (74, 'amazing panel daos yesterday livia'),\n",
       " (75,\n",
       "  'donating commons stack iteration gitcoin grant helping us raise 14 000 dai build future high functioning public goods focused economies'),\n",
       " (76, 'work gravity'),\n",
       " (77, 'coming soft gov call last week sorry late praise'),\n",
       " (78,\n",
       "  'mentioning retweeting commons stack socials past week thank helping us grow commons stack community spreading message'),\n",
       " (79, 'great work organizing gitbook'),\n",
       " (80, 'dank memes meme party'),\n",
       " (81, 'working hatch backend'),\n",
       " (82,\n",
       "  'mentioning retweeting commons stack socials past week thank helping us grow commons stack community spreading message'),\n",
       " (83, 'lot communication juan strengthens trust relationship'),\n",
       " (84, 'rallying around last push get hash contracts frozen'),\n",
       " (85, 'great new landing page'),\n",
       " (86, 'cs dev diagrams'),\n",
       " (87, 'awesome article automating ostrom effective dao management'),\n",
       " (88,\n",
       "  'work tec test hatch dao spec https forum tecommons org tec test hatch implementation specification 226'),\n",
       " (89, 'donating commons stack community fund panvala league grant'),\n",
       " (90, \"lot work ' see\"),\n",
       " (91, 'joining tec weekly sync'),\n",
       " (92,\n",
       "  'starting getting stuck reverse engineering unravelling compound finance'),\n",
       " (93, 'test'),\n",
       " (94, 'joining tec community call https www youtube com watch v s8rqw7bf8gk'),\n",
       " (95,\n",
       "  'donating commons stack iteration gitcoin grant helping us raise 14 000 dai build future high functioning public goods focused economies'),\n",
       " (96, 'continuing hard work params offering support comms well rock'),\n",
       " (97, 'putting heads finding bug'),\n",
       " (98, 'joining tec weekly sync https www youtube com watch v efhuvu oc0'),\n",
       " (99, 'showing forum posts contributing')}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "praise_set = set()\n",
    "for index_, row in df_praise.iloc[:100,:].iterrows():\n",
    "    praise_set.add((index_, row[3]))\n",
    "    \n",
    "praise_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "d829831b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{('TEC1',\n",
       "  'comms article blog organized presentation graphic design website marketing seo social platform discord telegram forum discourse medium linkedin website article retweeting retweet retweets organized presentation communications'),\n",
       " ('TEC10', 'transparency youtube recording record'),\n",
       " ('TEC12', 'participation retweets'),\n",
       " ('TEC13', 'fundraising donating fundraising'),\n",
       " ('TEC14', 'technical infrastructure'),\n",
       " ('TEC15', 'tec community building'),\n",
       " ('TEC16', 'personal praise'),\n",
       " ('TEC17', 'gravity conflict nonviolent communication'),\n",
       " ('TEC2', 'culture build soft gov survey vote voting debate'),\n",
       " ('TEC3', 'params parameters param parties party'),\n",
       " ('TEC4', 'legal strategy'),\n",
       " ('TEC5', 'commons swarm tech dev dapp tech dapp app bug'),\n",
       " ('TEC6', 'hatch outreach onboarding onboard outreach'),\n",
       " ('TEC7', 'omega survey philosophy'),\n",
       " ('TEC8', 'stewards github steward project management'),\n",
       " ('TEC9', 'labs lab')}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tag_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "7b484ed1",
   "metadata": {},
   "outputs": [],
   "source": [
    "a = {1,2,3}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "b16004e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "8344e97a",
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_praise_set = set(itertools.islice(praise_set, 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "c32732a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_tag_dict = set(itertools.islice(tag_dict, 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "00fcda00",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-117-071be6b5bcf1>:7: UserWarning: [W008] Evaluating Doc.similarity based on empty vectors.\n",
      "  temp_1 = [row_tag[0] for row_tag in tag_dict for item_praise in set(index_praise[1].split(\" \")) for item_tag in set(row_tag[1].split(\" \")) if nlp(item_praise).similarity(nlp(item_tag)) == 1]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "row 99 is done.\n",
      "Total run time for 100 rows is: 404.6406993865967\n"
     ]
    }
   ],
   "source": [
    "list_tag = []\n",
    "temp_1 = []\n",
    "counter = 0\n",
    "\n",
    "start_time = time.time()\n",
    "for index_praise in praise_set:\n",
    "    temp_1 = [row_tag[0] for row_tag in tag_dict for item_praise in set(index_praise[1].split(\" \")) for item_tag in set(row_tag[1].split(\" \")) if nlp(item_praise).similarity(nlp(item_tag)) == 1]\n",
    "    if len(temp_1)>=1:\n",
    "        list_tag.append((index_praise[0], check_freq(temp_1)))\n",
    "        temp_1 = []\n",
    "    else:\n",
    "        list_tag.append((index_praise[0],np.nan))\n",
    "    print(f\"row {counter} is done.\", end='\\r')\n",
    "    counter+=1\n",
    "\n",
    "end_time = time.time()\n",
    "print(f\"\\nTotal run time for {len(praise_set)} rows is: {end_time - start_time}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "795d1e50",
   "metadata": {},
   "outputs": [],
   "source": [
    "output = pd.DataFrame(list_tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "5a4c19de",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>97</td>\n",
       "      <td>putting heads finding bug</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>mentioning retweeting te commons socials past week thank helping us grow token engineering commons community spreading message</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>masana temples recommendation nice mix low high beats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>48</td>\n",
       "      <td>contributing latest tec praise quant blog</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57</td>\n",
       "      <td>joining commons stack simulator sync continuing push project finish line</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>60</td>\n",
       "      <td>starting cadcad bootcamp study group</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>51</td>\n",
       "      <td>tackling remaining issues swiss membership dapp</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>46</td>\n",
       "      <td>visible community work hatch super cool</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>2</td>\n",
       "      <td>joining legal weekly sync</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>22</td>\n",
       "      <td>awesome people interacted juan recently conversations amazing</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     0  \\\n",
       "0   97   \n",
       "1    6   \n",
       "2    1   \n",
       "3   48   \n",
       "4   57   \n",
       "..  ..   \n",
       "95  60   \n",
       "96  51   \n",
       "97  46   \n",
       "98   2   \n",
       "99  22   \n",
       "\n",
       "                                                                                                                                 1  \n",
       "0                                                                                                        putting heads finding bug  \n",
       "1   mentioning retweeting te commons socials past week thank helping us grow token engineering commons community spreading message  \n",
       "2                                                                            masana temples recommendation nice mix low high beats  \n",
       "3                                                                                        contributing latest tec praise quant blog  \n",
       "4                                                         joining commons stack simulator sync continuing push project finish line  \n",
       "..                                                                                                                             ...  \n",
       "95                                                                                            starting cadcad bootcamp study group  \n",
       "96                                                                                 tackling remaining issues swiss membership dapp  \n",
       "97                                                                                         visible community work hatch super cool  \n",
       "98                                                                                                       joining legal weekly sync  \n",
       "99                                                                   awesome people interacted juan recently conversations amazing  \n",
       "\n",
       "[100 rows x 2 columns]"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "praise = pd.DataFrame(praise_set)\n",
    "praise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "d7257ffe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TEC2</td>\n",
       "      <td>culture build soft gov survey vote voting debate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TEC14</td>\n",
       "      <td>technical infrastructure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TEC5</td>\n",
       "      <td>commons swarm tech dev dapp tech dapp app bug</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TEC10</td>\n",
       "      <td>transparency youtube recording record</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TEC3</td>\n",
       "      <td>params parameters param parties party</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>TEC7</td>\n",
       "      <td>omega survey philosophy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>TEC9</td>\n",
       "      <td>labs lab</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>TEC4</td>\n",
       "      <td>legal strategy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>TEC17</td>\n",
       "      <td>gravity conflict nonviolent communication</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>TEC1</td>\n",
       "      <td>comms article blog organized presentation graphic design website marketing seo social platform discord telegram forum discourse medium linkedin website article retweeting retweet retweets organized presentation communications</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>TEC6</td>\n",
       "      <td>hatch outreach onboarding onboard outreach</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>TEC13</td>\n",
       "      <td>fundraising donating fundraising</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>TEC15</td>\n",
       "      <td>tec community building</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>TEC8</td>\n",
       "      <td>stewards github steward project management</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>TEC12</td>\n",
       "      <td>participation retweets</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>TEC16</td>\n",
       "      <td>personal praise</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        0  \\\n",
       "0    TEC2   \n",
       "1   TEC14   \n",
       "2    TEC5   \n",
       "3   TEC10   \n",
       "4    TEC3   \n",
       "5    TEC7   \n",
       "6    TEC9   \n",
       "7    TEC4   \n",
       "8   TEC17   \n",
       "9    TEC1   \n",
       "10   TEC6   \n",
       "11  TEC13   \n",
       "12  TEC15   \n",
       "13   TEC8   \n",
       "14  TEC12   \n",
       "15  TEC16   \n",
       "\n",
       "                                                                                                                                                                                                                                    1  \n",
       "0                                                                                                                                                                                    culture build soft gov survey vote voting debate  \n",
       "1                                                                                                                                                                                                            technical infrastructure  \n",
       "2                                                                                                                                                                                       commons swarm tech dev dapp tech dapp app bug  \n",
       "3                                                                                                                                                                                               transparency youtube recording record  \n",
       "4                                                                                                                                                                                               params parameters param parties party  \n",
       "5                                                                                                                                                                                                             omega survey philosophy  \n",
       "6                                                                                                                                                                                                                            labs lab  \n",
       "7                                                                                                                                                                                                                      legal strategy  \n",
       "8                                                                                                                                                                                           gravity conflict nonviolent communication  \n",
       "9   comms article blog organized presentation graphic design website marketing seo social platform discord telegram forum discourse medium linkedin website article retweeting retweet retweets organized presentation communications  \n",
       "10                                                                                                                                                                                         hatch outreach onboarding onboard outreach  \n",
       "11                                                                                                                                                                                                   fundraising donating fundraising  \n",
       "12                                                                                                                                                                                                             tec community building  \n",
       "13                                                                                                                                                                                         stewards github steward project management  \n",
       "14                                                                                                                                                                                                             participation retweets  \n",
       "15                                                                                                                                                                                                                    personal praise  "
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tags = pd.DataFrame(tag_dict)\n",
    "tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "id": "170699b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "praise['original_label'] = output[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "id": "f26fbc96",
   "metadata": {},
   "outputs": [],
   "source": [
    "praise = praise.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "id": "a862bc2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "praise['accuracy'] = praise['label'] == praise['original_label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "id": "f5322b4f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4142857142857143"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "praise['accuracy'].sum() / len(praise)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "id": "b480b5a8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>label</th>\n",
       "      <th>original_label</th>\n",
       "      <th>accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>97</td>\n",
       "      <td>putting heads finding bug</td>\n",
       "      <td>TEC5</td>\n",
       "      <td>TEC5</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>mentioning retweeting te commons socials past week thank helping us grow token engineering commons community spreading message</td>\n",
       "      <td>TEC1</td>\n",
       "      <td>TEC5</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>48</td>\n",
       "      <td>contributing latest tec praise quant blog</td>\n",
       "      <td>TEC1</td>\n",
       "      <td>TEC1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57</td>\n",
       "      <td>joining commons stack simulator sync continuing push project finish line</td>\n",
       "      <td>TEC2</td>\n",
       "      <td>TEC5</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>4</td>\n",
       "      <td>mentioning retweeting te commons socials past week thank helping us grow token engineering commons community spreading message</td>\n",
       "      <td>TEC1</td>\n",
       "      <td>TEC5</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>78</td>\n",
       "      <td>mentioning retweeting commons stack socials past week thank helping us grow commons stack community spreading message</td>\n",
       "      <td>TEC1</td>\n",
       "      <td>TEC5</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>98</td>\n",
       "      <td>joining tec weekly sync https www youtube com watch v efhuvu oc0</td>\n",
       "      <td>TEC10</td>\n",
       "      <td>TEC10</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>51</td>\n",
       "      <td>tackling remaining issues swiss membership dapp</td>\n",
       "      <td>TEC2</td>\n",
       "      <td>TEC5</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>46</td>\n",
       "      <td>visible community work hatch super cool</td>\n",
       "      <td>TEC15</td>\n",
       "      <td>TEC6</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>2</td>\n",
       "      <td>joining legal weekly sync</td>\n",
       "      <td>TEC4</td>\n",
       "      <td>TEC4</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>70 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     0  \\\n",
       "0   97   \n",
       "1    6   \n",
       "3   48   \n",
       "4   57   \n",
       "5    4   \n",
       "..  ..   \n",
       "93  78   \n",
       "94  98   \n",
       "96  51   \n",
       "97  46   \n",
       "98   2   \n",
       "\n",
       "                                                                                                                                 1  \\\n",
       "0                                                                                                        putting heads finding bug   \n",
       "1   mentioning retweeting te commons socials past week thank helping us grow token engineering commons community spreading message   \n",
       "3                                                                                        contributing latest tec praise quant blog   \n",
       "4                                                         joining commons stack simulator sync continuing push project finish line   \n",
       "5   mentioning retweeting te commons socials past week thank helping us grow token engineering commons community spreading message   \n",
       "..                                                                                                                             ...   \n",
       "93           mentioning retweeting commons stack socials past week thank helping us grow commons stack community spreading message   \n",
       "94                                                                joining tec weekly sync https www youtube com watch v efhuvu oc0   \n",
       "96                                                                                 tackling remaining issues swiss membership dapp   \n",
       "97                                                                                         visible community work hatch super cool   \n",
       "98                                                                                                       joining legal weekly sync   \n",
       "\n",
       "    label original_label  accuracy  \n",
       "0    TEC5           TEC5      True  \n",
       "1    TEC1           TEC5     False  \n",
       "3    TEC1           TEC1      True  \n",
       "4    TEC2           TEC5     False  \n",
       "5    TEC1           TEC5     False  \n",
       "..    ...            ...       ...  \n",
       "93   TEC1           TEC5     False  \n",
       "94  TEC10          TEC10      True  \n",
       "96   TEC2           TEC5     False  \n",
       "97  TEC15           TEC6     False  \n",
       "98   TEC4           TEC4      True  \n",
       "\n",
       "[70 rows x 5 columns]"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "praise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "7b80dd84",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(97, 'TEC5'), (6, 'TEC5'), (48, nan), (1, nan), (57, 'TEC5')]"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "2c99b6e0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['TEC5']"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "['TEC5']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "3ec45ba6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>97</td>\n",
       "      <td>TEC5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>TEC5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>48</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57</td>\n",
       "      <td>TEC5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    0     1\n",
       "0  97  TEC5\n",
       "1   6  TEC5\n",
       "2  48   NaN\n",
       "3   1   NaN\n",
       "4  57  TEC5"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(list_tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "b4db233d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index_</th>\n",
       "      <th>tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>97</td>\n",
       "      <td>TEC5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>TEC5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>48</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57</td>\n",
       "      <td>TEC5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index_   tag\n",
       "0      97  TEC5\n",
       "1       6  TEC5\n",
       "2      48   NaN\n",
       "3       1   NaN\n",
       "4      57  TEC5"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_temp = pd.DataFrame(list_tag, columns=['index_', 'tag'])\n",
    "df_temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b7eedffb",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sample_100 = pd.merge(df_praise, df_temp, how=\"left\", left_index=True, right_on='index_')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "77926ea3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sample_100 = df_sample_100.loc[df_sample_100.index.dropna()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "53c92c96",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Category Code', 'To', 'From', 'Reason for dishing', 'Server', 'Date',\n",
       "       'Room', 'v1 norm', 'v2 norm', 'v3 norm', 'Avg %', 'IH per Praise',\n",
       "       'IH per person', 'index_', 'tag'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_sample_100.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "83bd423f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_compare = df_sample_100[['Category Code', 'tag', 'Reason for dishing']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "2de7efe0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Category Code          0\n",
       "tag                   30\n",
       "Reason for dishing     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_compare.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "7b7d9347",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Category Code</th>\n",
       "      <th>tag</th>\n",
       "      <th>Reason for dishing</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>TEC14</td>\n",
       "      <td>TEC5</td>\n",
       "      <td>putting heads finding bug</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.0</th>\n",
       "      <td>TEC12</td>\n",
       "      <td>TEC5</td>\n",
       "      <td>mentioning retweeting te commons socials past week thank helping us grow token engineering commons community spreading message</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2.0</th>\n",
       "      <td>TEC12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>masana temples recommendation nice mix low high beats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3.0</th>\n",
       "      <td>TEC10</td>\n",
       "      <td>TEC1</td>\n",
       "      <td>contributing latest tec praise quant blog</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4.0</th>\n",
       "      <td>TEC12</td>\n",
       "      <td>TEC5</td>\n",
       "      <td>joining commons stack simulator sync continuing push project finish line</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95.0</th>\n",
       "      <td>TEC5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>starting cadcad bootcamp study group</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96.0</th>\n",
       "      <td>TEC5</td>\n",
       "      <td>TEC5</td>\n",
       "      <td>tackling remaining issues swiss membership dapp</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97.0</th>\n",
       "      <td>TEC5</td>\n",
       "      <td>TEC6</td>\n",
       "      <td>visible community work hatch super cool</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98.0</th>\n",
       "      <td>TEC12</td>\n",
       "      <td>TEC4</td>\n",
       "      <td>joining legal weekly sync</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99.0</th>\n",
       "      <td>TEC16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>awesome people interacted juan recently conversations amazing</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Category Code   tag  \\\n",
       "0.0          TEC14  TEC5   \n",
       "1.0          TEC12  TEC5   \n",
       "2.0          TEC12   NaN   \n",
       "3.0          TEC10  TEC1   \n",
       "4.0          TEC12  TEC5   \n",
       "...            ...   ...   \n",
       "95.0          TEC5   NaN   \n",
       "96.0          TEC5  TEC5   \n",
       "97.0          TEC5  TEC6   \n",
       "98.0         TEC12  TEC4   \n",
       "99.0         TEC16   NaN   \n",
       "\n",
       "                                                                                                                  Reason for dishing  \n",
       "0.0                                                                                                        putting heads finding bug  \n",
       "1.0   mentioning retweeting te commons socials past week thank helping us grow token engineering commons community spreading message  \n",
       "2.0                                                                            masana temples recommendation nice mix low high beats  \n",
       "3.0                                                                                        contributing latest tec praise quant blog  \n",
       "4.0                                                         joining commons stack simulator sync continuing push project finish line  \n",
       "...                                                                                                                              ...  \n",
       "95.0                                                                                            starting cadcad bootcamp study group  \n",
       "96.0                                                                                 tackling remaining issues swiss membership dapp  \n",
       "97.0                                                                                         visible community work hatch super cool  \n",
       "98.0                                                                                                       joining legal weekly sync  \n",
       "99.0                                                                   awesome people interacted juan recently conversations amazing  \n",
       "\n",
       "[100 rows x 3 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_compare.sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a41df56",
   "metadata": {},
   "outputs": [],
   "source": [
    "n praise records\n",
    "m tags\n",
    "\n",
    "n_i has p_i words\n",
    "m_j has q_j words\n",
    "\n",
    "so total complexity is\n",
    "\n",
    "for each i in n\n",
    "    for each j in m\n",
    "     for each p_i_k in p_i\n",
    "        for each q_j_l in q_j\n",
    "            sims.append(compare_similarity(p_i_k, q_j_l))\n",
    "        list_of_tags.append(most_frequent(sim))\n",
    "        sims = []\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "99987fbd",
   "metadata": {},
   "source": [
    "Start with a praise data set. For each praise record, we will compare the similarity to each tag label.\n",
    "\n",
    "Give the first row of praise data, split the 'reason for praise' into a list of words.\n",
    "Now go through each"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "5bd7bd0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# list_tag = []\n",
    "# temp_1 = []\n",
    "# temp_2 = []\n",
    "# temp_3 = []\n",
    "# counter = 0\n",
    "\n",
    "# start_time = time.time()\n",
    "# for index_praise in praise_set:\n",
    "#     for row_tag in tag_dict:\n",
    "#         temp_set_1 = set(index_praise[1].split(\" \"))\n",
    "#         for item_praise in temp_set_1:\n",
    "#             temp_set_2 = set(row_tag[1].split(\" \"))\n",
    "#             for item_tag in temp_set_2:\n",
    "#                 tag_token = nlp(item_tag)\n",
    "#                 reason_token = nlp(item_praise)\n",
    "#                 if reason_token.similarity(tag_token) == 1:\n",
    "#                     temp_1.append(row_tag[0])\n",
    "#             if len(temp_1)>=1:\n",
    "#                 temp_2.append(check_freq(temp_1))\n",
    "#                 temp_1 = []\n",
    "#         if len(temp_2)>=1:\n",
    "#             temp_3.append(check_freq(temp_2))\n",
    "#             temp_2 = []\n",
    "#     if len(temp_3)>=1:\n",
    "#         list_tag.append((index_praise[0], check_freq(temp_3)))\n",
    "#         temp_3 = []\n",
    "#     else:\n",
    "#         list_tag.append((index_praise[0],np.nan))\n",
    "#     print(f\"row {counter} is done.\", end='\\r')\n",
    "#     counter+=1\n",
    "\n",
    "# end_time = time.time()\n",
    "# print(f\"\\nTotal run time for {df_praise.shape[0]} rows is: {end_time - start_time}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "9821017f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i in range(1, 101):\n",
    "#     print('#'*i+' '*(101-i)+f'{i}%', end='\\r')\n",
    "#     time.sleep(0.1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}