notebooks/3.0-ja-clustering2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import datetime as dt\n",
    "import re\n",
    "#import gensim\n",
    "import sklearn; # sklearn.get_config()['working_memory']\n",
    "#from sklearn.metrics import pairwise_distances, pairwise_distances_chunked # not used\n",
    "#from sklearn.metrics.pairwise import cosine_similarity, cosine_distances # not used\n",
    "#from scipy.spatial.distance import pdist, cosine # not used\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.cluster import AgglomerativeClustering\n",
    "# Visualisation\n",
    "import seaborn as sns\n",
    "import plotly.express as px\n",
    "\n",
    "#import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "from nltk.stem.lancaster import LancasterStemmer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = os.path.join(os.environ[\"HOME\"],\"Dropbox\",\"EUvsVirus\")\n",
    "\n",
    "data_import = pd.read_csv(os.path.join(data_dir,\"all_Data.tsv\"), sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "stopwords = stopwords.words('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = (data_import.dropna(subset=pd.Index([\"text\"]))\n",
    "                   .assign(uniqueID = lambda x: x.ProjURL.str.extract(r\"(?<=https://devpost.com/software/)(.+$)\"),\n",
    "                           text = lambda x: x.text.str.lower(), # not removing non-word chars yet because later we need the '\\n' to separate on lines\n",
    "                           ) \n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculating the headings which should be removed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "headings = (data.text.str.split('\\n', expand=True) #splitting into lines\n",
    "              .melt(value_name='lines') # melting into one column\n",
    "                .lines.value_counts() # selecting the 'lines' column and counting frequency\n",
    "                .reset_index()\n",
    ")\n",
    "headings.columns = ['headings','counts']\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Visual inspection of the 200 most common lines showed that lines present 6+ times were not very defining. \n",
    "\n",
    "These will be removed by creating a variable called headings6plus and removing these headings from the text. Note: This is not removing the word from the entire text, just the line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "                                              headings  counts\n100                               what we have learned       7\n101                                  what inspired us?       7\n102                                            funding       7\n103                                                 no       7\n104                                    ... coming soon       7\n105                                             design       7\n106                                          challenge       7\n107                                            backend       7\n108                 accomplishments that i'm proud of:       7\n109              mitigation or risk avoidance measures       7\n110                                        pitch video       7\n111                our solution’s impact on the crisis       7\n112                                        development       6\n113                                      communication       6\n114               ## accomplishments that i'm proud of       6\n115                                        description       6\n116                                         technology       6\n117                                           features       6\n118                                           about us       6\n119                                    the inspiration       6\n120                                                why       6\n121                           ## challenges i ran into       6\n122                                         thank you!       6\n123                                    how we build it       6\n124               the url to the pitchvideo (required)       6\n125                                              links       6\n126                                             python       6\n127        the value of our solution after the crisis:       6\n128                                  proposed solution       6\n129                                       development:       6\n130                                           progress       6\n131                our solution’s impact to the crisis       6\n132                                            sources       6\n133                                           frontend       6\n134                                          resources       5\n135                                               how?       5\n136                                      the solution:       5\n137                                         milestones       5\n138                                    how to build it       5\n139                                        video pitch       5\n140             4. the solution’s impact to the crisis       5\n141                                         mitigation       5\n142                                  customer segments       5\n143                                         * yes / no       5\n144                                         validation       5\n145  under this challenge, we aim at addressing iss...       5\n146                                         our vision       5\n147                               technical complexity       5\n148                                      the prototype       5\n149                                        references:       5",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>headings</th>\n      <th>counts</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <td>100</td>\n      <td>what we have learned</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <td>101</td>\n      <td>what inspired us?</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <td>102</td>\n      <td>funding</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <td>103</td>\n      <td>no</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <td>104</td>\n      <td>... coming soon</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <td>105</td>\n      <td>design</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <td>106</td>\n      <td>challenge</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <td>107</td>\n      <td>backend</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <td>108</td>\n      <td>accomplishments that i'm proud of:</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <td>109</td>\n      <td>mitigation or risk avoidance measures</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <td>110</td>\n      <td>pitch video</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <td>111</td>\n      <td>our solution’s impact on the crisis</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <td>112</td>\n      <td>development</td>\n      <td>6</td>\n    </tr>\n    <tr>\n      <td>113</td>\n      <td>communication</td>\n      <td>6</td>\n    </tr>\n    <tr>\n      <td>114</td>\n      <td>## accomplishments that i'm proud of</td>\n      <td>6</td>\n    </tr>\n    <tr>\n      <td>115</td>\n      <td>description</td>\n      <td>6</td>\n    </tr>\n    <tr>\n      <td>116</td>\n      <td>technology</td>\n      <td>6</td>\n    </tr>\n    <tr>\n      <td>117</td>\n      <td>features</td>\n      <td>6</td>\n    </tr>\n    <tr>\n      <td>118</td>\n      <td>about us</td>\n      <td>6</td>\n    </tr>\n    <tr>\n      <td>119</td>\n      <td>the inspiration</td>\n      <td>6</td>\n    </tr>\n    <tr>\n      <td>120</td>\n      <td>why</td>\n      <td>6</td>\n    </tr>\n    <tr>\n      <td>121</td>\n      <td>## challenges i ran into</td>\n      <td>6</td>\n    </tr>\n    <tr>\n      <td>122</td>\n      <td>thank you!</td>\n      <td>6</td>\n    </tr>\n    <tr>\n      <td>123</td>\n      <td>how we build it</td>\n      <td>6</td>\n    </tr>\n    <tr>\n      <td>124</td>\n      <td>the url to the pitchvideo (required)</td>\n      <td>6</td>\n    </tr>\n    <tr>\n      <td>125</td>\n      <td>links</td>\n      <td>6</td>\n    </tr>\n    <tr>\n      <td>126</td>\n      <td>python</td>\n      <td>6</td>\n    </tr>\n    <tr>\n      <td>127</td>\n      <td>the value of our solution after the crisis:</td>\n      <td>6</td>\n    </tr>\n    <tr>\n      <td>128</td>\n      <td>proposed solution</td>\n      <td>6</td>\n    </tr>\n    <tr>\n      <td>129</td>\n      <td>development:</td>\n      <td>6</td>\n    </tr>\n    <tr>\n      <td>130</td>\n      <td>progress</td>\n      <td>6</td>\n    </tr>\n    <tr>\n      <td>131</td>\n      <td>our solution’s impact to the crisis</td>\n      <td>6</td>\n    </tr>\n    <tr>\n      <td>132</td>\n      <td>sources</td>\n      <td>6</td>\n    </tr>\n    <tr>\n      <td>133</td>\n      <td>frontend</td>\n      <td>6</td>\n    </tr>\n    <tr>\n      <td>134</td>\n      <td>resources</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <td>135</td>\n      <td>how?</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <td>136</td>\n      <td>the solution:</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <td>137</td>\n      <td>milestones</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <td>138</td>\n      <td>how to build it</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <td>139</td>\n      <td>video pitch</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <td>140</td>\n      <td>4. the solution’s impact to the crisis</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <td>141</td>\n      <td>mitigation</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <td>142</td>\n      <td>customer segments</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <td>143</td>\n      <td>* yes / no</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <td>144</td>\n      <td>validation</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <td>145</td>\n      <td>under this challenge, we aim at addressing iss...</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <td>146</td>\n      <td>our vision</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <td>147</td>\n      <td>technical complexity</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <td>148</td>\n      <td>the prototype</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <td>149</td>\n      <td>references:</td>\n      <td>5</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 7
    }
   ],
   "source": [
    "headings.iloc[100:150]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "0                                         \n1                              inspiration\n2                             what it does\n3        accomplishments that i'm proud of\n4                           how i built it\n                      ...                 \n129                           development:\n130                               progress\n131    our solution’s impact to the crisis\n132                                sources\n133                               frontend\nName: headings, Length: 134, dtype: object"
     },
     "metadata": {},
     "execution_count": 8
    }
   ],
   "source": [
    "headings6plus = headings.loc[headings.counts >= 6,'headings'] # counts of 6+\n",
    "headings6plus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lol = list of lines\n",
    "# low = list of words\n",
    "data = (data.assign(headings_rmvd = data.text.str.split('\\n') # splitting into lines\n",
    "                                             .apply(lambda lol: [line for line in lol if line not in list(headings6plus)]) # filtering out headings\n",
    "                                             .apply(lambda lol: ' '.join(lol)), # joining the lines back together\n",
    "                    words = lambda x: x.headings_rmvd.str.replace('\\W',' ').str.split() # spliting on non-word chars and white space\n",
    "                                                         .apply(lambda low: [word for word in low if word not in stopwords]) # removing stopwords\n",
    "                   )\n",
    ")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "(2069, 8)\n"
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "  Challenge SubChallenge                                            ProjURL  \\\n0    Health    Equipment                  https://devpost.com/software/evam   \n1    Health    Equipment            https://devpost.com/software/nanomaskcz   \n2    Health    Equipment  https://devpost.com/software/ecological-medica...   \n3    Health    Equipment  https://devpost.com/software/ecological-medica...   \n4    Health    Equipment  https://devpost.com/software/innovative-respir...   \n\n                     title                                               text  \\\n0                    EVAM   inspiration\\nthere is a huge shortage in the s...   \n1               NanomaskCZ  inspiration\\nthe story of technical university...   \n2  Ecological medical coat  inspiration\\nwhat it does\\neconomic medical co...   \n3  Ecological medical coat  inspiration\\nthe simplicity and the economical...   \n4           Respire Action  inspiration\\n• a recent study shows that over ...   \n\n                                            uniqueID  \\\n0                                               evam   \n1                                         nanomaskcz   \n2                     ecological-medical-coat-bfe2jp   \n3                            ecological-medical-coat   \n4  innovative-respiratory-mask-with-nanocellulose...   \n\n                                       headings_rmvd  \\\n0  there is a huge shortage in the supply chain f...   \n1  the story of technical university liberec (tul...   \n2  economic medical combination in a single patte...   \n3  the simplicity and the economical way of desig...   \n4  • a recent study shows that over 20% of caregi...   \n\n                                               words  \n0  [huge, shortage, supply, chain, ppe, since, vi...  \n1  [story, technical, university, liberec, tul, m...  \n2  [economic, medical, combination, single, patte...  \n3  [simplicity, economical, way, designing, garme...  \n4  [recent, study, shows, 20, caregivers, nurses,...  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Challenge</th>\n      <th>SubChallenge</th>\n      <th>ProjURL</th>\n      <th>title</th>\n      <th>text</th>\n      <th>uniqueID</th>\n      <th>headings_rmvd</th>\n      <th>words</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <td>0</td>\n      <td>Health</td>\n      <td>Equipment</td>\n      <td>https://devpost.com/software/evam</td>\n      <td>EVAM</td>\n      <td>inspiration\\nthere is a huge shortage in the s...</td>\n      <td>evam</td>\n      <td>there is a huge shortage in the supply chain f...</td>\n      <td>[huge, shortage, supply, chain, ppe, since, vi...</td>\n    </tr>\n    <tr>\n      <td>1</td>\n      <td>Health</td>\n      <td>Equipment</td>\n      <td>https://devpost.com/software/nanomaskcz</td>\n      <td>NanomaskCZ</td>\n      <td>inspiration\\nthe story of technical university...</td>\n      <td>nanomaskcz</td>\n      <td>the story of technical university liberec (tul...</td>\n      <td>[story, technical, university, liberec, tul, m...</td>\n    </tr>\n    <tr>\n      <td>2</td>\n      <td>Health</td>\n      <td>Equipment</td>\n      <td>https://devpost.com/software/ecological-medica...</td>\n      <td>Ecological medical coat</td>\n      <td>inspiration\\nwhat it does\\neconomic medical co...</td>\n      <td>ecological-medical-coat-bfe2jp</td>\n      <td>economic medical combination in a single patte...</td>\n      <td>[economic, medical, combination, single, patte...</td>\n    </tr>\n    <tr>\n      <td>3</td>\n      <td>Health</td>\n      <td>Equipment</td>\n      <td>https://devpost.com/software/ecological-medica...</td>\n      <td>Ecological medical coat</td>\n      <td>inspiration\\nthe simplicity and the economical...</td>\n      <td>ecological-medical-coat</td>\n      <td>the simplicity and the economical way of desig...</td>\n      <td>[simplicity, economical, way, designing, garme...</td>\n    </tr>\n    <tr>\n      <td>4</td>\n      <td>Health</td>\n      <td>Equipment</td>\n      <td>https://devpost.com/software/innovative-respir...</td>\n      <td>Respire Action</td>\n      <td>inspiration\\n• a recent study shows that over ...</td>\n      <td>innovative-respiratory-mask-with-nanocellulose...</td>\n      <td>• a recent study shows that over 20% of caregi...</td>\n      <td>[recent, study, shows, 20, caregivers, nurses,...</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 10
    }
   ],
   "source": [
    "print(data.shape)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Checking the word count of the submissions now.\n",
    "\n",
    "45 submissions have only 1 word (after heading removal). These are not so useful.\n",
    "After visual inspection I will remove entries with 10 words or less after headings removal. I don't think you could anyway get meaningful information from these submissions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "0      5\n1     45\n2     31\n3     10\n4      7\n5      8\n6      2\n7      1\n8      2\n9      2\n10     4\n11     3\n12     2\n13     1\n16     1\nName: words, dtype: int64"
     },
     "metadata": {},
     "execution_count": 11
    }
   ],
   "source": [
    "data.words.apply(len).value_counts().sort_index().head(15)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "for x in data.text[data.words.apply(len) == 10]:\n",
    "    print(x)\n",
    "    print('---\\n---\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data.loc[data.words.apply(len) > 10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "     Challenge SubChallenge  \\\n0       Health    Equipment   \n1       Health    Equipment   \n2       Health    Equipment   \n3       Health    Equipment   \n4       Health    Equipment   \n...        ...          ...   \n2154     Other        Other   \n2155     Other        Other   \n2156     Other        Other   \n2157     Other        Other   \n2158     Other        Other   \n\n                                                ProjURL  \\\n0                     https://devpost.com/software/evam   \n1               https://devpost.com/software/nanomaskcz   \n2     https://devpost.com/software/ecological-medica...   \n3     https://devpost.com/software/ecological-medica...   \n4     https://devpost.com/software/innovative-respir...   \n...                                                 ...   \n2154  https://devpost.com/software/computer-modeling...   \n2155               https://devpost.com/software/beahero   \n2156  https://devpost.com/software/solution-blockcha...   \n2157   https://devpost.com/software/civictechhub-rgp8zk   \n2158     https://devpost.com/software/info-virus-ph9idw   \n\n                                                title  \\\n0                                               EVAM    \n1                                          NanomaskCZ   \n2                             Ecological medical coat   \n3                             Ecological medical coat   \n4                                      Respire Action   \n...                                               ...   \n2154  Computer discovery of novel anti COVID-19 drugs   \n2155                                          BeAHero   \n2156                          Solution Blockchain API   \n2157                                     CivicTechHub   \n2158                                       info-virus   \n\n                                                   text  \\\n0     inspiration\\nthere is a huge shortage in the s...   \n1     inspiration\\nthe story of technical university...   \n2     inspiration\\nwhat it does\\neconomic medical co...   \n3     inspiration\\nthe simplicity and the economical...   \n4     inspiration\\n• a recent study shows that over ...   \n...                                                 ...   \n2154  inspiration\\nthe methods of virtual screening ...   \n2155  inspiration\\nwe live among heroes. in fact, we...   \n2156  inspiration\\nthe future of companies will be c...   \n2157  please join us in our slack channel: https://j...   \n2158  in the media, the conversation is fundamentall...   \n\n                                               uniqueID  \\\n0                                                  evam   \n1                                            nanomaskcz   \n2                        ecological-medical-coat-bfe2jp   \n3                               ecological-medical-coat   \n4     innovative-respiratory-mask-with-nanocellulose...   \n...                                                 ...   \n2154    computer-modeling-of-new-drugs-against-covid-19   \n2155                                            beahero   \n2156                            solution-blockchain-api   \n2157                                civictechhub-rgp8zk   \n2158                                  info-virus-ph9idw   \n\n                                          headings_rmvd  \\\n0     there is a huge shortage in the supply chain f...   \n1     the story of technical university liberec (tul...   \n2     economic medical combination in a single patte...   \n3     the simplicity and the economical way of desig...   \n4     • a recent study shows that over 20% of caregi...   \n...                                                 ...   \n2154  the methods of virtual screening and molecular...   \n2155  we live among heroes. in fact, we are all hero...   \n2156  the future of companies will be collaboration,...   \n2157  please join us in our slack channel: https://j...   \n2158  in the media, the conversation is fundamentall...   \n\n                                                  words  \n0     [huge, shortage, supply, chain, ppe, since, vi...  \n1     [story, technical, university, liberec, tul, m...  \n2     [economic, medical, combination, single, patte...  \n3     [simplicity, economical, way, designing, garme...  \n4     [recent, study, shows, 20, caregivers, nurses,...  \n...                                                 ...  \n2154  [methods, virtual, screening, molecular, model...  \n2155  [live, among, heroes, fact, heroes, stay, home...  \n2156  [future, companies, collaboration, distributio...  \n2157  [please, join, us, slack, channel, https, join...  \n2158  [media, conversation, fundamentally, crisis, p...  \n\n[1952 rows x 8 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Challenge</th>\n      <th>SubChallenge</th>\n      <th>ProjURL</th>\n      <th>title</th>\n      <th>text</th>\n      <th>uniqueID</th>\n      <th>headings_rmvd</th>\n      <th>words</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <td>0</td>\n      <td>Health</td>\n      <td>Equipment</td>\n      <td>https://devpost.com/software/evam</td>\n      <td>EVAM</td>\n      <td>inspiration\\nthere is a huge shortage in the s...</td>\n      <td>evam</td>\n      <td>there is a huge shortage in the supply chain f...</td>\n      <td>[huge, shortage, supply, chain, ppe, since, vi...</td>\n    </tr>\n    <tr>\n      <td>1</td>\n      <td>Health</td>\n      <td>Equipment</td>\n      <td>https://devpost.com/software/nanomaskcz</td>\n      <td>NanomaskCZ</td>\n      <td>inspiration\\nthe story of technical university...</td>\n      <td>nanomaskcz</td>\n      <td>the story of technical university liberec (tul...</td>\n      <td>[story, technical, university, liberec, tul, m...</td>\n    </tr>\n    <tr>\n      <td>2</td>\n      <td>Health</td>\n      <td>Equipment</td>\n      <td>https://devpost.com/software/ecological-medica...</td>\n      <td>Ecological medical coat</td>\n      <td>inspiration\\nwhat it does\\neconomic medical co...</td>\n      <td>ecological-medical-coat-bfe2jp</td>\n      <td>economic medical combination in a single patte...</td>\n      <td>[economic, medical, combination, single, patte...</td>\n    </tr>\n    <tr>\n      <td>3</td>\n      <td>Health</td>\n      <td>Equipment</td>\n      <td>https://devpost.com/software/ecological-medica...</td>\n      <td>Ecological medical coat</td>\n      <td>inspiration\\nthe simplicity and the economical...</td>\n      <td>ecological-medical-coat</td>\n      <td>the simplicity and the economical way of desig...</td>\n      <td>[simplicity, economical, way, designing, garme...</td>\n    </tr>\n    <tr>\n      <td>4</td>\n      <td>Health</td>\n      <td>Equipment</td>\n      <td>https://devpost.com/software/innovative-respir...</td>\n      <td>Respire Action</td>\n      <td>inspiration\\n• a recent study shows that over ...</td>\n      <td>innovative-respiratory-mask-with-nanocellulose...</td>\n      <td>• a recent study shows that over 20% of caregi...</td>\n      <td>[recent, study, shows, 20, caregivers, nurses,...</td>\n    </tr>\n    <tr>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <td>2154</td>\n      <td>Other</td>\n      <td>Other</td>\n      <td>https://devpost.com/software/computer-modeling...</td>\n      <td>Computer discovery of novel anti COVID-19 drugs</td>\n      <td>inspiration\\nthe methods of virtual screening ...</td>\n      <td>computer-modeling-of-new-drugs-against-covid-19</td>\n      <td>the methods of virtual screening and molecular...</td>\n      <td>[methods, virtual, screening, molecular, model...</td>\n    </tr>\n    <tr>\n      <td>2155</td>\n      <td>Other</td>\n      <td>Other</td>\n      <td>https://devpost.com/software/beahero</td>\n      <td>BeAHero</td>\n      <td>inspiration\\nwe live among heroes. in fact, we...</td>\n      <td>beahero</td>\n      <td>we live among heroes. in fact, we are all hero...</td>\n      <td>[live, among, heroes, fact, heroes, stay, home...</td>\n    </tr>\n    <tr>\n      <td>2156</td>\n      <td>Other</td>\n      <td>Other</td>\n      <td>https://devpost.com/software/solution-blockcha...</td>\n      <td>Solution Blockchain API</td>\n      <td>inspiration\\nthe future of companies will be c...</td>\n      <td>solution-blockchain-api</td>\n      <td>the future of companies will be collaboration,...</td>\n      <td>[future, companies, collaboration, distributio...</td>\n    </tr>\n    <tr>\n      <td>2157</td>\n      <td>Other</td>\n      <td>Other</td>\n      <td>https://devpost.com/software/civictechhub-rgp8zk</td>\n      <td>CivicTechHub</td>\n      <td>please join us in our slack channel: https://j...</td>\n      <td>civictechhub-rgp8zk</td>\n      <td>please join us in our slack channel: https://j...</td>\n      <td>[please, join, us, slack, channel, https, join...</td>\n    </tr>\n    <tr>\n      <td>2158</td>\n      <td>Other</td>\n      <td>Other</td>\n      <td>https://devpost.com/software/info-virus-ph9idw</td>\n      <td>info-virus</td>\n      <td>in the media, the conversation is fundamentall...</td>\n      <td>info-virus-ph9idw</td>\n      <td>in the media, the conversation is fundamentall...</td>\n      <td>[media, conversation, fundamentally, crisis, p...</td>\n    </tr>\n  </tbody>\n</table>\n<p>1952 rows × 8 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 13
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#from nltk.stem.porter import PorterStemmer\n",
    "#from nltk.stem.snowball import SnowballStemmer\n",
    "#from nltk.stem.lancaster import LancasterStemmer\n",
    "ps = PorterStemmer()\n",
    "ss = SnowballStemmer('english')\n",
    "ls = LancasterStemmer()\n",
    "# low = list of words\n",
    "data = data.assign(stemmed_words = (data.words.apply(lambda low: [ps.stem(word) for word in low])\n",
    "                                              #.apply(lambda low: [ss.stem(word) for word in low])\n",
    "                                              #.apply(lambda low: [ls.stem(word) for word in low])\n",
    "                                    )\n",
    "                  )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "0       [huge, shortag, suppli, chain, ppe, sinc, viru...\n1       [stori, technic, univers, liberec, tul, made, ...\n2       [econom, medic, combin, singl, pattern, 180x16...\n3       [simplic, econom, way, design, garment, quickl...\n4       [recent, studi, show, 20, caregiv, nurs, docto...\n                              ...                        \n2154    [method, virtual, screen, molecular, model, pl...\n2155    [live, among, hero, fact, hero, stay, home, aw...\n2156    [futur, compani, collabor, distribut, consensu...\n2157    [pleas, join, us, slack, channel, http, join, ...\n2158    [media, convers, fundament, crisi, precaut, al...\nName: stemmed_words, Length: 1952, dtype: object"
     },
     "metadata": {},
     "execution_count": 15
    }
   ],
   "source": [
    "data.stemmed_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# low = list of words\n",
    "tfidf_model = TfidfVectorizer(strip_accents='unicode', stop_words='english')\n",
    "tfidf = tfidf_model.fit_transform(data.stemmed_words.apply(lambda low: ' '.join(low)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are way too many 'words' which are just numbers. These also need to be excluded."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "['00',\n '000',\n '0000',\n '0000000000000952',\n '0001',\n '0002',\n '000eur',\n '000gbp',\n '000usd',\n '001']"
     },
     "metadata": {},
     "execution_count": 19
    }
   ],
   "source": [
    "tfidf_model.get_feature_names()[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "25445\n"
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "                words\n0                  00\n1                 000\n2                0000\n3    0000000000000952\n4                0001\n..                ...\n995              3976\n996       39800840840\n997        3a03de0452\n998              3a05\n999           3a20797\n\n[1000 rows x 1 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>words</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <td>0</td>\n      <td>00</td>\n    </tr>\n    <tr>\n      <td>1</td>\n      <td>000</td>\n    </tr>\n    <tr>\n      <td>2</td>\n      <td>0000</td>\n    </tr>\n    <tr>\n      <td>3</td>\n      <td>0000000000000952</td>\n    </tr>\n    <tr>\n      <td>4</td>\n      <td>0001</td>\n    </tr>\n    <tr>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <td>995</td>\n      <td>3976</td>\n    </tr>\n    <tr>\n      <td>996</td>\n      <td>39800840840</td>\n    </tr>\n    <tr>\n      <td>997</td>\n      <td>3a03de0452</td>\n    </tr>\n    <tr>\n      <td>998</td>\n      <td>3a05</td>\n    </tr>\n    <tr>\n      <td>999</td>\n      <td>3a20797</td>\n    </tr>\n  </tbody>\n</table>\n<p>1000 rows × 1 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 20
    }
   ],
   "source": [
    "tfidf_vocab = pd.DataFrame({'words': tfidf_model.get_feature_names()})\n",
    "print(len(tfidf_vocab))\n",
    "tfidf_vocab[:1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_vocab = tfidf_vocab.assign(start_end_letter = lambda df: df.words.apply(lambda word: False if re.search('^[a-z].*[a-z]$',word) is None else True)\n",
    ") # boolean for if the word starts and ends in a letter and contains zero or more characters between the first and last letter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "             words  start_end_letter\n1708           a2w              True\n1712          a63f              True\n1714   a7891b8a6li              True\n1717      a879124b              True\n1718         a9rad              True\n...            ...               ...\n25424     zwischen              True\n25425         zxol              True\n25426      zytokin              True\n25427          zzp              True\n25428        złoti              True\n\n[23084 rows x 2 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>words</th>\n      <th>start_end_letter</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <td>1708</td>\n      <td>a2w</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <td>1712</td>\n      <td>a63f</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <td>1714</td>\n      <td>a7891b8a6li</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <td>1717</td>\n      <td>a879124b</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <td>1718</td>\n      <td>a9rad</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <td>25424</td>\n      <td>zwischen</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <td>25425</td>\n      <td>zxol</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <td>25426</td>\n      <td>zytokin</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <td>25427</td>\n      <td>zzp</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <td>25428</td>\n      <td>złoti</td>\n      <td>True</td>\n    </tr>\n  </tbody>\n</table>\n<p>23084 rows × 2 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 22
    }
   ],
   "source": [
    "tfidf_vocab.loc[tfidf_vocab.start_end_letter]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# low = list of words\n",
    "tfidf_model = TfidfVectorizer(strip_accents='unicode', stop_words='english', vocabulary=tfidf_vocab.loc[tfidf_vocab.start_end_letter,'words'])\n",
    "tfidf = tfidf_model.fit_transform(data.words.apply(lambda low: ' '.join(low)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "AgglomerativeClustering(affinity='cosine', compute_full_tree='auto',\n                        connectivity=None, distance_threshold=None,\n                        linkage='complete', memory=None, n_clusters=100,\n                        pooling_func='deprecated')"
     },
     "metadata": {},
     "execution_count": 24
    }
   ],
   "source": [
    "hc100 = AgglomerativeClustering(n_clusters=100,affinity = 'cosine', linkage='complete')\n",
    "hc100.fit(tfidf.toarray())\n",
    "#took approx 1min to run with sparse matrix of 1952 rows and 23084 words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "alignmentgroup": "True",
         "bingroup": "x",
         "hoverlabel": {
          "namelength": 0
         },
         "hovertemplate": "x=%{x}<br>count=%{y}",
         "legendgroup": "",
         "marker": {
          "color": "#636efa"
         },
         "name": "",
         "nbinsx": 100,
         "offsetgroup": "",
         "orientation": "v",
         "showlegend": false,
         "type": "histogram",
         "x": [
          72,
          91,
          91,
          91,
          64,
          64,
          85,
          51,
          37,
          19,
          64,
          51,
          29,
          51,
          60,
          26,
          5,
          1,
          37,
          64,
          64,
          1,
          65,
          4,
          79,
          0,
          97,
          9,
          5,
          64,
          51,
          78,
          39,
          64,
          37,
          2,
          64,
          51,
          72,
          66,
          64,
          37,
          64,
          64,
          19,
          1,
          61,
          64,
          19,
          64,
          30,
          37,
          65,
          64,
          5,
          21,
          51,
          64,
          60,
          64,
          61,
          64,
          52,
          64,
          51,
          37,
          0,
          61,
          64,
          64,
          77,
          64,
          72,
          64,
          72,
          0,
          1,
          28,
          64,
          66,
          64,
          11,
          1,
          51,
          48,
          64,
          66,
          52,
          51,
          51,
          11,
          9,
          61,
          28,
          76,
          91,
          64,
          51,
          35,
          93,
          16,
          72,
          64,
          75,
          70,
          64,
          75,
          51,
          84,
          30,
          37,
          51,
          64,
          37,
          37,
          64,
          91,
          91,
          47,
          39,
          64,
          60,
          51,
          6,
          83,
          61,
          0,
          61,
          0,
          61,
          9,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          83,
          36,
          65,
          36,
          40,
          65,
          0,
          61,
          61,
          83,
          29,
          64,
          47,
          18,
          0,
          65,
          83,
          61,
          61,
          0,
          25,
          61,
          0,
          40,
          61,
          65,
          77,
          0,
          33,
          0,
          0,
          0,
          0,
          0,
          61,
          0,
          0,
          7,
          48,
          29,
          0,
          30,
          0,
          46,
          40,
          28,
          5,
          5,
          76,
          77,
          54,
          54,
          91,
          39,
          28,
          66,
          35,
          62,
          34,
          5,
          51,
          37,
          5,
          29,
          3,
          53,
          5,
          0,
          60,
          33,
          76,
          0,
          40,
          84,
          5,
          81,
          75,
          37,
          36,
          83,
          19,
          77,
          35,
          83,
          98,
          75,
          98,
          13,
          17,
          50,
          62,
          3,
          39,
          94,
          75,
          83,
          13,
          22,
          37,
          33,
          98,
          78,
          21,
          53,
          68,
          13,
          41,
          58,
          53,
          60,
          75,
          83,
          18,
          13,
          21,
          83,
          57,
          56,
          54,
          60,
          60,
          13,
          77,
          71,
          75,
          77,
          22,
          83,
          98,
          75,
          9,
          3,
          83,
          83,
          77,
          12,
          14,
          8,
          75,
          13,
          28,
          3,
          76,
          49,
          65,
          75,
          23,
          83,
          33,
          79,
          22,
          77,
          26,
          77,
          60,
          48,
          41,
          77,
          18,
          77,
          53,
          76,
          77,
          82,
          42,
          77,
          77,
          57,
          75,
          77,
          68,
          0,
          60,
          75,
          1,
          45,
          13,
          19,
          3,
          22,
          84,
          98,
          8,
          22,
          88,
          77,
          30,
          40,
          83,
          18,
          5,
          77,
          7,
          8,
          79,
          83,
          13,
          39,
          77,
          17,
          60,
          77,
          13,
          77,
          13,
          76,
          42,
          83,
          87,
          37,
          83,
          57,
          77,
          83,
          7,
          53,
          17,
          48,
          9,
          75,
          13,
          9,
          22,
          3,
          3,
          83,
          83,
          94,
          13,
          48,
          75,
          89,
          75,
          9,
          22,
          83,
          3,
          77,
          76,
          77,
          45,
          3,
          23,
          75,
          83,
          71,
          77,
          13,
          61,
          77,
          77,
          13,
          45,
          42,
          75,
          18,
          3,
          22,
          13,
          10,
          11,
          82,
          54,
          45,
          48,
          21,
          13,
          77,
          3,
          9,
          83,
          75,
          83,
          75,
          33,
          76,
          77,
          4,
          37,
          46,
          75,
          68,
          77,
          3,
          3,
          5,
          60,
          95,
          77,
          34,
          18,
          75,
          77,
          10,
          76,
          75,
          75,
          27,
          33,
          83,
          60,
          18,
          75,
          75,
          26,
          95,
          22,
          42,
          94,
          42,
          77,
          9,
          9,
          5,
          83,
          22,
          11,
          22,
          33,
          77,
          1,
          83,
          38,
          10,
          76,
          13,
          75,
          45,
          42,
          3,
          16,
          75,
          42,
          77,
          77,
          45,
          77,
          13,
          83,
          60,
          77,
          56,
          77,
          76,
          43,
          72,
          75,
          77,
          75,
          13,
          60,
          77,
          28,
          5,
          75,
          77,
          2,
          18,
          3,
          75,
          75,
          9,
          9,
          34,
          35,
          36,
          8,
          8,
          8,
          3,
          35,
          84,
          8,
          39,
          17,
          36,
          42,
          8,
          15,
          8,
          42,
          36,
          8,
          42,
          11,
          37,
          15,
          42,
          0,
          73,
          73,
          8,
          73,
          36,
          77,
          65,
          73,
          73,
          8,
          36,
          35,
          73,
          42,
          73,
          23,
          20,
          76,
          36,
          20,
          34,
          75,
          35,
          35,
          15,
          42,
          8,
          28,
          42,
          73,
          82,
          8,
          35,
          20,
          83,
          36,
          35,
          37,
          0,
          29,
          42,
          98,
          83,
          97,
          48,
          83,
          94,
          87,
          22,
          18,
          50,
          10,
          83,
          53,
          33,
          5,
          13,
          83,
          53,
          75,
          98,
          53,
          53,
          60,
          7,
          53,
          40,
          53,
          20,
          73,
          15,
          20,
          82,
          20,
          98,
          29,
          27,
          27,
          30,
          20,
          20,
          29,
          82,
          73,
          31,
          19,
          16,
          7,
          82,
          15,
          31,
          7,
          53,
          36,
          76,
          75,
          25,
          61,
          0,
          24,
          75,
          15,
          75,
          77,
          74,
          73,
          40,
          8,
          61,
          75,
          75,
          16,
          75,
          75,
          47,
          47,
          15,
          7,
          82,
          20,
          20,
          20,
          75,
          20,
          45,
          13,
          75,
          73,
          5,
          13,
          18,
          15,
          7,
          73,
          76,
          13,
          20,
          77,
          18,
          83,
          51,
          93,
          23,
          20,
          75,
          81,
          39,
          75,
          75,
          82,
          76,
          28,
          8,
          98,
          73,
          76,
          73,
          42,
          77,
          20,
          73,
          20,
          3,
          65,
          82,
          84,
          75,
          87,
          48,
          32,
          10,
          33,
          73,
          42,
          87,
          33,
          58,
          88,
          61,
          5,
          33,
          17,
          94,
          29,
          7,
          89,
          73,
          8,
          73,
          75,
          19,
          0,
          77,
          42,
          20,
          9,
          13,
          48,
          13,
          13,
          13,
          27,
          6,
          95,
          58,
          76,
          30,
          18,
          89,
          27,
          62,
          15,
          22,
          83,
          62,
          9,
          24,
          33,
          22,
          13,
          8,
          13,
          26,
          13,
          39,
          8,
          77,
          9,
          6,
          49,
          9,
          29,
          82,
          29,
          32,
          20,
          51,
          10,
          84,
          53,
          17,
          14,
          30,
          22,
          77,
          17,
          13,
          20,
          35,
          5,
          13,
          98,
          20,
          77,
          50,
          49,
          13,
          76,
          11,
          3,
          60,
          10,
          36,
          28,
          51,
          56,
          39,
          92,
          51,
          40,
          83,
          18,
          42,
          2,
          81,
          22,
          76,
          81,
          54,
          42,
          82,
          14,
          10,
          6,
          9,
          17,
          9,
          55,
          29,
          22,
          48,
          43,
          12,
          2,
          9,
          2,
          48,
          9,
          76,
          24,
          11,
          1,
          9,
          29,
          94,
          21,
          9,
          9,
          9,
          10,
          58,
          76,
          79,
          31,
          14,
          4,
          9,
          98,
          33,
          98,
          98,
          48,
          24,
          14,
          38,
          96,
          23,
          26,
          77,
          21,
          55,
          79,
          79,
          23,
          11,
          76,
          36,
          96,
          21,
          14,
          11,
          86,
          9,
          49,
          57,
          5,
          36,
          1,
          94,
          23,
          58,
          86,
          79,
          94,
          9,
          11,
          34,
          2,
          2,
          72,
          14,
          82,
          17,
          6,
          21,
          96,
          77,
          86,
          79,
          4,
          79,
          33,
          77,
          3,
          48,
          76,
          98,
          23,
          96,
          59,
          9,
          5,
          75,
          27,
          9,
          13,
          23,
          79,
          43,
          96,
          59,
          9,
          9,
          11,
          77,
          32,
          5,
          86,
          26,
          76,
          26,
          23,
          59,
          23,
          79,
          25,
          5,
          9,
          98,
          79,
          96,
          47,
          96,
          23,
          77,
          25,
          86,
          9,
          9,
          48,
          2,
          23,
          75,
          59,
          9,
          6,
          9,
          9,
          11,
          77,
          26,
          26,
          88,
          72,
          57,
          77,
          9,
          79,
          94,
          5,
          11,
          9,
          9,
          98,
          98,
          83,
          26,
          97,
          59,
          82,
          59,
          86,
          97,
          24,
          13,
          49,
          23,
          72,
          57,
          75,
          4,
          47,
          77,
          23,
          25,
          45,
          79,
          9,
          79,
          23,
          26,
          23,
          10,
          75,
          26,
          82,
          77,
          82,
          47,
          57,
          75,
          84,
          21,
          97,
          26,
          33,
          75,
          26,
          75,
          72,
          26,
          11,
          77,
          62,
          98,
          76,
          3,
          62,
          33,
          77,
          9,
          94,
          17,
          27,
          25,
          34,
          2,
          77,
          98,
          9,
          97,
          3,
          21,
          57,
          33,
          13,
          77,
          9,
          61,
          49,
          42,
          97,
          1,
          5,
          25,
          3,
          32,
          61,
          30,
          23,
          11,
          5,
          29,
          21,
          64,
          97,
          9,
          72,
          17,
          77,
          75,
          53,
          75,
          9,
          2,
          75,
          48,
          9,
          62,
          57,
          9,
          77,
          1,
          76,
          14,
          98,
          21,
          23,
          26,
          14,
          45,
          76,
          26,
          21,
          14,
          9,
          77,
          79,
          79,
          22,
          14,
          77,
          31,
          79,
          79,
          45,
          78,
          5,
          40,
          19,
          79,
          59,
          10,
          92,
          31,
          79,
          79,
          6,
          59,
          23,
          23,
          79,
          48,
          14,
          23,
          45,
          57,
          79,
          77,
          17,
          26,
          75,
          26,
          33,
          57,
          86,
          46,
          24,
          77,
          45,
          77,
          9,
          24,
          2,
          77,
          9,
          86,
          3,
          26,
          45,
          39,
          49,
          27,
          4,
          5,
          69,
          6,
          45,
          77,
          98,
          98,
          59,
          25,
          98,
          98,
          3,
          13,
          27,
          13,
          34,
          23,
          3,
          98,
          5,
          79,
          98,
          98,
          98,
          98,
          96,
          11,
          18,
          21,
          44,
          22,
          90,
          50,
          49,
          12,
          84,
          11,
          67,
          68,
          16,
          5,
          98,
          9,
          13,
          2,
          33,
          53,
          5,
          18,
          41,
          5,
          22,
          77,
          26,
          33,
          34,
          78,
          18,
          11,
          9,
          23,
          5,
          13,
          27,
          10,
          12,
          9,
          3,
          38,
          5,
          23,
          9,
          98,
          98,
          25,
          55,
          13,
          8,
          76,
          32,
          11,
          5,
          27,
          22,
          10,
          79,
          50,
          9,
          95,
          9,
          55,
          33,
          22,
          27,
          1,
          98,
          47,
          22,
          26,
          5,
          98,
          77,
          28,
          10,
          9,
          1,
          98,
          26,
          11,
          75,
          76,
          3,
          96,
          95,
          5,
          95,
          98,
          9,
          60,
          9,
          98,
          98,
          9,
          77,
          5,
          53,
          98,
          77,
          60,
          98,
          75,
          5,
          21,
          98,
          5,
          75,
          98,
          90,
          29,
          6,
          90,
          90,
          95,
          90,
          33,
          90,
          90,
          90,
          90,
          62,
          90,
          5,
          95,
          90,
          90,
          90,
          90,
          43,
          90,
          43,
          90,
          95,
          90,
          90,
          90,
          9,
          90,
          90,
          77,
          90,
          90,
          77,
          90,
          7,
          90,
          5,
          90,
          90,
          90,
          90,
          90,
          90,
          71,
          90,
          95,
          90,
          90,
          9,
          90,
          90,
          90,
          90,
          77,
          90,
          43,
          90,
          77,
          90,
          25,
          59,
          27,
          62,
          10,
          10,
          11,
          6,
          27,
          58,
          77,
          59,
          77,
          26,
          10,
          68,
          43,
          8,
          12,
          59,
          59,
          59,
          98,
          33,
          74,
          98,
          74,
          68,
          9,
          10,
          89,
          10,
          9,
          98,
          59,
          9,
          10,
          5,
          24,
          43,
          59,
          10,
          59,
          92,
          9,
          2,
          10,
          68,
          74,
          68,
          10,
          74,
          5,
          84,
          6,
          33,
          5,
          22,
          98,
          22,
          77,
          98,
          12,
          52,
          75,
          77,
          85,
          75,
          26,
          10,
          95,
          6,
          48,
          33,
          47,
          47,
          10,
          9,
          4,
          10,
          33,
          9,
          9,
          13,
          26,
          5,
          63,
          30,
          49,
          60,
          77,
          77,
          5,
          13,
          71,
          34,
          98,
          75,
          33,
          77,
          67,
          27,
          5,
          9,
          55,
          3,
          90,
          11,
          75,
          32,
          9,
          23,
          9,
          32,
          11,
          9,
          11,
          3,
          75,
          27,
          79,
          16,
          26,
          29,
          76,
          23,
          5,
          21,
          38,
          26,
          32,
          83,
          79,
          77,
          7,
          71,
          98,
          77,
          1,
          79,
          33,
          23,
          69,
          26,
          98,
          5,
          33,
          10,
          29,
          32,
          5,
          75,
          11,
          43,
          98,
          27,
          98,
          77,
          22,
          42,
          9,
          3,
          72,
          77,
          59,
          5,
          5,
          79,
          5,
          79,
          90,
          32,
          13,
          5,
          75,
          77,
          5,
          75,
          75,
          18,
          75,
          14,
          5,
          90,
          5,
          95,
          21,
          77,
          60,
          45,
          60,
          3,
          98,
          77,
          27,
          16,
          21,
          58,
          90,
          75,
          71,
          24,
          79,
          5,
          77,
          98,
          42,
          98,
          98,
          33,
          88,
          84,
          23,
          5,
          3,
          75,
          92,
          1,
          1,
          9,
          11,
          56,
          44,
          99,
          80,
          12,
          85,
          32,
          28,
          25,
          2,
          27,
          12,
          43,
          44,
          99,
          12,
          43,
          68,
          50,
          43,
          9,
          24,
          99,
          92,
          44,
          38,
          11,
          56,
          89,
          80,
          1,
          1,
          44,
          75,
          82,
          43,
          9,
          92,
          94,
          99,
          50,
          50,
          68,
          5,
          99,
          59,
          99,
          2,
          77,
          43,
          80,
          74,
          43,
          10,
          56,
          11,
          9,
          1,
          99,
          24,
          9,
          44,
          43,
          9,
          43,
          98,
          43,
          44,
          43,
          9,
          22,
          92,
          2,
          43,
          34,
          17,
          43,
          82,
          43,
          33,
          9,
          34,
          99,
          99,
          76,
          25,
          50,
          80,
          12,
          9,
          99,
          50,
          87,
          43,
          43,
          99,
          74,
          44,
          43,
          44,
          48,
          50,
          12,
          38,
          7,
          33,
          56,
          43,
          77,
          29,
          2,
          43,
          49,
          25,
          55,
          1,
          9,
          11,
          2,
          33,
          8,
          75,
          5,
          10,
          2,
          1,
          2,
          77,
          10,
          3,
          32,
          10,
          79,
          27,
          10,
          70,
          54,
          10,
          53,
          9,
          9,
          5,
          45,
          12,
          87,
          44,
          44,
          10,
          11,
          50,
          50,
          50,
          44,
          50,
          50,
          27,
          11,
          79,
          50,
          43,
          26,
          9,
          95,
          10,
          32,
          11,
          44,
          43,
          10,
          50,
          99,
          46,
          50,
          4,
          59,
          44,
          41,
          44,
          97,
          44,
          77,
          44,
          59,
          4,
          43,
          41,
          2,
          68,
          27,
          99,
          7,
          27,
          49,
          2,
          26,
          37,
          2,
          11,
          9,
          8,
          99,
          76,
          4,
          43,
          37,
          77,
          10,
          50,
          69,
          44,
          26,
          9,
          9,
          99,
          4,
          22,
          76,
          44,
          9,
          34,
          23,
          25,
          44,
          48,
          75,
          10,
          10,
          63,
          94,
          2,
          7,
          12,
          77,
          76,
          86,
          9,
          76,
          59,
          21,
          11,
          86,
          33,
          43,
          86,
          29,
          77,
          3,
          36,
          63,
          77,
          69,
          75,
          11,
          96,
          16,
          11,
          97,
          94,
          11,
          88,
          88,
          32,
          69,
          27,
          75,
          38,
          88,
          9,
          6,
          9,
          9,
          77,
          21,
          4,
          85,
          69,
          98,
          77,
          77,
          9,
          18,
          26,
          9,
          83,
          47,
          77,
          26,
          71,
          9,
          79,
          9,
          77,
          27,
          25,
          79,
          9,
          1,
          26,
          77,
          27,
          88,
          3,
          22,
          76,
          77,
          77,
          69,
          38,
          9,
          47,
          88,
          26,
          16,
          8,
          33,
          87,
          75,
          21,
          75,
          12,
          40,
          36,
          3,
          35,
          5,
          27,
          75,
          42,
          75,
          23,
          1,
          77,
          0,
          7,
          82,
          98,
          29,
          23,
          63,
          24,
          9,
          88,
          61,
          5,
          8,
          71,
          13,
          68,
          3,
          3,
          5,
          51,
          75,
          23,
          47,
          19,
          21,
          82,
          9,
          11,
          77,
          72,
          96,
          5,
          27,
          52,
          5,
          59,
          5,
          27,
          44,
          73,
          76,
          32,
          75,
          65,
          5,
          20,
          40,
          49,
          24,
          26
         ],
         "xaxis": "x",
         "yaxis": "y"
        }
       ],
       "layout": {
        "barmode": "relative",
        "legend": {
         "tracegroupgap": 0
        },
        "margin": {
         "t": 60
        },
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "xaxis": {
         "anchor": "y",
         "domain": [
          0,
          1
         ],
         "title": {
          "text": "x"
         }
        },
        "yaxis": {
         "anchor": "x",
         "domain": [
          0,
          1
         ],
         "title": {
          "text": "count"
         }
        }
       }
      }
     },
     "metadata": {}
    }
   ],
   "source": [
    "px.histogram(x=hc100.labels_, nbins = 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "AgglomerativeClustering(affinity='cosine', compute_full_tree='auto',\n                        connectivity=None, distance_threshold=None,\n                        linkage='complete', memory=None, n_clusters=200,\n                        pooling_func='deprecated')"
     },
     "metadata": {},
     "execution_count": 26
    }
   ],
   "source": [
    "hc200 = AgglomerativeClustering(n_clusters=200,affinity = 'cosine', linkage='complete')\n",
    "hc200.fit(tfidf.toarray())\n",
    "#took approx 2min to run with sparse matrix of 1952 rows and 23084 words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "alignmentgroup": "True",
         "bingroup": "x",
         "hoverlabel": {
          "namelength": 0
         },
         "hovertemplate": "x=%{x}<br>count=%{y}",
         "legendgroup": "",
         "marker": {
          "color": "#636efa"
         },
         "name": "",
         "nbinsx": 200,
         "offsetgroup": "",
         "orientation": "v",
         "showlegend": false,
         "type": "histogram",
         "x": [
          158,
          183,
          183,
          183,
          129,
          129,
          171,
          107,
          34,
          95,
          129,
          107,
          175,
          156,
          82,
          24,
          15,
          51,
          34,
          129,
          129,
          162,
          32,
          58,
          39,
          87,
          48,
          75,
          15,
          129,
          156,
          38,
          78,
          129,
          34,
          36,
          129,
          156,
          158,
          133,
          129,
          34,
          129,
          129,
          95,
          51,
          182,
          129,
          95,
          129,
          137,
          34,
          32,
          129,
          15,
          147,
          156,
          129,
          134,
          129,
          182,
          129,
          104,
          129,
          156,
          28,
          87,
          182,
          129,
          129,
          57,
          129,
          158,
          129,
          158,
          157,
          51,
          72,
          129,
          133,
          129,
          20,
          51,
          156,
          94,
          129,
          133,
          104,
          156,
          156,
          20,
          44,
          182,
          72,
          47,
          183,
          129,
          156,
          124,
          187,
          122,
          158,
          129,
          151,
          141,
          129,
          151,
          156,
          84,
          168,
          28,
          156,
          129,
          34,
          34,
          129,
          183,
          183,
          5,
          78,
          129,
          82,
          107,
          102,
          83,
          182,
          157,
          140,
          49,
          140,
          130,
          87,
          49,
          49,
          49,
          87,
          87,
          49,
          83,
          12,
          32,
          12,
          70,
          32,
          49,
          182,
          140,
          83,
          7,
          129,
          5,
          112,
          49,
          32,
          83,
          140,
          140,
          49,
          132,
          182,
          49,
          70,
          182,
          32,
          19,
          49,
          60,
          49,
          49,
          49,
          49,
          49,
          140,
          49,
          49,
          55,
          172,
          7,
          87,
          137,
          87,
          22,
          14,
          71,
          15,
          15,
          35,
          19,
          110,
          110,
          183,
          194,
          71,
          133,
          178,
          61,
          37,
          15,
          107,
          34,
          15,
          7,
          42,
          85,
          15,
          87,
          82,
          46,
          4,
          87,
          14,
          84,
          15,
          163,
          151,
          34,
          12,
          83,
          95,
          19,
          178,
          83,
          90,
          169,
          90,
          50,
          106,
          101,
          61,
          0,
          194,
          93,
          151,
          83,
          50,
          180,
          28,
          41,
          90,
          38,
          147,
          85,
          68,
          50,
          159,
          155,
          174,
          82,
          151,
          83,
          26,
          52,
          63,
          83,
          13,
          6,
          110,
          82,
          134,
          50,
          21,
          143,
          151,
          19,
          180,
          83,
          90,
          151,
          75,
          42,
          83,
          83,
          97,
          150,
          40,
          108,
          151,
          164,
          71,
          42,
          35,
          100,
          32,
          169,
          123,
          83,
          46,
          39,
          195,
          21,
          189,
          19,
          134,
          2,
          114,
          97,
          112,
          57,
          85,
          4,
          19,
          196,
          120,
          21,
          57,
          13,
          151,
          57,
          68,
          87,
          134,
          151,
          86,
          91,
          50,
          139,
          42,
          180,
          84,
          90,
          142,
          195,
          131,
          21,
          30,
          70,
          83,
          112,
          43,
          97,
          55,
          108,
          39,
          83,
          50,
          194,
          19,
          166,
          134,
          21,
          164,
          19,
          52,
          35,
          120,
          83,
          10,
          28,
          83,
          13,
          19,
          83,
          55,
          85,
          166,
          94,
          130,
          151,
          50,
          31,
          180,
          42,
          42,
          83,
          83,
          93,
          164,
          94,
          151,
          179,
          151,
          130,
          195,
          83,
          0,
          21,
          35,
          21,
          91,
          42,
          123,
          151,
          83,
          143,
          21,
          52,
          140,
          19,
          21,
          50,
          91,
          77,
          151,
          112,
          42,
          180,
          52,
          115,
          20,
          144,
          110,
          91,
          2,
          147,
          50,
          21,
          42,
          130,
          83,
          151,
          83,
          151,
          46,
          35,
          57,
          58,
          34,
          22,
          169,
          68,
          19,
          53,
          42,
          45,
          82,
          23,
          21,
          37,
          112,
          151,
          19,
          115,
          47,
          151,
          151,
          1,
          41,
          83,
          134,
          112,
          151,
          151,
          33,
          23,
          195,
          77,
          93,
          77,
          97,
          130,
          130,
          56,
          83,
          195,
          20,
          195,
          46,
          19,
          86,
          83,
          136,
          115,
          47,
          52,
          151,
          91,
          77,
          42,
          122,
          151,
          120,
          19,
          19,
          91,
          19,
          50,
          83,
          134,
          19,
          6,
          19,
          35,
          66,
          158,
          151,
          21,
          151,
          50,
          82,
          21,
          71,
          45,
          151,
          97,
          36,
          112,
          42,
          151,
          151,
          31,
          31,
          160,
          124,
          12,
          108,
          108,
          103,
          42,
          178,
          84,
          108,
          194,
          116,
          12,
          77,
          103,
          118,
          108,
          120,
          12,
          108,
          77,
          20,
          34,
          118,
          120,
          157,
          8,
          8,
          108,
          8,
          12,
          19,
          32,
          8,
          8,
          108,
          12,
          178,
          8,
          77,
          8,
          123,
          27,
          4,
          12,
          27,
          160,
          151,
          178,
          178,
          118,
          120,
          108,
          71,
          120,
          8,
          196,
          103,
          178,
          27,
          83,
          12,
          124,
          34,
          87,
          7,
          77,
          90,
          83,
          48,
          94,
          83,
          93,
          10,
          180,
          112,
          101,
          176,
          83,
          85,
          41,
          56,
          164,
          83,
          85,
          151,
          90,
          85,
          85,
          134,
          55,
          85,
          70,
          85,
          67,
          8,
          121,
          27,
          190,
          27,
          90,
          7,
          1,
          127,
          30,
          27,
          27,
          7,
          196,
          8,
          186,
          139,
          126,
          73,
          196,
          118,
          186,
          184,
          174,
          12,
          4,
          151,
          132,
          140,
          87,
          59,
          151,
          121,
          151,
          21,
          149,
          8,
          14,
          108,
          140,
          151,
          151,
          126,
          151,
          151,
          5,
          5,
          121,
          73,
          196,
          67,
          67,
          27,
          151,
          67,
          91,
          52,
          151,
          8,
          45,
          164,
          112,
          121,
          73,
          8,
          4,
          50,
          67,
          21,
          112,
          83,
          107,
          187,
          123,
          67,
          151,
          163,
          194,
          151,
          151,
          196,
          4,
          71,
          103,
          90,
          8,
          4,
          8,
          77,
          21,
          27,
          8,
          67,
          42,
          32,
          196,
          84,
          151,
          10,
          172,
          9,
          176,
          46,
          8,
          120,
          10,
          46,
          155,
          80,
          140,
          56,
          96,
          106,
          93,
          7,
          73,
          179,
          8,
          103,
          8,
          151,
          95,
          87,
          21,
          120,
          67,
          130,
          164,
          172,
          164,
          164,
          52,
          18,
          102,
          23,
          155,
          47,
          30,
          181,
          179,
          154,
          61,
          121,
          180,
          83,
          61,
          130,
          59,
          41,
          195,
          50,
          108,
          164,
          33,
          164,
          78,
          173,
          21,
          130,
          98,
          100,
          130,
          7,
          196,
          7,
          9,
          67,
          107,
          115,
          84,
          85,
          116,
          117,
          30,
          180,
          21,
          106,
          52,
          27,
          124,
          56,
          52,
          90,
          27,
          57,
          101,
          100,
          50,
          35,
          20,
          0,
          134,
          115,
          12,
          71,
          156,
          6,
          194,
          92,
          107,
          70,
          83,
          112,
          120,
          145,
          163,
          180,
          35,
          163,
          110,
          120,
          196,
          40,
          176,
          102,
          31,
          106,
          130,
          111,
          175,
          180,
          2,
          66,
          65,
          36,
          130,
          36,
          94,
          130,
          35,
          170,
          20,
          29,
          130,
          175,
          93,
          63,
          130,
          130,
          31,
          115,
          155,
          47,
          39,
          167,
          152,
          58,
          130,
          3,
          41,
          3,
          3,
          94,
          59,
          40,
          136,
          193,
          123,
          33,
          57,
          63,
          111,
          39,
          39,
          123,
          17,
          47,
          153,
          193,
          63,
          40,
          105,
          192,
          130,
          100,
          13,
          56,
          153,
          29,
          93,
          123,
          188,
          192,
          39,
          93,
          44,
          17,
          64,
          148,
          36,
          88,
          40,
          196,
          166,
          98,
          177,
          193,
          21,
          192,
          39,
          58,
          39,
          60,
          57,
          53,
          2,
          47,
          3,
          123,
          193,
          25,
          81,
          56,
          151,
          18,
          130,
          50,
          123,
          39,
          11,
          193,
          146,
          81,
          130,
          105,
          97,
          9,
          56,
          192,
          24,
          35,
          24,
          123,
          25,
          109,
          39,
          79,
          56,
          130,
          119,
          39,
          193,
          5,
          193,
          123,
          97,
          79,
          198,
          130,
          130,
          2,
          145,
          109,
          151,
          146,
          81,
          98,
          81,
          130,
          105,
          97,
          24,
          33,
          131,
          88,
          13,
          57,
          130,
          39,
          93,
          56,
          17,
          130,
          44,
          3,
          119,
          83,
          24,
          48,
          25,
          190,
          25,
          192,
          48,
          16,
          50,
          100,
          123,
          158,
          13,
          151,
          58,
          5,
          57,
          123,
          185,
          91,
          39,
          31,
          39,
          123,
          189,
          123,
          176,
          151,
          189,
          190,
          19,
          190,
          5,
          13,
          151,
          84,
          63,
          48,
          189,
          60,
          151,
          189,
          125,
          158,
          189,
          20,
          57,
          61,
          3,
          4,
          53,
          61,
          60,
          97,
          31,
          93,
          116,
          1,
          79,
          160,
          197,
          21,
          90,
          31,
          48,
          42,
          63,
          13,
          60,
          50,
          21,
          31,
          182,
          100,
          120,
          48,
          86,
          45,
          79,
          42,
          9,
          182,
          168,
          123,
          17,
          56,
          74,
          63,
          129,
          48,
          31,
          88,
          116,
          21,
          151,
          85,
          151,
          44,
          148,
          151,
          94,
          31,
          61,
          13,
          31,
          57,
          51,
          47,
          40,
          3,
          177,
          123,
          24,
          40,
          91,
          35,
          24,
          63,
          152,
          81,
          57,
          39,
          39,
          180,
          40,
          97,
          167,
          39,
          39,
          91,
          38,
          56,
          14,
          95,
          39,
          25,
          62,
          92,
          167,
          39,
          39,
          98,
          25,
          123,
          123,
          39,
          94,
          152,
          123,
          91,
          13,
          39,
          21,
          166,
          33,
          151,
          24,
          46,
          13,
          192,
          22,
          16,
          21,
          91,
          21,
          81,
          16,
          148,
          57,
          31,
          192,
          0,
          24,
          91,
          194,
          100,
          127,
          113,
          56,
          69,
          98,
          91,
          97,
          90,
          90,
          146,
          79,
          90,
          90,
          42,
          50,
          1,
          164,
          64,
          123,
          42,
          90,
          56,
          39,
          119,
          90,
          90,
          90,
          193,
          20,
          181,
          147,
          165,
          180,
          89,
          101,
          100,
          65,
          84,
          20,
          135,
          68,
          122,
          56,
          90,
          130,
          50,
          145,
          96,
          85,
          56,
          112,
          114,
          43,
          180,
          97,
          33,
          96,
          64,
          38,
          26,
          20,
          130,
          109,
          56,
          164,
          1,
          62,
          65,
          31,
          0,
          136,
          56,
          123,
          130,
          90,
          90,
          79,
          111,
          50,
          142,
          47,
          9,
          20,
          56,
          127,
          180,
          176,
          39,
          101,
          130,
          23,
          130,
          111,
          60,
          180,
          1,
          29,
          90,
          5,
          180,
          33,
          56,
          90,
          19,
          72,
          115,
          130,
          29,
          119,
          24,
          20,
          151,
          47,
          42,
          193,
          23,
          56,
          23,
          119,
          130,
          134,
          31,
          90,
          90,
          130,
          57,
          43,
          85,
          90,
          21,
          134,
          90,
          151,
          56,
          63,
          90,
          45,
          151,
          90,
          89,
          175,
          98,
          89,
          89,
          23,
          89,
          60,
          89,
          89,
          89,
          89,
          61,
          89,
          56,
          23,
          89,
          89,
          89,
          89,
          66,
          89,
          66,
          89,
          23,
          89,
          89,
          89,
          31,
          89,
          89,
          97,
          89,
          89,
          21,
          89,
          73,
          89,
          56,
          89,
          89,
          89,
          89,
          89,
          89,
          143,
          89,
          23,
          89,
          89,
          130,
          89,
          89,
          89,
          89,
          21,
          89,
          66,
          89,
          21,
          89,
          185,
          25,
          18,
          61,
          62,
          62,
          17,
          98,
          1,
          188,
          97,
          25,
          19,
          33,
          62,
          68,
          66,
          142,
          65,
          25,
          25,
          25,
          3,
          46,
          149,
          90,
          149,
          68,
          130,
          62,
          179,
          62,
          130,
          90,
          25,
          130,
          62,
          56,
          16,
          66,
          25,
          62,
          25,
          92,
          130,
          148,
          62,
          68,
          149,
          68,
          62,
          149,
          56,
          84,
          98,
          46,
          56,
          195,
          90,
          195,
          21,
          90,
          199,
          191,
          151,
          57,
          171,
          151,
          189,
          115,
          23,
          98,
          94,
          41,
          5,
          5,
          115,
          31,
          58,
          115,
          41,
          130,
          130,
          164,
          24,
          56,
          128,
          137,
          100,
          134,
          21,
          57,
          56,
          164,
          143,
          37,
          90,
          169,
          96,
          19,
          135,
          1,
          56,
          130,
          111,
          0,
          89,
          17,
          151,
          138,
          130,
          123,
          130,
          9,
          17,
          130,
          17,
          53,
          151,
          18,
          39,
          126,
          24,
          74,
          47,
          123,
          56,
          63,
          76,
          189,
          9,
          83,
          39,
          57,
          184,
          143,
          3,
          57,
          29,
          39,
          60,
          123,
          69,
          24,
          3,
          56,
          96,
          115,
          74,
          9,
          56,
          151,
          17,
          66,
          90,
          154,
          90,
          57,
          180,
          120,
          44,
          53,
          158,
          97,
          146,
          56,
          56,
          39,
          56,
          39,
          89,
          138,
          50,
          43,
          125,
          97,
          56,
          151,
          151,
          26,
          151,
          117,
          43,
          89,
          56,
          23,
          147,
          21,
          134,
          91,
          134,
          42,
          90,
          19,
          18,
          126,
          177,
          155,
          89,
          151,
          143,
          16,
          39,
          56,
          57,
          90,
          120,
          90,
          90,
          46,
          131,
          84,
          123,
          56,
          0,
          151,
          92,
          29,
          29,
          130,
          17,
          6,
          54,
          99,
          161,
          150,
          171,
          9,
          72,
          132,
          145,
          127,
          65,
          11,
          54,
          99,
          199,
          11,
          68,
          101,
          11,
          75,
          16,
          99,
          92,
          54,
          76,
          20,
          6,
          179,
          161,
          86,
          29,
          54,
          151,
          196,
          66,
          44,
          92,
          93,
          99,
          101,
          101,
          68,
          45,
          99,
          146,
          99,
          145,
          57,
          11,
          161,
          149,
          66,
          176,
          6,
          20,
          75,
          29,
          99,
          16,
          75,
          165,
          66,
          130,
          66,
          90,
          66,
          54,
          11,
          130,
          180,
          92,
          36,
          66,
          64,
          106,
          66,
          196,
          11,
          96,
          130,
          64,
          99,
          99,
          4,
          132,
          101,
          161,
          199,
          130,
          99,
          101,
          10,
          11,
          66,
          99,
          149,
          54,
          66,
          54,
          172,
          101,
          199,
          76,
          73,
          96,
          6,
          11,
          21,
          175,
          36,
          11,
          100,
          79,
          111,
          29,
          130,
          20,
          36,
          60,
          142,
          151,
          56,
          115,
          36,
          29,
          36,
          57,
          115,
          0,
          9,
          115,
          39,
          18,
          115,
          141,
          110,
          176,
          85,
          130,
          130,
          56,
          91,
          65,
          10,
          54,
          54,
          115,
          17,
          101,
          101,
          101,
          54,
          101,
          101,
          154,
          17,
          39,
          101,
          11,
          33,
          130,
          23,
          115,
          9,
          17,
          54,
          66,
          115,
          101,
          99,
          22,
          101,
          113,
          146,
          54,
          159,
          54,
          48,
          54,
          97,
          165,
          25,
          113,
          11,
          159,
          145,
          68,
          18,
          99,
          73,
          154,
          100,
          148,
          24,
          34,
          145,
          20,
          75,
          173,
          99,
          4,
          58,
          66,
          34,
          57,
          176,
          101,
          69,
          54,
          33,
          130,
          130,
          99,
          58,
          180,
          4,
          54,
          130,
          160,
          123,
          132,
          54,
          2,
          151,
          115,
          115,
          128,
          93,
          145,
          55,
          65,
          57,
          4,
          192,
          130,
          4,
          25,
          63,
          105,
          192,
          46,
          11,
          192,
          7,
          57,
          0,
          153,
          128,
          21,
          69,
          151,
          105,
          193,
          126,
          105,
          48,
          93,
          105,
          80,
          80,
          138,
          69,
          154,
          151,
          136,
          131,
          44,
          102,
          44,
          44,
          21,
          177,
          58,
          171,
          69,
          3,
          57,
          57,
          44,
          26,
          24,
          44,
          83,
          5,
          97,
          24,
          143,
          81,
          39,
          130,
          57,
          154,
          79,
          39,
          130,
          29,
          24,
          97,
          1,
          80,
          0,
          180,
          47,
          57,
          57,
          69,
          76,
          44,
          5,
          131,
          24,
          126,
          142,
          46,
          10,
          125,
          63,
          151,
          150,
          70,
          12,
          53,
          178,
          56,
          18,
          151,
          120,
          151,
          123,
          162,
          97,
          87,
          73,
          144,
          90,
          74,
          123,
          128,
          170,
          130,
          80,
          140,
          43,
          142,
          143,
          52,
          68,
          0,
          0,
          56,
          107,
          151,
          123,
          5,
          95,
          63,
          144,
          31,
          20,
          57,
          88,
          193,
          43,
          18,
          191,
          43,
          25,
          43,
          154,
          165,
          8,
          4,
          138,
          151,
          32,
          43,
          67,
          14,
          100,
          170,
          33
         ],
         "xaxis": "x",
         "yaxis": "y"
        }
       ],
       "layout": {
        "barmode": "relative",
        "legend": {
         "tracegroupgap": 0
        },
        "margin": {
         "t": 60
        },
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "xaxis": {
         "anchor": "y",
         "domain": [
          0,
          1
         ],
         "title": {
          "text": "x"
         }
        },
        "yaxis": {
         "anchor": "x",
         "domain": [
          0,
          1
         ],
         "title": {
          "text": "count"
         }
        }
       }
      }
     },
     "metadata": {}
    }
   ],
   "source": [
    "px.histogram(x=hc200.labels_, nbins = 200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "tags": [
     "outputPrepend"
    ]
   },
   "outputs": [],
   "source": [
    "data = data.assign(hc100_labels = hc100.labels_,\n",
    "                   hc200_labels = hc200.labels_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = data[['ProjURL','hc100_labels']]\n",
    "x.columns = ['ID','cl']\n",
    "x.to_csv(os.path.join(data_dir,'ja-clust100.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "datax = data[['ProjURL','hc200_labels']]\n",
    "x.columns = ['ID','cl']\n",
    "x.to_csv(os.path.join(data_dir,'ja-clust200.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "'/Users/jillianaugustine/Dropbox/EUvsVirus'"
     },
     "metadata": {},
     "execution_count": 65
    }
   ],
   "source": [
    "data_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "def topwords(series,n):\n",
    "    ''' returns the top_n words of a series as a list'''\n",
    "    return series.value_counts().head(n).index.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "hc100_labels\n0       [use, ventil, air, pressur, devic]\n1           [use, task, time, work, solut]\n2       [work, remot, peopl, use, teacher]\n3           [risk, data, user, use, covid]\n4     [european, need, develop, educ, dog]\n                      ...                 \n95         [bot, chatbot, use, help, user]\n96     [tourism, solut, busi, peopl, need]\n97     [devic, distanc, use, busi, social]\n98           [help, app, need, peopl, use]\n99    [student, learn, educ, use, teacher]\nName: word, Length: 100, dtype: object"
     },
     "metadata": {},
     "execution_count": 158
    }
   ],
   "source": [
    "split_words_and_labels.groupby('hc100_labels').word.agg(topwords, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "hc200_labels\n0              [tool, data, use, develop, user]\n1              [women, app, need, help, health]\n2          [network, data, event, system, peer]\n3          [custom, busi, merchant, need, fund]\n4      [use, research, data, develop, platform]\n                         ...                   \n195             [app, user, use, develop, work]\n196    [research, 19, remedi, develop, project]\n197           [screener, dasha, set, next, way]\n198    [kan, strategi, talent, opportun, thank]\n199     [peopl, young, project, platform, work]\nName: word, Length: 200, dtype: object"
     },
     "metadata": {},
     "execution_count": 112
    }
   ],
   "source": [
    "split_words_and_labels.groupby('hc200_labels').word.agg(topwords, 5)"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python37464bitconda584155ea88da456b88db3adc5785edec",
   "display_name": "Python 3.7.4 64-bit (conda)"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}