hierarchical-clustering.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hierarchical Clustering Panel of Economists\n",
    "## Author: Oliver Gladfelter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "from sklearn.cluster import AgglomerativeClustering\n",
    "from sklearn.preprocessing import normalize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"output-data/answersAsColumns.csv\")\n",
    "\n",
    "data = data[data['name']!='Amy Finkelstein'] # too many no opinion / did not answers\n",
    "data = data[data['name']!='Raj Chetty'] # too many no opinion / did not answers\n",
    "data = data.reset_index()\n",
    "del data['index']\n",
    "\n",
    "data = data.replace({'Strongly Agree': 1, 'Agree': 1, 'Uncertain': 2, 'Disagree': 3, 'Strongly Disagree': 3, 'Did Not Answer': 0, 'No Opinion': 4})\n",
    "data = data.fillna(0)\n",
    "\n",
    "data['questions_not_answered'] = (data == 0).astype(int).sum(axis=1) # count number of 0s in rows\n",
    "data = data[data['questions_not_answered']<=52] # remove if there's a lot of missing values\n",
    "\n",
    "data2 = data.iloc[:,1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlYAAAGyCAYAAAAmmIoWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de5wkVX338c+PXS4Ccl/kKouyoqBxxAUUbyOIiIJg1ERXYUHiah5QUROBRKOPGgNJVKLBmDXogrgCwQt4iYm3jY9JhCwyERXzQIgicluJoFEUiSd/nDPSDLPsTNevd7p3P+/Xa17TXV3z6zPV1VXfOqe6OkopSJIkqbtN5roBkiRJGwqDlSRJUhKDlSRJUhKDlSRJUhKDlSRJUhKDlSRJUhKDlaSREBEnRMRX57odkvRADFaS+hYR342IuyLiJxFxR0T8c0S8MiLctkjaKLnxk9TV0aWUBwN7AWcCpwHnrs8GRMT8Ya4naeNhsJKUopRyZynlMuC3gaUR8eiI2Dwi/jwiboiIWyPi/RHxIICIGI+IGyPi9RFxW0TcHBEnTtaLiB0j4rKI+HFEXAE8vPf5IqJExMkRcS1wbZt2SET8a0Tc2X4f0jP/3hHxlda79oWIOCciLmiPLWz1ToqIG4Avtel/GxG3tHpfiYj9e+qtiIj3RcTfRcR/R8Q/RcQuEXF2RPwoIr4TEY/rmf+0iPhBe/5/j4jDBvAySJpjBitJqUopVwA3Ak8BzgIeAYwB+wC7A3/UM/suwLZt+knAORGxfXvsHODnwK7Ay9rPVMcCBwP7RcQOwGeA9wA7Au8CPhMRO7Z5VwJXtMfeAhw3Tb2nAY8Cjmj3/w5YBOwMfB34yJT5fwt4I7AT8AvgX9p8OwGXtDYQEfsCpwAHtt69I4DvTvP8kkacwUrSINwE7AC8HHhtKeW/Sik/Ad4BvKhnvl8Cby2l/LKU8lngv4F9I2Ie8Hzgj0opPy2lfBM4b5rn+ZNW+y7gOcC1pZQPl1LuKaV8FPgOcHREPBQ4sNW7u5TyVeCyaeq9pT3fXQCllA+WUn5SSvkFNYw9NiK27Zn/E6WUK0spPwc+Afy8lHJ+KeV/gIuAyR6r/wE2pwbATUsp3y2l/MdsFqik0WCwkjQIuwPzgS2BK9uJ7XcAnwMW9Mx3eynlnp77PwO2bvPMB77f89j3pnme3sd3m2ae77W27Ab8VynlZ2v52/tNi4h5EXFmRPxHRPyYe3uYduqZ/9ae23dNc39rgFLKdcCp1HB2W0RcGBG7TfP8kkacwUpSqog4kBpmPkkNF/uXUrZrP9uWUraeQZk1wD3Anj3THjrNfKXn9k3UE+h7PRT4AXAzsENEbNnz2J7cX2+9JcAxwDOow5UL2/RYR9unVUpZWUp5cmtjoQ6TStrAGKwkpYiIbSLiKOBC4IJSyr8BHwDeHRE7t3l2j4gjHqgOQBtK+zjwlojYMiL2A5au488+CzwiIpZExPyI+G1gP+DTpZTvAatbvc0i4onA0euo92DqeVO3U3ve3rGudq9NROwbEYdGxObU88buog4PStrAGKwkdfWpiPgJdRjtD6knbE9+uu804Drga2047QvAvjOsewp1KO0WYAXwoQeauZRyO3AU8HpqGHoDcFQp5YdtlpcAT2yPvZ16DtQvHqDk+dShxB8A3wa+NsN2T2dz6qUofkj9f3YG/qBDPUlDKkop655LkjYwEXER8J1Sypvnui2SNhz2WEnaKETEgRHx8IjYJCKeRT1/6pNz3S5JGxavLixpY7EL9bytHanX2frdUspVc9skSRsahwIlSZKSOBQoSZKUxGAlSZKUZCjOsdppp53KwoUL57oZkiRJ63TllVf+sJSyYLrHhiJYLVy4kNWrV891MyRJktYpIqb7ii3AoUBJkqQ0BitJkqQkBitJkqQkBitJkqQkBitJkqQkBitJkqQkBitJkqQkBitJkqQkBitJkqQkBitJkqQkBitJkqQkBitJkqQkBitJkqQkBitJkqQkBitJkqQk8+e6AaNs+XJYuXKuWyFpWCxZAsuWzXUrJM0le6w6WLkSJibmuhWShsHEhAdakuyx6mxsDFatmutWSJpr4+Nz3QJJw8AeK0mSpCQGK0mSpCQGK0mSpCQGK0mSpCQGK0mSpCQGK0mSpCQGK0mSpCQGK0mSpCQGK0mSpCQGK0mSpCQGK0mSpCQGK0mSpCTrDFYR8cGIuC0ivtkzbYeI+HxEXNt+b9+mR0S8JyKui4hvRMQBg2y8JEnSMJlJj9UK4FlTpp0OfLGUsgj4YrsPcCSwqP0sA/4qp5mSJEnDb53BqpTyFeC/pkw+Bjiv3T4POLZn+vml+hqwXUTsmtVYSZKkYdbvOVYPKaXcDNB+79ym7w58v2e+G9s0SZKkDV72yesxzbQy7YwRyyJidUSsXrNmTXIzJEmS1r9+g9Wtk0N87fdtbfqNwJ498+0B3DRdgVLK8lLK4lLK4gULFvTZDEmSpOHRb7C6DFjabi8FLu2Zfnz7dOATgDsnhwwlSZI2dPPXNUNEfBQYB3aKiBuBNwNnAhdHxEnADcAL2+yfBZ4NXAf8DDhxAG2WJEkaSusMVqWUF6/locOmmbcAJ3dtlCRJ0ijyyuuSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJOgWriHhtRHwrIr4ZER+NiC0iYu+IuDwiro2IiyJis6zGSpIkDbO+g1VE7A68GlhcSnk0MA94EXAW8O5SyiLgR8BJGQ2VJEkadl2HAucDD4qI+cCWwM3AocAl7fHzgGM7PockSdJI6DtYlVJ+APw5cAM1UN0JXAncUUq5p812I7B710ZKkiSNgi5DgdsDxwB7A7sBWwFHTjNrWcvfL4uI1RGxes2aNf02Q5IkaWh0GQp8BvCfpZQ1pZRfAh8HDgG2a0ODAHsAN033x6WU5aWUxaWUxQsWLOjQDEmSpOHQJVjdADwhIraMiAAOA74NfBl4QZtnKXBptyZKkiSNhi7nWF1OPUn968DVrdZy4DTgdRFxHbAjcG5COyVJkobe/HXPsnallDcDb54y+XrgoC51JUmSRpFXXpckSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUrSKVhFxHYRcUlEfCciromIJ0bEDhHx+Yi4tv3ePquxkiRJw6xrj9VfAJ8rpTwSeCxwDXA68MVSyiLgi+2+JEnSBq/vYBUR2wBPBc4FKKXcXUq5AzgGOK/Ndh5wbNdGSpIkjYIuPVYPA9YAH4qIqyLibyJiK+AhpZSbAdrvnRPaKUmSNPS6BKv5wAHAX5VSHgf8lFkM+0XEsohYHRGr16xZ06EZkiRJw6FLsLoRuLGUcnm7fwk1aN0aEbsCtN+3TffHpZTlpZTFpZTFCxYs6NAMSZKk4dB3sCql3AJ8PyL2bZMOA74NXAYsbdOWApd2aqEkSdKImN/x718FfCQiNgOuB06khrWLI+Ik4AbghR2fQ5IkaSR0ClallAlg8TQPHdalriRJ0ijyyuuSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJ5s91AyR1tHw5rFw5163QxNn19/ipc9sOVUuWwLJlc90KbYQMVtKoW7kSJiZgbGyuW7JRWzVmoBoaExP1t8FKc8BgJW0IxsZg1aq5boU0HMbH57oF2oh5jpUkSVISg5UkSVISg5UkSVISg5UkSVISg5UkSVISg5UkSVISg5UkSVISg5UkSVISg5UkSVISg5UkSVISg5UkSVISg5UkSVISg5UkSVKSzsEqIuZFxFUR8el2f++IuDwiro2IiyJis+7NlCRJGn4ZPVavAa7puX8W8O5SyiLgR8BJCc8hSZI09DoFq4jYA3gO8DftfgCHApe0Wc4Dju3yHJIkSaOia4/V2cAbgF+1+zsCd5RS7mn3bwR27/gckiRJI6HvYBURRwG3lVKu7J08zaxlLX+/LCJWR8TqNWvW9NsMSZKkodGlx+pJwHMj4rvAhdQhwLOB7SJifptnD+Cm6f64lLK8lLK4lLJ4wYIFHZohSZI0HPoOVqWUM0ope5RSFgIvAr5USnkJ8GXgBW22pcClnVspSZI0AgZxHavTgNdFxHXUc67OHcBzSJIkDZ35655l3Uopq4BV7fb1wEEZdSVJkkaJV16XJElKYrCSJElKYrCSJElKYrCSJElKYrCSJElKYrCSJElKYrCSJElKYrCSJElKYrCSJElKYrCSJElKYrCSJElKYrCSJElKYrCSJElKYrCSJElKYrCSJElKYrCSJElKYrCSJElKYrCSJElKYrCSJElKYrCSJElKMn+uGzAoy69czsqrVw70OSZuORuA8RWnDvR5AJY8ZgnLHr9s4M8jSZL6t8EGq5VXr2TilgnGdhkb2HOMnT74QAUwccsEgMFKkqQht8EGK4CxXcZYdcKquW5GZ+Mrxue6CZIkaQY8x0qSJCmJwUqSJCmJwUqSJCmJwUqSJCnJBn3yuiQNjeXLYeVgLwGjZqJ+kprx8TltxkZlyRJY5ifXwR4rSVo/Vq68d4evwRobqz9aPyYmPGjoYY+VJK0vY2OwatVct0LKZc/gfdhjJUmSlMQeqwHJ/EqdySuvZ1wo1K/GkSRpcOyxGpDJr9TJMLbLWMpX80zcMjHw70+UJGljZo/VAA3bV+r41TiSJA2WPVaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJDFaSJElJ/K5ASZKG3fLlsHLlXLdiehMT9ff4+Jw2Y62WLIFly9bb09ljJUnSsFu58t4AM2zGxurPMJqYWO+BdKR6rJZfuZyVV89sAU3cUlfA8RXj65x3yWOWsOzx6y/NSpI0a2NjsGrVXLditMxBL9pI9VitvHrlrwPTuoztMsbYLutO0BO3TMw4rEmSJD2QkeqxghqYVp2wKq3eTHq0JEmSZmKkeqwkSZKGmcFKkiQpycgNBUpSuvXxUfb18ZH09fyxckn3Z4+VJK2Pj7IP+iPpc/Cxckn3Z4+VJMHof5R9WC/OKG1kDFaSJGl0zGbofjZD8ElD6Q4FSpKk0TGbofuZDsEnDqXbY6WN2zB//9ZMDfv3dM2UJ15LmqnsofvE7afBaoTM5it9pjObr/lZmw3u638mj3yG9XuuZmJY237zzXDrrTOb9847Z37EaADTqMg8cMs+gPJ9NDAGqxEy+ZU+M/mqnun0+3eTJoPZBhWsYPRPWh5W4+M1WGUGv8mdizsEjYLMAzffRyOj72AVEXsC5wO7AL8ClpdS/iIidgAuAhYC3wV+q5Tyo+5NFeR/pc9s+PU/mrUh7q6X1othPHDzfTRQXXqs7gFeX0r5ekQ8GLgyIj4PnAB8sZRyZkScDpwOnNa9qRL550QN4vwku9glaaPV96cCSyk3l1K+3m7/BLgG2B04BjivzXYecGzXRkq/ln0hx+yLNnqRRknaqKWcYxURC4HHAZcDDyml3Aw1fEXEzhnPIf1adtf6IHrBPMFUkjZKna9jFRFbAx8DTi2l/HgWf7csIlZHxOo1a9Z0bYbUv8xesMweMHu/JGnkdOqxiohNqaHqI6WUj7fJt0bErq23alfgtun+tpSyHFgOsHjx4tKlHVJnnmAqSUrQ5VOBAZwLXFNKeVfPQ5cBS4Ez2+9LO7VQAzfT62PN9jpYG9w1ryRJWocuQ4FPAo4DDo2IifbzbGqgOjwirgUOb/c1xCavj7UuY7uMzfhaWBO3THS6mKkkSaOo7x6rUspXgVjLw4f1W1dzI/v6WF7zSpK0MfLK69IgZHzSMOMaW36qUJLWK4OVNAgZX2XR9dOFfm3F3FrfX/A9V1/GbXiX7sNgJQ3KXH/S0E8Vzq31/QXfc/Fl3IZ36X4MVpI0KHMdrgfN8C7dj8FKGiWzGV6azdCQwzkbnvUxFLk+hh9dNzViDFbSKJnN8NJMh4YcztkwrY+hyHXVvvlmuPXW/uvfeWf3byDY0IKZH4wZegYradRkDy85nLPhmuuhyPHxGqzm4vwv2DAPGvxgzNAzWEmSBmcuw92GetAwDIFZa2WwkiSNjkGdZwgObylFl6+0kSRp/ZocCpuJsbHZnWu4Pq87pg2WPVaSpNEyiKEwh7eUxB4rSZKkJAYrSZKkJA4FStIomenJ2564Lc0Je6wkaZTM9ORtT9yW5oQ9VpI0arxIrDS0DFaSJG3s/B7SNA4FSpK0sRvE9cE20iFme6wkSZJDzEnssZIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUpisJIkSUoykGAVEc+KiH+PiOsi4vRBPIckSdKwSQ9WETEPOAc4EtgPeHFE7Jf9PJIkScNmED1WBwHXlVKuL6XcDVwIHDOA55EkSRoqgwhWuwPf77l/Y5smSZK0QYtSSm7BiBcCR5RSfqfdPw44qJTyqinzLQOWtbv7Av+e2hBJkqTB2KuUsmC6B+YP4MluBPbsub8HcNPUmUopy4HlA3h+SZKkOTGIocB/BRZFxN4RsRnwIuCyATyPJEnSUEnvsSql3BMRpwB/D8wDPlhK+Vb280iSJA2b9HOsJEmSNlZeeV2SJCmJwUqSJCmJwWqAIiLmug0zNSptHZV2wmi1dVJWmyNiq4w6U2ruMorLVNLoyNjGDH2wioh9I+KJEbFp+7qc7PoPGkDNRQBlgCewJe4Ad4H8tkbEnhGx2eQONiI6r2sRcTBwSOfGDVBEPLldu41SShlkEEhcBxZGxLYRsW1GmyPiGOCsiNg5o32t5hHAJ7jvpVwGIuH/XxQRiyNiXtY2KyL2j4inRcSOGfWmqT/QwJq4rqbuAyLi6Ih4TWbNVneftg5snl07U0Q8ISKOa783G/BzZa0Du/XuW7JExOMjYpOMfeFQB6uI+E3gUuDtwLnAyRGxTWL9I4BTImKLxJqHA/8cES/LqtnqPrF9ufXhkBOEIuJI4D0RsU/nBt637nOAvwPeC3woIvYtpfyqS7hqr9V5wM+TmjlZ98jJINSxziYRsTXw18AZEfFK+HW4SnmfRcShEfHyiHj5ZO2EmkcAHwfeAbwvIrbvUjcingacBVxaSrmta/tazWe2mrsCr8+oOaX+wS20HAjdAnFEHAtcApwBvAt4RdcdQHuffhR4LXD+5MFQlvZ+fW1bf7NqHhQRT4qIxdD9ICMiHtHq/E9iWH0m8Dbg2xn1euoeRX1P/RmwYrLtSbUz93/PpV5L8hnA7wF7ZdVu9Q9oB5oHQdr26lnAx6jb2XdlvRdanX8BzouITTsXLKUM5Q+wKXAR8KR2//nUFfXtwDYJ9Y8E/g0Yn+ax6LPms4AJ4HzgD7rUmlL32a2tfwp8BnhuQlsPBm4ADp3msU36rBnUHoWrgXHgIdQd4U3A/v3WBp4M/AB4eru/dfv9oI7t3YJ6jbW7gGO6vk6t5hva/3w+8NqMmq3ukcA3qRvAVcCLE9aB8Vbz6cAjqRvZ7YB5HV6r1wG/127vBhze1rVt+2zjM4DrgP3bNuEfgKcmL9dr2//+SeDcfpcrsCP1gGK/dv9l1Ov6vRF4cIfX6P9Tv70Caq/dMxL//wOBn7ZlvGzyvdWx5nPa9uod7X3w113WVeAo4GfAyp5p8zq28RDg1p7lui01WGyZUPc7wOPa/fdRLzmU8Vr9ZluuB/e7zeuptSP1kkiPbvc/CLwQ2BnYIqGtRwFXtdf/YuAVCTWf3t4LTwYWUw+2XtrvejWl9vbA56gh+2Jgsy71hrrHCtgGWNRufwL4NLAZsKTj0c9+1BX+nFLKqojYsQ05Pgb6O7qKiHHgT4CTqDvWV0fE4aW9ah3aegDwVuCVpZQ3UFdWJodZ+mlrswj4cCnlS61r9TkRcXyr2VfvUvtfb6Im/2uB20op7wTOBP4hIh5RSvlVH239DeCfgNsjYi9geUS8n3r0vqi1d9bLoJTyc+BT1F7RsyNiKXQetryHGi7PAw6KiHdFxJ9E1Vfd1uNxKnBaKeXPqT0iJPQGPAZ4VSnly9SewKOANwPvjYh9+lyu9/TcvoQaLk4BzomI7fto4zzg+FKvhbcV9auv9oeUIbt5wFLgraWUZcDxwL4RcQn0tVzvAbYGJofXPwh8D1hAXbb9uJW6U7qiHVUfTO1l/+uIeEHC8MrWwPOA3wJeDCzt7bma7TobEVtSX+/XlVL+gBoqnxcRH4TZ91q0df8U6vp/d0Rc0Op07bm6HfglsGvU4dVPAn9F7WHqulzPLKVc1W6/GdghOg4JRsRC6kHLbdSeywM6tvEe4EHAI1sv2Dh1/T8beGOXXtaIeBw1VJ9QSjke+FvqgVtXi4G3lVK+WkpZTX0NnwLde8NKKT+iHmQfSe0gWB4RT5nsxe6n4ND+UI92LwOe0u7PA5YAF9AhoQKPpwar36H2Mn2B2jv2eeC9fdZ8NnBwz/1TqEcBfR2p99Q5CHhCu70DNbh8Cvhwv21ttcaBc6gh4OvU8LMauLDPevtQj353bMvyDVMefwOwgtpLNKPXrtXcj7qjOpW64bsReHVbLmdQj7pm3RsAbNp+H0M9Uns8NQyeBfwFfR4RAw8HTm+3X0890j6n4zqwVVt2zwHGgO+2ZfzPwMf6fK0eRev5ba/JCuD3qaHldODL9NEzDDyaGn4uBE5s0x4GvJ/6HaL9LoNN2u9nAbcAj+myTHvqngYcN2Xa/6Onl2WW9V7Z3pvHAX/ctlWvIKHXAvhD4I3t9oltHVjQZ619qAcs2wI7tmkHA1+ibrvu0ys8y3X1YlpPSJv2Z8C3gHf22dbdqAFwJ2pYvyDptX8scH3bprycemrMy6jDrjv0WXNez/tqHvUr3a6afJ0ml3UfdR8KPK3d/iPqfnExMH/KfDPeLwIvAK4Evga8qU07tG0LHtthuR5C7QjoXdeuoO5r+umx3IcazHamfj/f5PSD6dlfAZv3UXcx9458vAn4/Xb7cuBXwNF9LYOMFXRQP9QN/inUbvqn9kz/EjDWR71H9Nx+EvBu4D/axnByGOsLtCA3w5r7Trk/uQM4qK38e/VO77Ot89qb/mRgaZu2B3XnN95nzce29v0h9ehycvq/AK+eZVuPAr4B/CPwl8BzqTv/M3rmWcgsdlY9Nb8CfKC9Xq8ClvXMswc1vM6427Z3GbT7ewMfbbd/D7ibDkGIuhP4EHVDfS11I/gp+ugKn/J6nUo98rsC+NOe6VfQMyw4i+W6iroDmRyi3XfK/3AufQ4JAEcD/0ntCZqc9gFat33XH2oP7hntPdvPcGXvcn0pdTj0oT3TJnfg+/VRe1vgJW0deHfP9E+TcArDlOf6LHBAH3/Xuw58hJ6QCjyBun19EXXb+2Gm7LxnsEzfQg0rL6SeC/OX1HD9AWC7jv/zjtRzbC5o9w8AHtmh3n7AyVOmfY4+9i/T1J5PDYRfbPdfQj04nHFYnbJct+25/aa2XTmw3e/rQIM6BPZnwFE90z5Gz+kmfbZ1MkjOA7ZsbZ0MnIv6WFf/sa2L+/c8diBwebs9eSAzowPinrpfph6gLKKGt9dQc8D11H3hJbSD8Fkti64rz6B/2gt/MvXchWXUrvtvAQ+ZZZ3JcfrehHsQ8Lwp862g9RDNouZHe6bN77l9LvCpPv7n+9Vt0zefcv9c4JAObX1lW4HeO/mmpfYsnTiLtk49p2A59XLmiToAAAYpSURBVDy43ajncL2RemRwArVHbPs+ar6f1jvXuwyoG6pVzHBjzfTnamwPvIc6FPLt1t7bgd/usM6+tf3vR7f7Twf2TFhft6R21z+jZ9qfAi/o87V6H3DeNPO9hLohW+drtZbnmd/aeT11aPyk9to/vN9lOqX+84Gv0kev4lqW69uA73PfcHUhPT3QfTzPJj23j6f2Lm7VoV5Muf98am/DLrOss9ZzgCafg7pjubktk9+YxTK9qGfaa9p76Szu7R2+FNg14fXfiRpcv0M9eNkjY72aslxntX9ZR80V1NNErmQWAYjpt9mb9dx+E7CSOtrwDWDnPtt3ZFuez6QeFH8dWDjLGtO9ryY7GTahHgRsQw1Al81k2/JA62q7v0/7/1/Ylu2MAvZa6i6nbl+vo55z+Mz22MX9rF8pK86gf6jnVT29bexWTC6QWfz9VtSjkGXt73t3rA/quf186smme/VR84KexzZvv3eifjrkyR3a2lu3N7T9Zoe29v7/L6cOp50K/F/gmpmuoD0r6Qk99xcAn2m3H0btUXofdcc6o43KWmpexn03KidRT+Tcf4Y1H2i5ngn8Anh+u/80YJ8O6+uewON77s+2t/KBXq+l1NB2UHv8Kqb0ws1yuX6yZ33dAvg/s1mu63i+A6jnWrxzpq/9LGpfzOw3/lOXa+8O623t/34FtRf3GmDvhHa+jBrYs4YuN2/r/rfoGW6bxd+vdR3g3h3hIcAdM1kHHmiZTpnvpdQwvFPScngtuUPC0fNadV73e2puRh0VuYHZ9dSsc//Sbq+inh7S93KgfmDl1dSDqb9nlsOA62jrPOqB1t8Cf0PdD8yoJ/gB1tUt2v1tgTXUUDXj12wtdT/Vbj+bhA/HdF551udPe5H6/QTY1HH6j0x5fCk1qMx4YzVNzQumPL4l9YhttkeVa61L/WTUyW1l6tLW3p3Kk6nDN29nytDmDF+T6c4p2LVN26u9sWZ8rtkD1JzsXn4Y9aPssxoCmGYZrGzTN6GFExI+xdnzfF3OA1zr+sq9R6qfnuU6sK7lug81BD8qaxlk/3R9fdbxPnge8LvUHcCsQ8tanm8vOoT0aept2jb+s3qfzmId2I16EDvj3sW1va/aY/Op58VdQcLwWqu5PfV82HX2ps1mvaKed9r3sOID1D6BPsLaNMt16v7lEe216/t8qCn1Hkyfw9UzaOsnqaF1xuvtDNbVRdQwONv9wNrq7tSmbUMfw3+9PxvllzC3T4EsB+4qpbw0Ih5F3Zh8rpRyfVLNxdTzDG4r/X0Sbm11HwkcQe0Vuq5jzbtLKS+OiN8Abi+l/KDfdra686m9HpeWUg6LiJdSP7VxainlrsSajwH+uJTy4w5tnbpcx4BflFKu6bfmoPS09ZellBdFxMOob/5vl1Lu7rPm1OV6HPXE9Xd0Wa6jZJr3wf7Af5dSvjfHTVsvplkHXkI9IfiMUspP+6w59X31aOqB0BWllFsS275FqZ/qHXoREaXjjnYt26vJbcAPM9qZZZq2LqJ+2OKCUkpf1wybZl09nhqs3llKuaNDWyfrXlZKObS9B55MvWRMX+8BYOMMVgARsRP1pL1DqEcrTyul3JxU84nUI7XxUsqNA2jrU7tupKbUnJfV1lZ7BfUcjWdSu1yvTq55YinlGwk1e1+vedTrZKUsg2w9bX0SdR1IaesglusoGeT7YFRkrwNTlukm1G3rTV3bubGbZns1tMt1yvYK6gfCbk2ou4IBbK+y91nzMxo1ikopP4yIb1BP3Du8a6haS82UDfQ0dTsf+Q2ire26KptSe6k2BQ4rpVw7bDUnDer1GoTstg5yuY6SUVoHsg1qHZhmmQ7lzn/UjNJynaatnULVoNbVQdXdaINVu1jhs6ln/3fuURlUzUHVHUTN1t19d0S8DfjXpI10es1Jg3q9BiG7rYNcrqNklNaBbINaBzbmZTpIo7RcR2V7NbD3wMY6FAiDGacf1Nj/iLW18zkF66NmqztK52oMYh0YyHIdJaO0DgzCgN6vG/UyHZRRWq6jtL3KrrtRBytJkqRMw/5dgZIkSSPDYCVJkpTEYCVJkpTEYCVJkpTEYCVJkpTEYCVJkpTEYCVJkpTkfwEz11J/1aE9tgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x504 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import scipy.cluster.hierarchy as shc\n",
    "plt.figure(figsize=(10, 7))  \n",
    "plt.title(\"Dendrograms\")  \n",
    "dend = shc.dendrogram(shc.linkage(data2, method='ward'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compute clusters\n",
    "# number of clusters = 3, as determined using elbow method\n",
    "cluster = AgglomerativeClustering(n_clusters=3, affinity='euclidean', linkage='ward')  \n",
    "clusterArray = cluster.fit_predict(data2)\n",
    "\n",
    "# add cluster assignments to data\n",
    "data['clusterGroups'] = clusterArray\n",
    "\n",
    "data = data.reset_index()\n",
    "del data['index']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Agree\n",
      "128.33333333333334\n",
      "148.92307692307693\n",
      "146.0\n",
      "\n",
      "Disgree\n",
      "43.333333333333336\n",
      "53.92307692307692\n",
      "56.0\n",
      "\n",
      "Uncertain\n",
      "40.5\n",
      "52.46153846153846\n",
      "43.0\n",
      "\n",
      "No Opinion\n",
      "19.0\n",
      "14.076923076923077\n",
      "10.857142857142858\n",
      "\n",
      "Did not answer\n",
      "18.0\n",
      "18.0\n",
      "18.0\n"
     ]
    }
   ],
   "source": [
    "# validation testing:\n",
    "# number of agrees, disagrees, etc should be even across clusters\n",
    "\n",
    "uncertainList = []\n",
    "agreeList = []\n",
    "disagreeList = []\n",
    "noOpinionList = []\n",
    "didNotAnswerList = []\n",
    "\n",
    "for row in range(0,len(data)):\n",
    "    uncertains = 0\n",
    "    agrees = 0\n",
    "    disagrees = 0\n",
    "    noOpinions = 0\n",
    "    didNotAnswer = 0\n",
    "    \n",
    "    for column in data.columns[1:-2]:\n",
    "        if data[column][row] == 0:\n",
    "            didNotAnswer = didNotAnswer + 1\n",
    "        elif data[column][row] == 4:\n",
    "            noOpinions = noOpinions + 1\n",
    "        elif data[column][row] == 1:\n",
    "            agrees = agrees + 1\n",
    "        elif data[column][row] == 2:\n",
    "            uncertains = uncertains + 1\n",
    "        elif data[column][row] == 3:\n",
    "            disagrees = disagrees + 1\n",
    "            \n",
    "    noOpinionList.append(noOpinions)\n",
    "    uncertainList.append(uncertains)\n",
    "    agreeList.append(agrees)\n",
    "    disagreeList.append(disagrees)\n",
    "    \n",
    "data['agreeCount'] = agreeList\n",
    "data['uncertainCount'] = uncertainList\n",
    "data['disagreeCount'] = disagreeList\n",
    "data['noOpinionCount'] = noOpinionList\n",
    "data['didNotAnswerCount'] = didNotAnswer\n",
    "\n",
    "print(\"Agree\")\n",
    "print(data[data['clusterGroups']==0]['agreeCount'].mean())\n",
    "print(data[data['clusterGroups']==1]['agreeCount'].mean())\n",
    "print(data[data['clusterGroups']==2]['agreeCount'].mean())\n",
    "print(\"\")\n",
    "print(\"Disgree\")\n",
    "print(data[data['clusterGroups']==0]['disagreeCount'].mean())\n",
    "print(data[data['clusterGroups']==1]['disagreeCount'].mean())\n",
    "print(data[data['clusterGroups']==2]['disagreeCount'].mean())\n",
    "print(\"\")\n",
    "print(\"Uncertain\")\n",
    "print(data[data['clusterGroups']==0]['uncertainCount'].mean())\n",
    "print(data[data['clusterGroups']==1]['uncertainCount'].mean())\n",
    "print(data[data['clusterGroups']==2]['uncertainCount'].mean())\n",
    "print(\"\")\n",
    "print(\"No Opinion\")\n",
    "print(data[data['clusterGroups']==0]['noOpinionCount'].mean())\n",
    "print(data[data['clusterGroups']==1]['noOpinionCount'].mean())\n",
    "print(data[data['clusterGroups']==2]['noOpinionCount'].mean())\n",
    "print(\"\")\n",
    "print(\"Did not answer\")\n",
    "print(data[data['clusterGroups']==0]['didNotAnswerCount'].mean())\n",
    "print(data[data['clusterGroups']==1]['didNotAnswerCount'].mean())\n",
    "print(data[data['clusterGroups']==2]['didNotAnswerCount'].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "del data['didNotAnswerCount']\n",
    "del data['agreeCount']\n",
    "del data['uncertainCount']\n",
    "del data['disagreeCount']\n",
    "del data['noOpinionCount']\n",
    "\n",
    "import os\n",
    "data.to_csv(os.getcwd() + \"\\\\Documents\\\\hierarchyClusteringEconomists.csv\",index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}