Preprocess_complete.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>Preprocessing</b>\n",
    "\n",
    "import pandas as pd\n",
    "import glob\n",
    "\n",
    "# Allows for printing all columns and increases width; debug utility\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.width', 1000)\n",
    "\n",
    "# Select only the data files in the data folder\n",
    "#data_files = glob.glob('data/*data.csv')\n",
    "data_files = sorted(glob.glob('data/*data.csv'))\n",
    "\n",
    "# Create the dfs and merge them on the keys TIME and METROREG\n",
    "merged_data = None\n",
    "for file in data_files:\n",
    "    current_df = pd.read_csv(file, encoding=\"ISO-8859-1\")\n",
    "    if merged_data is not None:\n",
    "        merged_data = pd.merge(merged_data, current_df, on=[\"TIME\", \"METROREG\"])\n",
    "    else:\n",
    "        merged_data = current_df\n",
    "\n",
    "# Rename relevant columns and drop the useless ones\n",
    "# Though, be aware that measurement information is also lost\n",
    "renames = {'Value_x': 'employed_persons',\n",
    "           'Value_y': 'gdp',\n",
    "           'Value': 'population'}\n",
    "\n",
    "droppes = ['UNIT_x', 'WSTATUS', 'NACE_R2', 'Flag and Footnotes_x',\n",
    "           'UNIT_y', 'Flag and Footnotes_y', 'AGE', 'SEX', 'Flag and Footnotes']\n",
    "\n",
    "filtered_data = merged_data.rename(columns=renames).drop(columns=droppes)\n",
    "\n",
    "#print(filtered_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Rows we want to remove\n",
    "non_metro=['Non-metropolitan regions in Austria',\n",
    " 'Non-metropolitan regions in Belgium',\n",
    " 'Non-metropolitan regions in Bulgaria',\n",
    " 'Non-metropolitan regions in Croatia',\n",
    " 'Non-metropolitan regions in Czech Republic',\n",
    " 'Non-metropolitan regions in Denmark',\n",
    " 'Non-metropolitan regions in Estonia',\n",
    " 'Non-metropolitan regions in Finland',\n",
    " 'Non-metropolitan regions in Germany',\n",
    " 'Non-metropolitan regions in Greece',\n",
    " 'Non-metropolitan regions in Hungary',\n",
    " 'Non-metropolitan regions in Ireland',\n",
    " 'Non-metropolitan regions in Italy',\n",
    " 'Non-metropolitan regions in Latvia',\n",
    " 'Non-metropolitan regions in Lithuania',\n",
    " 'Non-metropolitan regions in Malta',\n",
    " 'Non-metropolitan regions in Netherlands',\n",
    " 'Non-metropolitan regions in North Macedonia',\n",
    " 'Non-metropolitan regions in Portugal',\n",
    " 'Non-metropolitan regions in Romania',\n",
    " 'Non-metropolitan regions in Serbia',\n",
    " 'Non-metropolitan regions in Slovakia',\n",
    " 'Non-metropolitan regions in Slovenia',\n",
    " 'Non-metropolitan regions in Spain',\n",
    " 'Non-metropolitan regions in Sweden',\n",
    " 'Non-metropolitan regions in United Kingdom']\n",
    "countries = ['United Kingdom',\"West Midlands urban area\",\"North Macedonia\",\"Austria\",\"Belgium\",\"Bulgaria\",\"Croatia\",\"Cyprus\",\"Czechia\",\"Denmark\",\"Estonia\",\"Finland\",\"France\",\"Germany\",\"Germany (until 1990 former territory of the FRG)\",\"Greece\",\"Hungary\",\"Ireland\",\"Italy\",\"Latvia\",\"Lithuania\",\"Luxembourg\",\"Malta\",\"Netherlands\",\"Poland\",\"Portugal\",\"Romania\",\"Serbia\",\"Slovakia\",\"Slovenia\",\"Spain\",\"Sweden\"]\n",
    "countries.extend(non_metro)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_data2=filtered_data\n",
    "for i in countries:\n",
    "    filtered_data2=filtered_data2[filtered_data2.METROREG!=i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import scipy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Preprocessing on the population column\n",
    "filtered_data2.population = filtered_data2.population.replace(':', np.nan) # replacing NAs (noted ':') by NaN\n",
    "filtered_data2.population = filtered_data2['population'].replace(r',', '',regex=True)  # removing commas\n",
    "filtered_data2.population = filtered_data2['population'].astype(float)  #changing column type\n",
    "\n",
    "# Preprocessing on the GDP column\n",
    "filtered_data2.gdp = filtered_data2.gdp.replace(':',np.nan )  # replacing missing values (noted ':') by NaN\n",
    "filtered_data2.gdp = filtered_data2['gdp'].replace(r',', '',regex=True)  # removing commas\n",
    "filtered_data2.gdp = filtered_data2['gdp'].astype(float)  # changing column type\n",
    "\n",
    "# Employed_persons column\n",
    "filtered_data2.employed_persons = filtered_data2.employed_persons.replace(':', np.nan)\n",
    "filtered_data2.employed_persons = filtered_data2['employed_persons'].replace(r',', '',regex=True)\n",
    "filtered_data2.employed_persons = filtered_data2['employed_persons'].astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TIME</th>\n",
       "      <th>METROREG</th>\n",
       "      <th>employed_persons</th>\n",
       "      <th>gdp</th>\n",
       "      <th>population</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2603</th>\n",
       "      <td>2000</td>\n",
       "      <td>Roma</td>\n",
       "      <td>1718.8</td>\n",
       "      <td>113416.07</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2604</th>\n",
       "      <td>2001</td>\n",
       "      <td>Roma</td>\n",
       "      <td>1736.7</td>\n",
       "      <td>120648.20</td>\n",
       "      <td>3704765.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2605</th>\n",
       "      <td>2002</td>\n",
       "      <td>Roma</td>\n",
       "      <td>1781.4</td>\n",
       "      <td>126515.83</td>\n",
       "      <td>3704522.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2606</th>\n",
       "      <td>2003</td>\n",
       "      <td>Roma</td>\n",
       "      <td>1822.7</td>\n",
       "      <td>129519.63</td>\n",
       "      <td>3715202.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2607</th>\n",
       "      <td>2004</td>\n",
       "      <td>Roma</td>\n",
       "      <td>1899.9</td>\n",
       "      <td>137763.33</td>\n",
       "      <td>3739767.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2608</th>\n",
       "      <td>2005</td>\n",
       "      <td>Roma</td>\n",
       "      <td>1919.8</td>\n",
       "      <td>142083.95</td>\n",
       "      <td>3777674.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2609</th>\n",
       "      <td>2006</td>\n",
       "      <td>Roma</td>\n",
       "      <td>1950.5</td>\n",
       "      <td>147783.45</td>\n",
       "      <td>3798630.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2610</th>\n",
       "      <td>2007</td>\n",
       "      <td>Roma</td>\n",
       "      <td>2022.0</td>\n",
       "      <td>154221.73</td>\n",
       "      <td>3823955.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2611</th>\n",
       "      <td>2008</td>\n",
       "      <td>Roma</td>\n",
       "      <td>2025.4</td>\n",
       "      <td>156050.12</td>\n",
       "      <td>3870783.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2612</th>\n",
       "      <td>2009</td>\n",
       "      <td>Roma</td>\n",
       "      <td>2015.0</td>\n",
       "      <td>154172.03</td>\n",
       "      <td>3912714.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2613</th>\n",
       "      <td>2010</td>\n",
       "      <td>Roma</td>\n",
       "      <td>2041.6</td>\n",
       "      <td>155411.68</td>\n",
       "      <td>3945294.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2614</th>\n",
       "      <td>2011</td>\n",
       "      <td>Roma</td>\n",
       "      <td>2041.3</td>\n",
       "      <td>159065.98</td>\n",
       "      <td>3979136.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2615</th>\n",
       "      <td>2012</td>\n",
       "      <td>Roma</td>\n",
       "      <td>2027.3</td>\n",
       "      <td>154486.97</td>\n",
       "      <td>3995250.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2616</th>\n",
       "      <td>2013</td>\n",
       "      <td>Roma</td>\n",
       "      <td>2024.9</td>\n",
       "      <td>152739.19</td>\n",
       "      <td>4039813.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2617</th>\n",
       "      <td>2014</td>\n",
       "      <td>Roma</td>\n",
       "      <td>2046.6</td>\n",
       "      <td>153286.69</td>\n",
       "      <td>4321244.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2618</th>\n",
       "      <td>2015</td>\n",
       "      <td>Roma</td>\n",
       "      <td>2071.7</td>\n",
       "      <td>153607.21</td>\n",
       "      <td>4342046.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2619</th>\n",
       "      <td>2016</td>\n",
       "      <td>Roma</td>\n",
       "      <td>2127.3</td>\n",
       "      <td>160992.96</td>\n",
       "      <td>4340474.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2620</th>\n",
       "      <td>2017</td>\n",
       "      <td>Roma</td>\n",
       "      <td>2151.9</td>\n",
       "      <td>163143.15</td>\n",
       "      <td>4353738.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2621</th>\n",
       "      <td>2018</td>\n",
       "      <td>Roma</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4355725.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      TIME METROREG  employed_persons        gdp  population\n",
       "2603  2000     Roma            1718.8  113416.07         NaN\n",
       "2604  2001     Roma            1736.7  120648.20   3704765.0\n",
       "2605  2002     Roma            1781.4  126515.83   3704522.0\n",
       "2606  2003     Roma            1822.7  129519.63   3715202.0\n",
       "2607  2004     Roma            1899.9  137763.33   3739767.0\n",
       "2608  2005     Roma            1919.8  142083.95   3777674.0\n",
       "2609  2006     Roma            1950.5  147783.45   3798630.0\n",
       "2610  2007     Roma            2022.0  154221.73   3823955.0\n",
       "2611  2008     Roma            2025.4  156050.12   3870783.0\n",
       "2612  2009     Roma            2015.0  154172.03   3912714.0\n",
       "2613  2010     Roma            2041.6  155411.68   3945294.0\n",
       "2614  2011     Roma            2041.3  159065.98   3979136.0\n",
       "2615  2012     Roma            2027.3  154486.97   3995250.0\n",
       "2616  2013     Roma            2024.9  152739.19   4039813.0\n",
       "2617  2014     Roma            2046.6  153286.69   4321244.0\n",
       "2618  2015     Roma            2071.7  153607.21   4342046.0\n",
       "2619  2016     Roma            2127.3  160992.96   4340474.0\n",
       "2620  2017     Roma            2151.9  163143.15   4353738.0\n",
       "2621  2018     Roma               NaN        NaN   4355725.0"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_data2[filtered_data2['METROREG'] == 'Roma']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Removed cities:  ['Leipzig', 'Dresden', 'Göttingen', 'Schwerin', 'Rostock', 'Neubrandenburg', 'Görlitz', 'Aachen', 'Zwickau', 'Dublin', 'Cork', 'Amsterdam', 'Rotterdam', 'Leeuwarden', 'Lódz', 'Kraków', 'Wroclaw', 'Poznan', 'Gdansk', 'Szczecin', 'Bydgoszcz - Torún', 'Lublin', 'Katowice', 'Bialystok', 'Kielce', 'Olsztyn', 'Rzeszów', 'Opole', 'Czestochowa', 'Bielsko-Biala', 'Tarnów', 'Glasgow', 'Edinburgh', 'Belfast', 'Norway', 'Oslo', 'Bergen']\n"
     ]
    }
   ],
   "source": [
    "#Define which cities should be removed from the dataset \n",
    "cities_removed = []\n",
    "for v in filtered_data2['METROREG'].unique():\n",
    "    if max(list(filtered_data2[filtered_data2['METROREG'] == v].isnull().sum()))>8:\n",
    "        cities_removed.append(v)\n",
    "print('Removed cities: ', cities_removed)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Bruxelles / Brussel' 'Antwerpen' 'Gent' 'Charleroi' 'Liège' 'Sofia'\n",
      " 'Plovdiv' 'Varna' 'Burgas' 'Praha' 'Brno' 'Ostrava' 'Plzen' 'København'\n",
      " 'Århus' 'Odense' 'Aalborg' 'Berlin' 'Hamburg' 'München' 'Köln'\n",
      " 'Frankfurt am Main' 'Stuttgart' 'Düsseldorf' 'Bremen' 'Hannover'\n",
      " 'Nürnberg' 'Bielefeld' 'Halle an der Saale' 'Magdeburg' 'Wiesbaden'\n",
      " 'Darmstadt' 'Freiburg im Breisgau' 'Regensburg' 'Erfurt' 'Augsburg'\n",
      " 'Bonn' 'Karlsruhe' 'Mönchengladbach' 'Mainz' 'Ruhrgebiet' 'Kiel'\n",
      " 'Saarbrücken' 'Koblenz' 'Kaiserslautern' 'Iserlohn' 'Flensburg'\n",
      " 'Konstanz' 'Gießen' 'Bayreuth' 'Aschaffenburg' 'Rosenheim' 'Offenburg'\n",
      " 'Schweinfurt' 'Wetzlar' 'Braunschweig-Salzgitter-Wolfsburg'\n",
      " 'Mannheim-Ludwigshafen' 'Münster' 'Lübeck' 'Kassel' 'Osnabrück'\n",
      " 'Oldenburg (Oldenburg)' 'Heidelberg' 'Paderborn' 'Würzburg' 'Bremerhaven'\n",
      " 'Heilbronn' 'Ulm' 'Pforzheim' 'Ingolstadt' 'Reutlingen' 'Siegen'\n",
      " 'Hildesheim' 'Wuppertal' 'Düren' 'Bocholt' 'Tallinn' 'Athina'\n",
      " 'Thessaloniki' 'Madrid' 'Barcelona' 'Valencia' 'Sevilla' 'Zaragoza'\n",
      " 'Málaga - Marbella' 'Murcia - Cartagena' 'Las Palmas' 'Valladolid'\n",
      " 'Palma de Mallorca' 'Vitoria/Gasteiz' 'Oviedo - Gijón' 'Pamplona/Iruña'\n",
      " 'Santander' 'Bilbao' 'Córdoba' 'Alicante/Alacant - Elche/Elx' 'Vigo'\n",
      " 'Santa Cruz de Tenerife' 'A Coruña' 'Granada' 'Guipúzcoa' 'Cádiz'\n",
      " 'Zagreb' 'Split' 'Roma' 'Milano' 'Napoli' 'Torino' 'Palermo' 'Genova'\n",
      " 'Firenze' 'Bari' 'Bologna' 'Catania' 'Venezia' 'Verona' 'Taranto'\n",
      " 'Cagliari' 'Padova' 'Brescia' 'Messina' 'Prato' 'Parma'\n",
      " 'Reggio nell Emilia' 'Bergamo' 'Lefkosia' 'Riga' 'Vilnius' 'Kaunas'\n",
      " 'Budapest' 'Miskolc' 'Pécs' 'Debrecen' 'Székesfehérvár' 'Valletta'\n",
      " \"s' Gravenhage\" 'Utrecht' 'Eindhoven' 'Tilburg' 'Groningen' 'Enschede'\n",
      " 'Arnhem - Nijmegen' 'Breda' 'Leiden' 'Zwolle' 'Wien' 'Graz' 'Linz'\n",
      " 'Salzburg' 'Innsbruck' 'Lisboa' 'Porto' 'Coimbra' 'Bucuresti'\n",
      " 'Cluj-Napoca' 'Timisoara' 'Craiova' 'Constanta' 'Iasi' 'Galati' 'Brasov'\n",
      " 'Ploiesti' 'Ljubljana' 'Maribor' 'Bratislava' 'Kosice' 'Helsinki'\n",
      " 'Tampere' 'Turku' 'Stockholm' 'Göteborg' 'Malmö' 'Uppsala' 'London'\n",
      " 'Leeds' 'Bradford' 'Liverpool' 'Manchester' 'Cardiff' 'Sheffield'\n",
      " 'Bristol' 'Newcastle upon Tyne' 'Leicester' 'Aberdeen' 'Cambridge'\n",
      " 'Exeter' 'Portsmouth' 'Coventry' 'Kingston upon Hull' 'Stoke-on-Trent'\n",
      " 'Nottingham' 'Kirklees' 'Doncaster' 'Sunderland' 'Medway'\n",
      " 'Brighton and Hove' 'Plymouth' 'Swansea' 'Derby' 'Southampton'\n",
      " 'Northampton' 'Bournemouth' 'Colchester'\n",
      " 'Blackburn - Blackpool - Preston' 'Middlesbrough' 'Oxford' 'Norwich'\n",
      " 'Cheshire West and Chester' 'Ipswich']\n"
     ]
    }
   ],
   "source": [
    "#Remove cities from the dataset\n",
    "for i in cities_removed: \n",
    "    indexMetroreg = filtered_data2[filtered_data2['METROREG'] == i].index\n",
    "    filtered_data2.drop(indexMetroreg, inplace=True)\n",
    "\n",
    "print(filtered_data2['METROREG'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.optimize import curve_fit\n",
    "\n",
    "def F(x,a,b):\n",
    "    return a*x+b\n",
    "\n",
    "def nan_position(k):\n",
    "    z=np.argwhere(np.isnan(np.array(k)))\n",
    "    l=[]\n",
    "    for i in z:\n",
    "        l.append(i[0])\n",
    "    return l\n",
    "\n",
    "def treat_missing(k):\n",
    "    #\n",
    "    #if not enough values-> drop the city, threshold of 8 missing values per column\n",
    "    #if all(x <=8 for x in list(k.isnull().sum())) == True:\n",
    "        #cities_removed.append(v)\n",
    "    #employed_persons\n",
    "    missing=k.isnull().sum()    \n",
    "    \n",
    "    if missing[2] != 0: #linear regression on employed persons\n",
    "        L1=list(k.employed_persons)\n",
    "        m1=nan_position(k.employed_persons) #gets position of missing\n",
    "        X1=[i for i in range(len(list(k.employed_persons))) if str(list(k.employed_persons)[i])!='nan']\n",
    "        Y1=[i for i in list(k.employed_persons) if str(i)!='nan']\n",
    "        params1 = curve_fit(F, xdata=X1, ydata=Y1) #performs linear regression, params[0] contains a and b\n",
    "        for i in m1:\n",
    "            L1[i] = params1[0][0]*i + params1[0][1]\n",
    "        k.employed_persons = L1\n",
    "    \n",
    "    if missing[3] != 0: #linear regression on gdp\n",
    "        L2=list(k.gdp)\n",
    "        m2=nan_position(k.gdp) #gets position of missing\n",
    "        X2=[i for i in range(len(list(k.gdp))) if str(list(k.gdp)[i])!='nan']\n",
    "        Y2=[i for i in list(k.gdp) if str(i)!='nan']\n",
    "        params2 = curve_fit(F, xdata=X2, ydata=Y2) #performs linear regression, params[0] contains a and b\n",
    "        for i in m2:\n",
    "            L2[i] = params2[0][0]*i + params2[0][1]\n",
    "        k.gdp = L2\n",
    "        \n",
    "    if missing[4] != 0: #linear regression on population\n",
    "        L3=list(k.population)\n",
    "        m3=nan_position(k.population) #gets position of missing\n",
    "        X3=[i for i in range(len(list(k.population))) if str(list(k.population)[i])!='nan']\n",
    "        Y3=[i for i in list(k.population) if str(i)!='nan']\n",
    "        params3 = curve_fit(F, xdata=X3, ydata=Y3) #performs linear regression, params[0] contains a and b\n",
    "        for i in m3:\n",
    "            L3[i] = params3[0][0]*i + params3[0][1]\n",
    "        k.population = L3   \n",
    "        \n",
    "    return k\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_data3 = pd.DataFrame(columns=['TIME', 'METROREG', 'employed_persons', 'gdp', 'population'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TIME</th>\n",
       "      <th>METROREG</th>\n",
       "      <th>employed_persons</th>\n",
       "      <th>gdp</th>\n",
       "      <th>population</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [TIME, METROREG, employed_persons, gdp, population]\n",
       "Index: []"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_data3\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Zoey\\anaconda3\\lib\\site-packages\\pandas\\core\\generic.py:5303: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self[name] = value\n"
     ]
    }
   ],
   "source": [
    "for v in filtered_data2['METROREG'].unique():\n",
    "    a = treat_missing(filtered_data2[filtered_data2['METROREG'] == v])\n",
    "    filtered_data3 = filtered_data3.append(a)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TIME</th>\n",
       "      <th>METROREG</th>\n",
       "      <th>employed_persons</th>\n",
       "      <th>gdp</th>\n",
       "      <th>population</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2000</td>\n",
       "      <td>Bruxelles / Brussel</td>\n",
       "      <td>1075.0</td>\n",
       "      <td>77281.00</td>\n",
       "      <td>2129716.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>2001</td>\n",
       "      <td>Bruxelles / Brussel</td>\n",
       "      <td>1100.0</td>\n",
       "      <td>81082.00</td>\n",
       "      <td>2138898.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>2002</td>\n",
       "      <td>Bruxelles / Brussel</td>\n",
       "      <td>1089.0</td>\n",
       "      <td>83710.00</td>\n",
       "      <td>2159154.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>2003</td>\n",
       "      <td>Bruxelles / Brussel</td>\n",
       "      <td>1088.4</td>\n",
       "      <td>85529.56</td>\n",
       "      <td>2178354.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>2004</td>\n",
       "      <td>Bruxelles / Brussel</td>\n",
       "      <td>1093.9</td>\n",
       "      <td>90776.48</td>\n",
       "      <td>2192352.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5733</th>\n",
       "      <td>2014</td>\n",
       "      <td>Ipswich</td>\n",
       "      <td>338.0</td>\n",
       "      <td>23270.54</td>\n",
       "      <td>739140.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5734</th>\n",
       "      <td>2015</td>\n",
       "      <td>Ipswich</td>\n",
       "      <td>340.0</td>\n",
       "      <td>26123.13</td>\n",
       "      <td>745096.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5735</th>\n",
       "      <td>2016</td>\n",
       "      <td>Ipswich</td>\n",
       "      <td>340.0</td>\n",
       "      <td>24324.07</td>\n",
       "      <td>749409.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5736</th>\n",
       "      <td>2017</td>\n",
       "      <td>Ipswich</td>\n",
       "      <td>345.0</td>\n",
       "      <td>23425.15</td>\n",
       "      <td>754039.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5737</th>\n",
       "      <td>2018</td>\n",
       "      <td>Ipswich</td>\n",
       "      <td>352.0</td>\n",
       "      <td>23818.20</td>\n",
       "      <td>758050.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3971 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      TIME             METROREG  employed_persons       gdp  population\n",
       "19    2000  Bruxelles / Brussel            1075.0  77281.00   2129716.0\n",
       "20    2001  Bruxelles / Brussel            1100.0  81082.00   2138898.0\n",
       "21    2002  Bruxelles / Brussel            1089.0  83710.00   2159154.0\n",
       "22    2003  Bruxelles / Brussel            1088.4  85529.56   2178354.0\n",
       "23    2004  Bruxelles / Brussel            1093.9  90776.48   2192352.0\n",
       "...    ...                  ...               ...       ...         ...\n",
       "5733  2014              Ipswich             338.0  23270.54    739140.0\n",
       "5734  2015              Ipswich             340.0  26123.13    745096.0\n",
       "5735  2016              Ipswich             340.0  24324.07    749409.0\n",
       "5736  2017              Ipswich             345.0  23425.15    754039.0\n",
       "5737  2018              Ipswich             352.0  23818.20    758050.0\n",
       "\n",
       "[3971 rows x 5 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_data3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TIME</th>\n",
       "      <th>METROREG</th>\n",
       "      <th>employed_persons</th>\n",
       "      <th>gdp</th>\n",
       "      <th>population</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2603</th>\n",
       "      <td>2000</td>\n",
       "      <td>Roma</td>\n",
       "      <td>1718.800000</td>\n",
       "      <td>113416.070000</td>\n",
       "      <td>3.559459e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2604</th>\n",
       "      <td>2001</td>\n",
       "      <td>Roma</td>\n",
       "      <td>1736.700000</td>\n",
       "      <td>120648.200000</td>\n",
       "      <td>3.704765e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2605</th>\n",
       "      <td>2002</td>\n",
       "      <td>Roma</td>\n",
       "      <td>1781.400000</td>\n",
       "      <td>126515.830000</td>\n",
       "      <td>3.704522e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2606</th>\n",
       "      <td>2003</td>\n",
       "      <td>Roma</td>\n",
       "      <td>1822.700000</td>\n",
       "      <td>129519.630000</td>\n",
       "      <td>3.715202e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2607</th>\n",
       "      <td>2004</td>\n",
       "      <td>Roma</td>\n",
       "      <td>1899.900000</td>\n",
       "      <td>137763.330000</td>\n",
       "      <td>3.739767e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2608</th>\n",
       "      <td>2005</td>\n",
       "      <td>Roma</td>\n",
       "      <td>1919.800000</td>\n",
       "      <td>142083.950000</td>\n",
       "      <td>3.777674e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2609</th>\n",
       "      <td>2006</td>\n",
       "      <td>Roma</td>\n",
       "      <td>1950.500000</td>\n",
       "      <td>147783.450000</td>\n",
       "      <td>3.798630e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2610</th>\n",
       "      <td>2007</td>\n",
       "      <td>Roma</td>\n",
       "      <td>2022.000000</td>\n",
       "      <td>154221.730000</td>\n",
       "      <td>3.823955e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2611</th>\n",
       "      <td>2008</td>\n",
       "      <td>Roma</td>\n",
       "      <td>2025.400000</td>\n",
       "      <td>156050.120000</td>\n",
       "      <td>3.870783e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2612</th>\n",
       "      <td>2009</td>\n",
       "      <td>Roma</td>\n",
       "      <td>2015.000000</td>\n",
       "      <td>154172.030000</td>\n",
       "      <td>3.912714e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2613</th>\n",
       "      <td>2010</td>\n",
       "      <td>Roma</td>\n",
       "      <td>2041.600000</td>\n",
       "      <td>155411.680000</td>\n",
       "      <td>3.945294e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2614</th>\n",
       "      <td>2011</td>\n",
       "      <td>Roma</td>\n",
       "      <td>2041.300000</td>\n",
       "      <td>159065.980000</td>\n",
       "      <td>3.979136e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2615</th>\n",
       "      <td>2012</td>\n",
       "      <td>Roma</td>\n",
       "      <td>2027.300000</td>\n",
       "      <td>154486.970000</td>\n",
       "      <td>3.995250e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2616</th>\n",
       "      <td>2013</td>\n",
       "      <td>Roma</td>\n",
       "      <td>2024.900000</td>\n",
       "      <td>152739.190000</td>\n",
       "      <td>4.039813e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2617</th>\n",
       "      <td>2014</td>\n",
       "      <td>Roma</td>\n",
       "      <td>2046.600000</td>\n",
       "      <td>153286.690000</td>\n",
       "      <td>4.321244e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2618</th>\n",
       "      <td>2015</td>\n",
       "      <td>Roma</td>\n",
       "      <td>2071.700000</td>\n",
       "      <td>153607.210000</td>\n",
       "      <td>4.342046e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2619</th>\n",
       "      <td>2016</td>\n",
       "      <td>Roma</td>\n",
       "      <td>2127.300000</td>\n",
       "      <td>160992.960000</td>\n",
       "      <td>4.340474e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2620</th>\n",
       "      <td>2017</td>\n",
       "      <td>Roma</td>\n",
       "      <td>2151.900000</td>\n",
       "      <td>163143.150000</td>\n",
       "      <td>4.353738e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2621</th>\n",
       "      <td>2018</td>\n",
       "      <td>Roma</td>\n",
       "      <td>2182.146405</td>\n",
       "      <td>169362.862078</td>\n",
       "      <td>4.355725e+06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      TIME METROREG  employed_persons            gdp    population\n",
       "2603  2000     Roma       1718.800000  113416.070000  3.559459e+06\n",
       "2604  2001     Roma       1736.700000  120648.200000  3.704765e+06\n",
       "2605  2002     Roma       1781.400000  126515.830000  3.704522e+06\n",
       "2606  2003     Roma       1822.700000  129519.630000  3.715202e+06\n",
       "2607  2004     Roma       1899.900000  137763.330000  3.739767e+06\n",
       "2608  2005     Roma       1919.800000  142083.950000  3.777674e+06\n",
       "2609  2006     Roma       1950.500000  147783.450000  3.798630e+06\n",
       "2610  2007     Roma       2022.000000  154221.730000  3.823955e+06\n",
       "2611  2008     Roma       2025.400000  156050.120000  3.870783e+06\n",
       "2612  2009     Roma       2015.000000  154172.030000  3.912714e+06\n",
       "2613  2010     Roma       2041.600000  155411.680000  3.945294e+06\n",
       "2614  2011     Roma       2041.300000  159065.980000  3.979136e+06\n",
       "2615  2012     Roma       2027.300000  154486.970000  3.995250e+06\n",
       "2616  2013     Roma       2024.900000  152739.190000  4.039813e+06\n",
       "2617  2014     Roma       2046.600000  153286.690000  4.321244e+06\n",
       "2618  2015     Roma       2071.700000  153607.210000  4.342046e+06\n",
       "2619  2016     Roma       2127.300000  160992.960000  4.340474e+06\n",
       "2620  2017     Roma       2151.900000  163143.150000  4.353738e+06\n",
       "2621  2018     Roma       2182.146405  169362.862078  4.355725e+06"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_data3[filtered_data3['METROREG'] == 'Roma']\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "Bruxelles / Brussel\n",
      "0\n",
      "Antwerpen\n",
      "0\n",
      "Gent\n",
      "0\n",
      "Charleroi\n",
      "0\n",
      "Liège\n",
      "1\n",
      "Sofia\n",
      "1\n",
      "Plovdiv\n",
      "1\n",
      "Varna\n",
      "1\n",
      "Burgas\n",
      "1\n",
      "Praha\n",
      "1\n",
      "Brno\n",
      "1\n",
      "Ostrava\n",
      "1\n",
      "Plzen\n",
      "7\n",
      "København\n",
      "7\n",
      "Århus\n",
      "7\n",
      "Odense\n",
      "7\n",
      "Aalborg\n",
      "1\n",
      "Berlin\n",
      "1\n",
      "Hamburg\n",
      "1\n",
      "München\n",
      "1\n",
      "Köln\n",
      "1\n",
      "Frankfurt am Main\n",
      "1\n",
      "Stuttgart\n",
      "1\n",
      "Düsseldorf\n",
      "1\n",
      "Bremen\n",
      "1\n",
      "Hannover\n",
      "1\n",
      "Nürnberg\n",
      "1\n",
      "Bielefeld\n",
      "7\n",
      "Halle an der Saale\n",
      "7\n",
      "Magdeburg\n",
      "1\n",
      "Wiesbaden\n",
      "1\n",
      "Darmstadt\n",
      "1\n",
      "Freiburg im Breisgau\n",
      "1\n",
      "Regensburg\n",
      "1\n",
      "Erfurt\n",
      "1\n",
      "Augsburg\n",
      "1\n",
      "Bonn\n",
      "1\n",
      "Karlsruhe\n",
      "1\n",
      "Mönchengladbach\n",
      "1\n",
      "Mainz\n",
      "1\n",
      "Ruhrgebiet\n",
      "1\n",
      "Kiel\n",
      "1\n",
      "Saarbrücken\n",
      "1\n",
      "Koblenz\n",
      "1\n",
      "Kaiserslautern\n",
      "1\n",
      "Iserlohn\n",
      "1\n",
      "Flensburg\n",
      "1\n",
      "Konstanz\n",
      "1\n",
      "Gießen\n",
      "1\n",
      "Bayreuth\n",
      "1\n",
      "Aschaffenburg\n",
      "1\n",
      "Rosenheim\n",
      "1\n",
      "Offenburg\n",
      "1\n",
      "Schweinfurt\n",
      "1\n",
      "Wetzlar\n",
      "1\n",
      "Braunschweig-Salzgitter-Wolfsburg\n",
      "1\n",
      "Mannheim-Ludwigshafen\n",
      "1\n",
      "Münster\n",
      "1\n",
      "Lübeck\n",
      "1\n",
      "Kassel\n",
      "1\n",
      "Osnabrück\n",
      "1\n",
      "Oldenburg (Oldenburg)\n",
      "1\n",
      "Heidelberg\n",
      "1\n",
      "Paderborn\n",
      "1\n",
      "Würzburg\n",
      "1\n",
      "Bremerhaven\n",
      "1\n",
      "Heilbronn\n",
      "1\n",
      "Ulm\n",
      "1\n",
      "Pforzheim\n",
      "1\n",
      "Ingolstadt\n",
      "1\n",
      "Reutlingen\n",
      "1\n",
      "Siegen\n",
      "1\n",
      "Hildesheim\n",
      "1\n",
      "Wuppertal\n",
      "1\n",
      "Düren\n",
      "1\n",
      "Bocholt\n",
      "0\n",
      "Tallinn\n",
      "1\n",
      "Athina\n",
      "1\n",
      "Thessaloniki\n",
      "1\n",
      "Madrid\n",
      "1\n",
      "Barcelona\n",
      "1\n",
      "Valencia\n",
      "1\n",
      "Sevilla\n",
      "1\n",
      "Zaragoza\n",
      "1\n",
      "Málaga - Marbella\n",
      "1\n",
      "Murcia - Cartagena\n",
      "2\n",
      "Las Palmas\n",
      "1\n",
      "Valladolid\n",
      "2\n",
      "Palma de Mallorca\n",
      "1\n",
      "Vitoria/Gasteiz\n",
      "1\n",
      "Oviedo - Gijón\n",
      "1\n",
      "Pamplona/Iruña\n",
      "1\n",
      "Santander\n",
      "1\n",
      "Bilbao\n",
      "1\n",
      "Córdoba\n",
      "1\n",
      "Alicante/Alacant - Elche/Elx\n",
      "1\n",
      "Vigo\n",
      "2\n",
      "Santa Cruz de Tenerife\n",
      "1\n",
      "A Coruña\n",
      "1\n",
      "Granada\n",
      "1\n",
      "Guipúzcoa\n",
      "1\n",
      "Cádiz\n",
      "1\n",
      "Zagreb\n",
      "1\n",
      "Split\n",
      "1\n",
      "Roma\n",
      "1\n",
      "Milano\n",
      "1\n",
      "Napoli\n",
      "1\n",
      "Torino\n",
      "1\n",
      "Palermo\n",
      "1\n",
      "Genova\n",
      "1\n",
      "Firenze\n",
      "1\n",
      "Bari\n",
      "1\n",
      "Bologna\n",
      "1\n",
      "Catania\n",
      "1\n",
      "Venezia\n",
      "1\n",
      "Verona\n",
      "1\n",
      "Taranto\n",
      "1\n",
      "Cagliari\n",
      "1\n",
      "Padova\n",
      "1\n",
      "Brescia\n",
      "1\n",
      "Messina\n",
      "1\n",
      "Prato\n",
      "1\n",
      "Parma\n",
      "1\n",
      "Reggio nell Emilia\n",
      "1\n",
      "Bergamo\n",
      "0\n",
      "Lefkosia\n",
      "1\n",
      "Riga\n",
      "1\n",
      "Vilnius\n",
      "1\n",
      "Kaunas\n",
      "1\n",
      "Budapest\n",
      "1\n",
      "Miskolc\n",
      "1\n",
      "Pécs\n",
      "1\n",
      "Debrecen\n",
      "1\n",
      "Székesfehérvár\n",
      "1\n",
      "Valletta\n",
      "3\n",
      "s' Gravenhage\n",
      "1\n",
      "Utrecht\n",
      "3\n",
      "Eindhoven\n",
      "3\n",
      "Tilburg\n",
      "3\n",
      "Groningen\n",
      "3\n",
      "Enschede\n",
      "3\n",
      "Arnhem - Nijmegen\n",
      "3\n",
      "Breda\n",
      "3\n",
      "Leiden\n",
      "3\n",
      "Zwolle\n",
      "1\n",
      "Wien\n",
      "1\n",
      "Graz\n",
      "1\n",
      "Linz\n",
      "1\n",
      "Salzburg\n",
      "1\n",
      "Innsbruck\n",
      "1\n",
      "Lisboa\n",
      "1\n",
      "Porto\n",
      "1\n",
      "Coimbra\n",
      "1\n",
      "Bucuresti\n",
      "1\n",
      "Cluj-Napoca\n",
      "1\n",
      "Timisoara\n",
      "1\n",
      "Craiova\n",
      "1\n",
      "Constanta\n",
      "1\n",
      "Iasi\n",
      "1\n",
      "Galati\n",
      "1\n",
      "Brasov\n",
      "1\n",
      "Ploiesti\n",
      "3\n",
      "Ljubljana\n",
      "3\n",
      "Maribor\n",
      "0\n",
      "Bratislava\n",
      "2\n",
      "Kosice\n",
      "1\n",
      "Helsinki\n",
      "1\n",
      "Tampere\n",
      "1\n",
      "Turku\n",
      "1\n",
      "Stockholm\n",
      "1\n",
      "Göteborg\n",
      "1\n",
      "Malmö\n",
      "1\n",
      "Uppsala\n",
      "2\n",
      "London\n",
      "2\n",
      "Leeds\n",
      "0\n",
      "Bradford\n",
      "2\n",
      "Liverpool\n",
      "2\n",
      "Manchester\n",
      "0\n",
      "Cardiff\n",
      "0\n",
      "Sheffield\n",
      "0\n",
      "Bristol\n",
      "0\n",
      "Newcastle upon Tyne\n",
      "0\n",
      "Leicester\n",
      "0\n",
      "Aberdeen\n",
      "0\n",
      "Cambridge\n",
      "0\n",
      "Exeter\n",
      "2\n",
      "Portsmouth\n",
      "0\n",
      "Coventry\n",
      "0\n",
      "Kingston upon Hull\n",
      "0\n",
      "Stoke-on-Trent\n",
      "0\n",
      "Nottingham\n",
      "2\n",
      "Kirklees\n",
      "0\n",
      "Doncaster\n",
      "0\n",
      "Sunderland\n",
      "0\n",
      "Medway\n",
      "0\n",
      "Brighton and Hove\n",
      "0\n",
      "Plymouth\n",
      "0\n",
      "Swansea\n",
      "0\n",
      "Derby\n",
      "2\n",
      "Southampton\n",
      "2\n",
      "Northampton\n",
      "0\n",
      "Bournemouth\n",
      "2\n",
      "Colchester\n",
      "2\n",
      "Blackburn - Blackpool - Preston\n",
      "0\n",
      "Middlesbrough\n",
      "0\n",
      "Oxford\n",
      "2\n",
      "Norwich\n",
      "2\n",
      "Cheshire West and Chester\n",
      "0\n",
      "Ipswich\n"
     ]
    }
   ],
   "source": [
    "cities_removed=[]\n",
    "for v in filtered_data2['METROREG'].unique():\n",
    "    print(max(list(filtered_data2[filtered_data2['METROREG'] == v].isnull().sum())))\n",
    "    print(v)\n",
    "    if max(list(filtered_data2[filtered_data2['METROREG'] == v].isnull().sum()))>8:\n",
    "        print(max(list(filtered_data2[filtered_data2['METROREG'] == v].isnull().sum())))\n",
    "        print(filtered_data2[filtered_data2['METROREG'] == v])\n",
    "        cities_removed.append(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TIME</th>\n",
       "      <th>METROREG</th>\n",
       "      <th>employed_persons</th>\n",
       "      <th>gdp</th>\n",
       "      <th>population</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2000</td>\n",
       "      <td>Belgium</td>\n",
       "      <td>4,109</td>\n",
       "      <td>256,376.4</td>\n",
       "      <td>10,239,085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2001</td>\n",
       "      <td>Belgium</td>\n",
       "      <td>4,165.2</td>\n",
       "      <td>264,334.9</td>\n",
       "      <td>10,263,414</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2002</td>\n",
       "      <td>Belgium</td>\n",
       "      <td>4,157.3</td>\n",
       "      <td>273,255.9</td>\n",
       "      <td>10,309,725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2003</td>\n",
       "      <td>Belgium</td>\n",
       "      <td>4,153.8</td>\n",
       "      <td>281,200.2</td>\n",
       "      <td>10,355,844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2004</td>\n",
       "      <td>Belgium</td>\n",
       "      <td>4,194.7</td>\n",
       "      <td>296,819.7</td>\n",
       "      <td>10,396,421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5885</th>\n",
       "      <td>2014</td>\n",
       "      <td>Non-metropolitan regions in Serbia</td>\n",
       "      <td>:</td>\n",
       "      <td>35,445.39</td>\n",
       "      <td>:</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5886</th>\n",
       "      <td>2015</td>\n",
       "      <td>Non-metropolitan regions in Serbia</td>\n",
       "      <td>2,582.15</td>\n",
       "      <td>35,690.25</td>\n",
       "      <td>:</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5887</th>\n",
       "      <td>2016</td>\n",
       "      <td>Non-metropolitan regions in Serbia</td>\n",
       "      <td>2,727.4</td>\n",
       "      <td>36,698.57</td>\n",
       "      <td>:</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5888</th>\n",
       "      <td>2017</td>\n",
       "      <td>Non-metropolitan regions in Serbia</td>\n",
       "      <td>2,802.61</td>\n",
       "      <td>39,157.73</td>\n",
       "      <td>7,040,272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5889</th>\n",
       "      <td>2018</td>\n",
       "      <td>Non-metropolitan regions in Serbia</td>\n",
       "      <td>:</td>\n",
       "      <td>:</td>\n",
       "      <td>7,001,444</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5890 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      TIME                            METROREG employed_persons        gdp  population\n",
       "0     2000                             Belgium            4,109  256,376.4  10,239,085\n",
       "1     2001                             Belgium          4,165.2  264,334.9  10,263,414\n",
       "2     2002                             Belgium          4,157.3  273,255.9  10,309,725\n",
       "3     2003                             Belgium          4,153.8  281,200.2  10,355,844\n",
       "4     2004                             Belgium          4,194.7  296,819.7  10,396,421\n",
       "...    ...                                 ...              ...        ...         ...\n",
       "5885  2014  Non-metropolitan regions in Serbia                :  35,445.39           :\n",
       "5886  2015  Non-metropolitan regions in Serbia         2,582.15  35,690.25           :\n",
       "5887  2016  Non-metropolitan regions in Serbia          2,727.4  36,698.57           :\n",
       "5888  2017  Non-metropolitan regions in Serbia         2,802.61  39,157.73   7,040,272\n",
       "5889  2018  Non-metropolitan regions in Serbia                :          :   7,001,444\n",
       "\n",
       "[5890 rows x 5 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cities_removed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max(list(filtered_data2[filtered_data2['METROREG'] == 'Roma'].isnull().sum()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all(x >=9 for x in list(filtered_data2[filtered_data2['METROREG'] == 'Bergen'].isnull().sum()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'according to various sources gdp per capita growth is linear in european countries since 1950 \\n(Real GDP per capita in developed countries, Ivan O. Kitov)'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"according to various sources gdp per capita growth is linear in european countries since 1950 \n",
    "(Real GDP per capita in developed countries, Ivan O. Kitov)\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TIME</th>\n",
       "      <th>METROREG</th>\n",
       "      <th>employed_persons</th>\n",
       "      <th>gdp</th>\n",
       "      <th>population</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>521</th>\n",
       "      <td>2008</td>\n",
       "      <td>Hamburg</td>\n",
       "      <td>1681.000000</td>\n",
       "      <td>128296.000000</td>\n",
       "      <td>3184610.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>522</th>\n",
       "      <td>2009</td>\n",
       "      <td>Hamburg</td>\n",
       "      <td>1701.000000</td>\n",
       "      <td>123952.000000</td>\n",
       "      <td>3187338.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>523</th>\n",
       "      <td>2010</td>\n",
       "      <td>Hamburg</td>\n",
       "      <td>1711.000000</td>\n",
       "      <td>127276.000000</td>\n",
       "      <td>3192336.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524</th>\n",
       "      <td>2011</td>\n",
       "      <td>Hamburg</td>\n",
       "      <td>1737.000000</td>\n",
       "      <td>130373.000000</td>\n",
       "      <td>3112650.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>525</th>\n",
       "      <td>2012</td>\n",
       "      <td>Hamburg</td>\n",
       "      <td>1762.000000</td>\n",
       "      <td>133523.000000</td>\n",
       "      <td>3130182.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>526</th>\n",
       "      <td>2013</td>\n",
       "      <td>Hamburg</td>\n",
       "      <td>1788.000000</td>\n",
       "      <td>138419.000000</td>\n",
       "      <td>3152229.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>527</th>\n",
       "      <td>2014</td>\n",
       "      <td>Hamburg</td>\n",
       "      <td>1800.000000</td>\n",
       "      <td>142644.000000</td>\n",
       "      <td>3173871.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>528</th>\n",
       "      <td>2015</td>\n",
       "      <td>Hamburg</td>\n",
       "      <td>1816.000000</td>\n",
       "      <td>149161.000000</td>\n",
       "      <td>3201905.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>529</th>\n",
       "      <td>2016</td>\n",
       "      <td>Hamburg</td>\n",
       "      <td>1849.000000</td>\n",
       "      <td>151759.000000</td>\n",
       "      <td>3243171.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>530</th>\n",
       "      <td>2017</td>\n",
       "      <td>Hamburg</td>\n",
       "      <td>1881.000000</td>\n",
       "      <td>157840.000000</td>\n",
       "      <td>3282164.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>531</th>\n",
       "      <td>2018</td>\n",
       "      <td>Hamburg</td>\n",
       "      <td>1870.496732</td>\n",
       "      <td>154076.738615</td>\n",
       "      <td>3309215.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>532</th>\n",
       "      <td>2000</td>\n",
       "      <td>München</td>\n",
       "      <td>1520.000000</td>\n",
       "      <td>114553.000000</td>\n",
       "      <td>2410083.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>533</th>\n",
       "      <td>2001</td>\n",
       "      <td>München</td>\n",
       "      <td>1551.000000</td>\n",
       "      <td>120348.000000</td>\n",
       "      <td>2446014.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>534</th>\n",
       "      <td>2002</td>\n",
       "      <td>München</td>\n",
       "      <td>1502.000000</td>\n",
       "      <td>122440.000000</td>\n",
       "      <td>2483013.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>535</th>\n",
       "      <td>2003</td>\n",
       "      <td>München</td>\n",
       "      <td>1531.000000</td>\n",
       "      <td>120382.000000</td>\n",
       "      <td>2501593.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>536</th>\n",
       "      <td>2004</td>\n",
       "      <td>München</td>\n",
       "      <td>1533.000000</td>\n",
       "      <td>124873.000000</td>\n",
       "      <td>2521332.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>537</th>\n",
       "      <td>2005</td>\n",
       "      <td>München</td>\n",
       "      <td>1545.000000</td>\n",
       "      <td>126908.000000</td>\n",
       "      <td>2531706.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>538</th>\n",
       "      <td>2006</td>\n",
       "      <td>München</td>\n",
       "      <td>1551.000000</td>\n",
       "      <td>131250.000000</td>\n",
       "      <td>2551737.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>539</th>\n",
       "      <td>2007</td>\n",
       "      <td>München</td>\n",
       "      <td>1586.000000</td>\n",
       "      <td>139235.000000</td>\n",
       "      <td>2593707.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>540</th>\n",
       "      <td>2008</td>\n",
       "      <td>München</td>\n",
       "      <td>1613.000000</td>\n",
       "      <td>136575.000000</td>\n",
       "      <td>2623249.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>541</th>\n",
       "      <td>2009</td>\n",
       "      <td>München</td>\n",
       "      <td>1623.000000</td>\n",
       "      <td>134088.000000</td>\n",
       "      <td>2644807.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>542</th>\n",
       "      <td>2010</td>\n",
       "      <td>München</td>\n",
       "      <td>1640.000000</td>\n",
       "      <td>137177.000000</td>\n",
       "      <td>2654726.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>543</th>\n",
       "      <td>2011</td>\n",
       "      <td>München</td>\n",
       "      <td>1669.000000</td>\n",
       "      <td>147566.000000</td>\n",
       "      <td>2649862.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544</th>\n",
       "      <td>2012</td>\n",
       "      <td>München</td>\n",
       "      <td>1704.000000</td>\n",
       "      <td>152539.000000</td>\n",
       "      <td>2690153.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>545</th>\n",
       "      <td>2013</td>\n",
       "      <td>München</td>\n",
       "      <td>1747.000000</td>\n",
       "      <td>157344.000000</td>\n",
       "      <td>2730606.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>546</th>\n",
       "      <td>2014</td>\n",
       "      <td>München</td>\n",
       "      <td>1778.000000</td>\n",
       "      <td>163698.000000</td>\n",
       "      <td>2768488.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>547</th>\n",
       "      <td>2015</td>\n",
       "      <td>München</td>\n",
       "      <td>1813.000000</td>\n",
       "      <td>171804.000000</td>\n",
       "      <td>2804001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548</th>\n",
       "      <td>2016</td>\n",
       "      <td>München</td>\n",
       "      <td>1850.000000</td>\n",
       "      <td>180032.000000</td>\n",
       "      <td>2848906.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>549</th>\n",
       "      <td>2017</td>\n",
       "      <td>München</td>\n",
       "      <td>1889.000000</td>\n",
       "      <td>189161.000000</td>\n",
       "      <td>2879107.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>550</th>\n",
       "      <td>2018</td>\n",
       "      <td>München</td>\n",
       "      <td>1855.954247</td>\n",
       "      <td>180289.797317</td>\n",
       "      <td>2883373.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>551</th>\n",
       "      <td>2000</td>\n",
       "      <td>Köln</td>\n",
       "      <td>985.000000</td>\n",
       "      <td>64392.000000</td>\n",
       "      <td>1851563.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>552</th>\n",
       "      <td>2001</td>\n",
       "      <td>Köln</td>\n",
       "      <td>993.000000</td>\n",
       "      <td>66568.000000</td>\n",
       "      <td>1854892.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>553</th>\n",
       "      <td>2002</td>\n",
       "      <td>Köln</td>\n",
       "      <td>1001.000000</td>\n",
       "      <td>66891.000000</td>\n",
       "      <td>1863724.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>554</th>\n",
       "      <td>2003</td>\n",
       "      <td>Köln</td>\n",
       "      <td>980.000000</td>\n",
       "      <td>67351.000000</td>\n",
       "      <td>1867887.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>555</th>\n",
       "      <td>2004</td>\n",
       "      <td>Köln</td>\n",
       "      <td>985.000000</td>\n",
       "      <td>68812.000000</td>\n",
       "      <td>1868077.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>556</th>\n",
       "      <td>2005</td>\n",
       "      <td>Köln</td>\n",
       "      <td>984.000000</td>\n",
       "      <td>68210.000000</td>\n",
       "      <td>1873580.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>557</th>\n",
       "      <td>2006</td>\n",
       "      <td>Köln</td>\n",
       "      <td>995.000000</td>\n",
       "      <td>71403.000000</td>\n",
       "      <td>1886528.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>558</th>\n",
       "      <td>2007</td>\n",
       "      <td>Köln</td>\n",
       "      <td>1013.000000</td>\n",
       "      <td>75337.000000</td>\n",
       "      <td>1892907.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>559</th>\n",
       "      <td>2008</td>\n",
       "      <td>Köln</td>\n",
       "      <td>1033.000000</td>\n",
       "      <td>73300.000000</td>\n",
       "      <td>1899296.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>560</th>\n",
       "      <td>2009</td>\n",
       "      <td>Köln</td>\n",
       "      <td>1038.000000</td>\n",
       "      <td>72475.000000</td>\n",
       "      <td>1898438.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>561</th>\n",
       "      <td>2010</td>\n",
       "      <td>Köln</td>\n",
       "      <td>1044.000000</td>\n",
       "      <td>74607.000000</td>\n",
       "      <td>1899930.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>562</th>\n",
       "      <td>2011</td>\n",
       "      <td>Köln</td>\n",
       "      <td>1064.000000</td>\n",
       "      <td>77385.000000</td>\n",
       "      <td>1891238.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>563</th>\n",
       "      <td>2012</td>\n",
       "      <td>Köln</td>\n",
       "      <td>1080.000000</td>\n",
       "      <td>79147.000000</td>\n",
       "      <td>1903646.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>564</th>\n",
       "      <td>2013</td>\n",
       "      <td>Köln</td>\n",
       "      <td>1098.000000</td>\n",
       "      <td>81058.000000</td>\n",
       "      <td>1917088.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>565</th>\n",
       "      <td>2014</td>\n",
       "      <td>Köln</td>\n",
       "      <td>1107.000000</td>\n",
       "      <td>85595.000000</td>\n",
       "      <td>1930036.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>566</th>\n",
       "      <td>2015</td>\n",
       "      <td>Köln</td>\n",
       "      <td>1125.000000</td>\n",
       "      <td>91880.000000</td>\n",
       "      <td>1947165.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>567</th>\n",
       "      <td>2016</td>\n",
       "      <td>Köln</td>\n",
       "      <td>1142.000000</td>\n",
       "      <td>93639.000000</td>\n",
       "      <td>1973455.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>568</th>\n",
       "      <td>2017</td>\n",
       "      <td>Köln</td>\n",
       "      <td>1160.000000</td>\n",
       "      <td>96636.000000</td>\n",
       "      <td>1987901.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>569</th>\n",
       "      <td>2018</td>\n",
       "      <td>Köln</td>\n",
       "      <td>1147.424837</td>\n",
       "      <td>92973.620941</td>\n",
       "      <td>1994524.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>570</th>\n",
       "      <td>2000</td>\n",
       "      <td>Frankfurt am Main</td>\n",
       "      <td>1444.000000</td>\n",
       "      <td>105046.000000</td>\n",
       "      <td>2481064.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     TIME           METROREG  employed_persons            gdp  population\n",
       "521  2008            Hamburg       1681.000000  128296.000000   3184610.0\n",
       "522  2009            Hamburg       1701.000000  123952.000000   3187338.0\n",
       "523  2010            Hamburg       1711.000000  127276.000000   3192336.0\n",
       "524  2011            Hamburg       1737.000000  130373.000000   3112650.0\n",
       "525  2012            Hamburg       1762.000000  133523.000000   3130182.0\n",
       "526  2013            Hamburg       1788.000000  138419.000000   3152229.0\n",
       "527  2014            Hamburg       1800.000000  142644.000000   3173871.0\n",
       "528  2015            Hamburg       1816.000000  149161.000000   3201905.0\n",
       "529  2016            Hamburg       1849.000000  151759.000000   3243171.0\n",
       "530  2017            Hamburg       1881.000000  157840.000000   3282164.0\n",
       "531  2018            Hamburg       1870.496732  154076.738615   3309215.0\n",
       "532  2000            München       1520.000000  114553.000000   2410083.0\n",
       "533  2001            München       1551.000000  120348.000000   2446014.0\n",
       "534  2002            München       1502.000000  122440.000000   2483013.0\n",
       "535  2003            München       1531.000000  120382.000000   2501593.0\n",
       "536  2004            München       1533.000000  124873.000000   2521332.0\n",
       "537  2005            München       1545.000000  126908.000000   2531706.0\n",
       "538  2006            München       1551.000000  131250.000000   2551737.0\n",
       "539  2007            München       1586.000000  139235.000000   2593707.0\n",
       "540  2008            München       1613.000000  136575.000000   2623249.0\n",
       "541  2009            München       1623.000000  134088.000000   2644807.0\n",
       "542  2010            München       1640.000000  137177.000000   2654726.0\n",
       "543  2011            München       1669.000000  147566.000000   2649862.0\n",
       "544  2012            München       1704.000000  152539.000000   2690153.0\n",
       "545  2013            München       1747.000000  157344.000000   2730606.0\n",
       "546  2014            München       1778.000000  163698.000000   2768488.0\n",
       "547  2015            München       1813.000000  171804.000000   2804001.0\n",
       "548  2016            München       1850.000000  180032.000000   2848906.0\n",
       "549  2017            München       1889.000000  189161.000000   2879107.0\n",
       "550  2018            München       1855.954247  180289.797317   2883373.0\n",
       "551  2000               Köln        985.000000   64392.000000   1851563.0\n",
       "552  2001               Köln        993.000000   66568.000000   1854892.0\n",
       "553  2002               Köln       1001.000000   66891.000000   1863724.0\n",
       "554  2003               Köln        980.000000   67351.000000   1867887.0\n",
       "555  2004               Köln        985.000000   68812.000000   1868077.0\n",
       "556  2005               Köln        984.000000   68210.000000   1873580.0\n",
       "557  2006               Köln        995.000000   71403.000000   1886528.0\n",
       "558  2007               Köln       1013.000000   75337.000000   1892907.0\n",
       "559  2008               Köln       1033.000000   73300.000000   1899296.0\n",
       "560  2009               Köln       1038.000000   72475.000000   1898438.0\n",
       "561  2010               Köln       1044.000000   74607.000000   1899930.0\n",
       "562  2011               Köln       1064.000000   77385.000000   1891238.0\n",
       "563  2012               Köln       1080.000000   79147.000000   1903646.0\n",
       "564  2013               Köln       1098.000000   81058.000000   1917088.0\n",
       "565  2014               Köln       1107.000000   85595.000000   1930036.0\n",
       "566  2015               Köln       1125.000000   91880.000000   1947165.0\n",
       "567  2016               Köln       1142.000000   93639.000000   1973455.0\n",
       "568  2017               Köln       1160.000000   96636.000000   1987901.0\n",
       "569  2018               Köln       1147.424837   92973.620941   1994524.0\n",
       "570  2000  Frankfurt am Main       1444.000000  105046.000000   2481064.0"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_data3[350:400]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEST = filtered_data2[filtered_data2['METROREG'] == 'Bergen']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TIME                0\n",
       "METROREG            0\n",
       "employed_persons    9\n",
       "gdp                 9\n",
       "population          5\n",
       "dtype: int64"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TEST.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TIME</th>\n",
       "      <th>METROREG</th>\n",
       "      <th>employed_persons</th>\n",
       "      <th>gdp</th>\n",
       "      <th>population</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5795</th>\n",
       "      <td>2000</td>\n",
       "      <td>Bergen</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5796</th>\n",
       "      <td>2001</td>\n",
       "      <td>Bergen</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5797</th>\n",
       "      <td>2002</td>\n",
       "      <td>Bergen</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5798</th>\n",
       "      <td>2003</td>\n",
       "      <td>Bergen</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5799</th>\n",
       "      <td>2004</td>\n",
       "      <td>Bergen</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5800</th>\n",
       "      <td>2005</td>\n",
       "      <td>Bergen</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>841447.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5801</th>\n",
       "      <td>2006</td>\n",
       "      <td>Bergen</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>850205.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5802</th>\n",
       "      <td>2007</td>\n",
       "      <td>Bergen</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>861277.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5803</th>\n",
       "      <td>2008</td>\n",
       "      <td>Bergen</td>\n",
       "      <td>485.0</td>\n",
       "      <td>47672.75</td>\n",
       "      <td>875361.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5804</th>\n",
       "      <td>2009</td>\n",
       "      <td>Bergen</td>\n",
       "      <td>485.0</td>\n",
       "      <td>45909.48</td>\n",
       "      <td>890255.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5805</th>\n",
       "      <td>2010</td>\n",
       "      <td>Bergen</td>\n",
       "      <td>485.0</td>\n",
       "      <td>53100.79</td>\n",
       "      <td>905122.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5806</th>\n",
       "      <td>2011</td>\n",
       "      <td>Bergen</td>\n",
       "      <td>494.0</td>\n",
       "      <td>55935.94</td>\n",
       "      <td>920327.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5807</th>\n",
       "      <td>2012</td>\n",
       "      <td>Bergen</td>\n",
       "      <td>510.0</td>\n",
       "      <td>62965.14</td>\n",
       "      <td>933685.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5808</th>\n",
       "      <td>2013</td>\n",
       "      <td>Bergen</td>\n",
       "      <td>522.0</td>\n",
       "      <td>64748.35</td>\n",
       "      <td>950294.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5809</th>\n",
       "      <td>2014</td>\n",
       "      <td>Bergen</td>\n",
       "      <td>530.0</td>\n",
       "      <td>63141.39</td>\n",
       "      <td>964534.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5810</th>\n",
       "      <td>2015</td>\n",
       "      <td>Bergen</td>\n",
       "      <td>536.0</td>\n",
       "      <td>61303.78</td>\n",
       "      <td>977762.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5811</th>\n",
       "      <td>2016</td>\n",
       "      <td>Bergen</td>\n",
       "      <td>523.0</td>\n",
       "      <td>57895.42</td>\n",
       "      <td>985823.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5812</th>\n",
       "      <td>2017</td>\n",
       "      <td>Bergen</td>\n",
       "      <td>522.0</td>\n",
       "      <td>58213.67</td>\n",
       "      <td>991987.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5813</th>\n",
       "      <td>2018</td>\n",
       "      <td>Bergen</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>996064.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      TIME METROREG  employed_persons       gdp  population\n",
       "5795  2000   Bergen               NaN       NaN         NaN\n",
       "5796  2001   Bergen               NaN       NaN         NaN\n",
       "5797  2002   Bergen               NaN       NaN         NaN\n",
       "5798  2003   Bergen               NaN       NaN         NaN\n",
       "5799  2004   Bergen               NaN       NaN         NaN\n",
       "5800  2005   Bergen               NaN       NaN    841447.0\n",
       "5801  2006   Bergen               NaN       NaN    850205.0\n",
       "5802  2007   Bergen               NaN       NaN    861277.0\n",
       "5803  2008   Bergen             485.0  47672.75    875361.0\n",
       "5804  2009   Bergen             485.0  45909.48    890255.0\n",
       "5805  2010   Bergen             485.0  53100.79    905122.0\n",
       "5806  2011   Bergen             494.0  55935.94    920327.0\n",
       "5807  2012   Bergen             510.0  62965.14    933685.0\n",
       "5808  2013   Bergen             522.0  64748.35    950294.0\n",
       "5809  2014   Bergen             530.0  63141.39    964534.0\n",
       "5810  2015   Bergen             536.0  61303.78    977762.0\n",
       "5811  2016   Bergen             523.0  57895.42    985823.0\n",
       "5812  2017   Bergen             522.0  58213.67    991987.0\n",
       "5813  2018   Bergen               NaN       NaN    996064.0"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TEST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "X=[i for i in list(TEST.population) if str(i)!='nan']\n",
    "Y=[i for i in range(len(list(TEST.population))) if str(list(TEST.population)[i])!='nan']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x280b459d340>"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEDCAYAAAA7jc+ZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAYqElEQVR4nO3df5Bd9X3e8ffTFQoCjKWUNQ0rDZIzsmArfojcarBpXWKcQTgei2qmHUhcpypUQwYwYRpiQWbayfgPMcFuQwdqRoMxoXZhHAIKTlzLGcgUtxNAV0hC6Fe9IyVoV6QscWS1tqZC8PSPe4Qvl7u6R6u7e+89PK+ZndU533N0P1ejfXT0Ped+P7JNRERU19/rdQERETGzEvQRERWXoI+IqLgEfURExSXoIyIqLkEfEVFxfRv0kh6R9IakV0se/y8k7Za0S9J/nen6IiIGhfr1OXpJnwT+L/CY7eUdjl0KfBv4lO2/k/QR22/MRp0REf2ub6/obT8P/Kh5n6RflPQ9SVsl/UDSRcXQvwEetP13xbkJ+YiIQt8G/RQ2Arfb/iXgt4H/XOz/GPAxSf9T0guSVvWswoiIPjOn1wWUJekc4BPAH0k6sfvniu9zgKXA1cBC4AeSlts+PNt1RkT0m4EJehr/+zhs+/I2Y+PAC7bfAg5I2kcj+LfMZoEREf1oYKZubB+hEeL/HEANlxXDm4BfLvafR2MqZ39PCo2I6DN9G/SSHgf+ElgmaVzSTcCvAzdJ2gHsAlYXh28G/lbSbuAvgLts/20v6o6I6Dd9+3hlRER0R99e0UdERHf05c3Y8847z4sXL+51GRERA2Pr1q1v2h5uN9aXQb948WLq9Xqvy4iIGBiS/nqqsY5TN53WnCmefvlPksYkvSLpiqaxVZL2FWPrp1d+REScjjJz9I8CJ/uk6XU0nllfCqwDvgYgaQh4sBgfBW6UNHo6xUZExKnrGPTt1pxpsZrGwmO2/QIwX9IvACuBMdv7bR8DnuBnj0NGRMQs6cZTNyPAwabt8WLfVPvbkrROUl1SfXJysgtlRUQEdCfo1WafT7K/Ldsbbdds14aH2944joiIaejGUzfjwKKm7YXAIWDuFPsjIqLJpm0T3Ld5H4cOH+WC+fO469plXL9iygmQU9aNK/pngC8UT99cCfzY9us0FhRbKmmJpLnADcWxERFR2LRtgruf2snE4aMYmDh8lLuf2smmbRNde42OV/TFmjNXA+dJGgf+PXAGgO2HgO8CnwHGgJ8Ca4ux45Juo7EOzRDwiO1dXas8IqIC7tu8j6Nvvf2efUffepv7Nu/r2lV9x6C3fWOHcQO3TjH2XRr/EERERBuHDh89pf3TkbVuIiJ66IL5805p/3Qk6CMiSti0bYKr7n2OJev/jKvufa5rc+h3XbuMeWcMvWffvDOGuOvaZV35/aFP17qJiOgnJ26YnphLP3HDFDjtefQT58/kUzcJ+oiIDmb6hun1K0a6GuytMnUTEdHBbNwwnUkJ+oiIDmbjhulMStBHRHQwGzdMZ1Lm6CMiOpiNG6YzKUEfEVHCTN8wnUkJ+oiohJleGGyQJegjYuDN5HPuVZCbsREx8E72nHsk6COiAgb9OfeZlqCPiIE36M+5z7QEfUQMvEF/zn2m5WZsRAy8QX/OfaYl6COiEgb5OfeZVmrqRtIqSfskjUla32Z8gaSnJb0i6SVJy5vG7pS0S9Krkh6XdGY330BERJxcx6CXNAQ8CFwHjAI3ShptOeweYLvtS4EvAPcX544AXwRqtpfT6B17Q/fKj4iITspc0a8Exmzvt30MeAJY3XLMKPAsgO29wGJJ5xdjc4B5kuYAZwGHulJ5RESUUiboR4CDTdvjxb5mO4A1AJJWAhcCC21PAF8BXgNeB35s+/vtXkTSOkl1SfXJyclTexcRMRBmqh1fnFyZoFebfW7ZvhdYIGk7cDuwDTguaQGNq/8lwAXA2ZI+3+5FbG+0XbNdGx4eLv0GImIwnFimYOLwUczPlilI2M+8MkE/Dixq2l5Iy/SL7SO219q+nMYc/TBwAPg0cMD2pO23gKeAT3Sl8ogYKFmmoHfKBP0WYKmkJZLm0riZ+kzzAZLmF2MANwPP2z5CY8rmSklnSRJwDbCne+VHxKDIMgW90zHobR8HbgM20wjpb9veJekWSbcUh10M7JK0l8bTOXcU574IPAm8DOwsXm9j199FRPS9LFPQO7Jbp9t7r1aruV6v97qMiOii1qWEobFMwYY1l+SDTl0gaavtWruxfDI2ImZFlinonQR9RMyaLFPQG1m9MiKi4hL0EREVl6CPiKi4BH1ERMUl6CMiKi5BHxFRcQn6iIiKS9BHRFRcgj4iouLyydiIeI9N2yayTEHFJOgj4l2tC4+daA4CJOwHWKZuIuJdaQ5STQn6iHhXmoNUU4I+It6V5iDVVCroJa2StE/SmKT1bcYXSHpa0iuSXpK0vGlsvqQnJe2VtEfSx7v5BiKie+66dhnzzhh6z755Zwxx17XLelRRdEPHoJc0BDxIo0XgKHCjpNGWw+4Bttu+lEZz8Pubxu4Hvmf7IuAy0jM2om9dv2KEDWsuYWT+PASMzJ+XDlAVUOapm5XAmO39AJKeAFYDu5uOGQU2ANjeK2mxpPOBo8AngX9VjB0DjnWt+ojoujQHqZ4yUzcjwMGm7fFiX7MdwBoASSuBC4GFwEeBSeAbkrZJeljS2e1eRNI6SXVJ9cnJyVN8GxERMZUyQa82+1o7it8LLJC0Hbgd2AYcp/E/hiuAr9leAfwEeN8cP4DtjbZrtmvDw8Nl64+IiA7KTN2MA4uathcCh5oPsH0EWAsgScCB4ussYNz2i8WhTzJF0EdExMwoc0W/BVgqaYmkucANwDPNBxRP1swtNm8Gnrd9xPbfAAclnbhlfw3vnduPiIgZ1vGK3vZxSbcBm4Eh4BHbuyTdUow/BFwMPCbpbRpBflPTb3E78K3iH4L9FFf+ERExO2S3Trf3Xq1Wc71e73UZEREDQ9JW27V2Y/lkbERExSXoIyIqLkEfEVFxCfqIiIpL45GIAZMOUHGqEvQRAyQdoGI6MnUTMUDSASqmI0EfMUDSASqmI0EfMUDSASqmI0EfMUDSASqmIzdjIwbIiRuueeomTkWCPmLApANUnKpM3UREVFyCPiKi4hL0EREVl6CPiKi4BH1ERMWVCnpJqyTtkzQm6X3NvSUtkPS0pFckvSRpecv4kKRtkv60W4VHREQ5HYNe0hDwIHAdMArcKGm05bB7gO22LwW+ANzfMn4HsOf0y42IiFNV5op+JTBme7/tY8ATwOqWY0aBZwFs7wUWSzofQNJC4FeBh7tWdURElFYm6EeAg03b48W+ZjuANQCSVgIXAguLsT8Afgd452QvImmdpLqk+uTkZImyIiKijDJBrzb73LJ9L7BA0nbgdmAbcFzSZ4E3bG/t9CK2N9qu2a4NDw+XKCsiIsooswTCOLCoaXshcKj5ANtHgLUAkgQcKL5uAD4n6TPAmcC5kr5p+/NdqD0iIkooc0W/BVgqaYmkuTTC+5nmAyTNL8YAbgaet33E9t22F9peXJz3XEI+ImJ2dbyit31c0m3AZmAIeMT2Lkm3FOMPARcDj0l6G9gN3DSDNUdExCmQ3Trd3nu1Ws31er3XZURMWxp4x2yTtNV2rd1YlimO6LI08I5+kyUQIrosDbyj3yToI7osDbyj3yToI7osDbyj3yToI7osDbyj3+RmbESXpYF39JsEfcQMSAPv6CeZuomIqLgEfURExSXoIyIqLkEfEVFxCfqIiIpL0EdEVFyCPiKi4hL0EREVl6CPiKi4UkEvaZWkfZLGJK1vM75A0tOSXpH0kqTlxf5Fkv5C0h5JuyTd0e03EDEdm7ZNcNW9z7Fk/Z9x1b3PsWnbRK9LipgxHYNe0hDwIHAdMArcKGm05bB7gO22LwW+ANxf7D8O/FvbFwNXAre2OTdiVp1oDDJx+CjmZ41BEvZRVWWu6FcCY7b32z4GPAGsbjlmFHgWwPZeYLGk822/bvvlYv//AfYAWQAkeiqNQeKDpkzQjwAHm7bHeX9Y7wDWAEhaCVwILGw+QNJiYAXwYrsXkbROUl1SfXJyskztEdOSxiDxQVMm6NVmX2tH8XuBBZK2A7cD22hM2zR+A+kc4I+B37J9pN2L2N5ou2a7Njw8XKr4iOlIY5D4oCkT9OPAoqbthcCh5gNsH7G91vblNOboh4EDAJLOoBHy37L9VFeqjjgNaQwSHzRl1qPfAiyVtASYAG4Afq35AEnzgZ8Wc/g3A8/bPiJJwNeBPbb/Q3dLj5ieNAaJD5qOQW/7uKTbgM3AEPCI7V2SbinGHwIuBh6T9DawG7ipOP0q4F8CO4tpHYB7bH+3y+8j4pSkMUh8kJTqMFUE83db9j3U9Ou/BJa2Oe9/0H6OPyIiZkk+GRsRUXEJ+oiIikvQR0RUXII+IqLiEvQRERWXoI+IqLgEfURExSXoIyIqLkEfEVFxCfqIiIpL0EdEVFyptW4iemHTtomsMBnRBQn66Esn+rqeaPl3oq8rkLCPOEWZuom+lL6uEd2ToI++lL6uEd2ToI++lL6uEd2ToI++lL6uEd1TKuglrZK0T9KYpPVtxhdIelrSK5JekrS87LkR7Vy/YoQNay5hZP48BIzMn8eGNZfkRmzENHR86kbSEPAg8CvAOLBF0jO2dzcddg+w3fY/k3RRcfw1Jc+NaCt9XSO6o8wV/UpgzPZ+28eAJ4DVLceMAs8C2N4LLJZ0fslzIyJiBpUJ+hHgYNP2eLGv2Q5gDYCklcCFwMKS51Kct05SXVJ9cnKyXPUREdFRmaBXm31u2b4XWCBpO3A7sA04XvLcxk57o+2a7drw8HCJsiIioowyn4wdBxY1bS8EDjUfYPsIsBZAkoADxddZnc6NiIiZVeaKfguwVNISSXOBG4Bnmg+QNL8YA7gZeL4I/47nRkTEzOp4RW/7uKTbgM3AEPCI7V2SbinGHwIuBh6T9DawG7jpZOfOzFuJiIh2ZLedMu+pWq3mer3e6zIiIgaGpK22a+3G8snYiIiKS9BHRFRcgj4iouIS9BERFZcOUzFtafUXMRgS9DEtafUXMTgydRPTklZ/EYMjQR/TklZ/EYMjQR/TklZ/EYMjQR/TklZ/EYMjN2NjWk7ccM1TNxH9L0Ef05ZWfxGDIVM3EREVl6CPiKi4BH1ERMUl6CMiKq5U0EtaJWmfpDFJ69uMf1jSdyTtkLRL0tqmsTuLfa9KelzSmd18AxERcXIdg17SEPAgcB0wCtwoabTlsFuB3bYvA64GvipprqQR4ItAzfZyGu0Eb+hi/RER0UGZK/qVwJjt/baPAU8Aq1uOMfAhSQLOAX4EHC/G5gDzJM0BzgIOdaXyiIgopUzQjwAHm7bHi33NHqDRIPwQsBO4w/Y7tieArwCvAa8DP7b9/dOuOiIiSisT9Gqzr7Wj+LXAduAC4HLgAUnnSlpA4+p/STF2tqTPt30RaZ2kuqT65ORk6TcQEREnVybox4FFTdsLef/0y1rgKTeMAQeAi4BPAwdsT9p+C3gK+ES7F7G90XbNdm14ePhU30dEREyhTNBvAZZKWiJpLo2bqc+0HPMacA2ApPOBZcD+Yv+Vks4q5u+vAfZ0q/iIiOis41o3to9Lug3YTOOpmUds75J0SzH+EPBl4FFJO2lM9XzJ9pvAm5KeBF6mcXN2G7BxZt5KtJN2fxEhu3W6vfdqtZrr9Xqvyxh4re3+oLGU8IY1lyTsIypG0lbbtXZj+WRshaXdX0RAgr7S0u4vIiBBX2lp9xcRkKCvtLT7iwhIh6lKS7u/iIAEfeWl3V9EZOomIqLiEvQRERWXoI+IqLgEfURExSXoIyIqLkEfEVFxCfqIiIpL0EdEVFyCPiKi4hL0EREVlyUQ+kC6QEXETCp1RS9plaR9ksYkrW8z/mFJ35G0Q9IuSWubxuZLelLSXkl7JH28m29g0J3oAjVx+CgGJg4f5e6ndrJp20SvS4uIiugY9JKGgAeB64BR4EZJoy2H3Qrstn0ZcDXw1aKROMD9wPdsXwRcRpqDv0e6QEXETCtzRb8SGLO93/Yx4AlgdcsxBj4kScA5wI+A45LOBT4JfB3A9jHbh7tWfQWkC1REzLQyQT8CHGzaHi/2NXsAuBg4BOwE7rD9DvBRYBL4hqRtkh6WdHa7F5G0TlJdUn1ycvJU38fASheoiJhpZYJebfa5ZftaYDtwAXA58EBxNT8HuAL4mu0VwE+A983xA9jeaLtmuzY8PFy2/oGXLlARMdPKBP04sKhpeyGNK/dma4Gn3DAGHAAuKs4dt/1icdyTNII/CtevGGHDmksYmT8PASPz57FhzSV56iYiuqbM45VbgKWSlgATwA3Ar7Uc8xpwDfADSecDy4D9tt+UdFDSMtv7imN2d6/8akgXqIiYSR2D3vZxSbcBm4Eh4BHbuyTdUow/BHwZeFTSThpTPV+y/WbxW9wOfKt4Cmc/jav/iIiYJbJbp9t7r1aruV6v97qMiIiBIWmr7Vq7sSyBEBFRcQn6iIiKS9BHRFRcgj4iouKyemUJWV0yIgZZgr6DE6tLnlh47MTqkkDCPiIGQqZuOsjqkhEx6BL0HWR1yYgYdAn6DrK6ZEQMugR9B1ldMiIGXW7GdnDihmueuomIQZWgLyGrS0bEIMvUTURExSXoIyIqLkEfEVFxlZmjzzIFERHtVSLos0xBRMTUSk3dSFolaZ+kMUnr24x/WNJ3JO2QtEvS2pbxIUnbJP1ptwpvlmUKIiKm1jHoJQ0BDwLXAaPAjZJGWw67Fdht+zLgauCrRY/YE+4A9nSl4jayTEFExNTKXNGvBMZs77d9DHgCWN1yjIEPSRJwDvAj4DiApIXArwIPd63qFlmmICJiamWCfgQ42LQ9Xuxr9gBwMXAI2AncYfudYuwPgN8B3uEkJK2TVJdUn5ycLFP7u7JMQUTE1MoEvdrsc8v2tcB24ALgcuABSedK+izwhu2tnV7E9kbbNdu14eHhEmX9zPUrRtiw5hJG5s9DwMj8eWxYc0luxEZEUO6pm3FgUdP2QhpX7s3WAvfaNjAm6QBwEXAV8DlJnwHOBM6V9E3bnz/90t8ryxRERLRX5op+C7BU0pLiBusNwDMtx7wGXAMg6XxgGbDf9t22F9peXJz33EyEfERETK3jFb3t45JuAzYDQ8AjtndJuqUYfwj4MvCopJ00pnq+ZPvNGaw7IiJKUmO2pb/UajXX6/VelxERMTAkbbVdazeWtW4iIiouQR8RUXF9OXUjaRL4617X0cZ5wKDee0jtvZHaZ9+g1g2nV/uFtts+m96XQd+vJNWnmgPrd6m9N1L77BvUumHmas/UTURExSXoIyIqLkF/ajb2uoDTkNp7I7XPvkGtG2ao9szRR0RUXK7oIyIqLkEfEVFxCfqSJM2X9KSkvZL2SPp4r2sqQ9KdRXvHVyU9LunMXtd0MpIekfSGpFeb9v28pD+X9MPi+4Je1tjOFHXfV/x9eUXS05Lm97LGqbSrvWnstyVZ0nm9qK2TqWqXdHvR/nSXpN/vVX0nM8XfmcslvSBpe9GfY2U3XitBX979wPdsXwRcxgy2RuwWSSPAF4Ga7eU0FqW7obdVdfQosKpl33rgWdtLgWeL7X7zKO+v+8+B5bYvBf4XcPdsF1XSo7y/diQtAn6Fxuq0/epRWmqX9Ms0uuBdavsfAl/pQV1lPMr7/9x/H/g925cD/67YPm0J+hIknQt8Evg6gO1jtg/3tqrS5gDzJM0BzuL9vQT6iu3nabSibLYa+MPi138IXD+rRZXQrm7b37d9vNh8gUYvh74zxZ85wH+k0R2ub5/YmKL236TRH+P/Fce8MeuFlTBF7QbOLX79Ybr085qgL+ejwCTwDUnbJD0s6exeF9WJ7QkaVzOvAa8DP7b9/d5WNS3n234doPj+kR7XMx3/GvhvvS6iLEmfAyZs7+h1LdPwMeCfSHpR0n+X9I96XdAp+C3gPkkHafzsduV/gQn6cuYAVwBfs70C+An9OX3wHsVc9mpgCY02j2dLSuOXWSbpd4HjwLd6XUsZks4CfpfG1MEgmgMsAK4E7gK+LaldS9R+9JvAnbYXAXdSzCKcrgR9OePAuO0Xi+0naQR/v/s0cMD2pO23gKeAT/S4pun435J+AaD43pf/FW9H0m8AnwV+3YPzoZVfpHFxsEPSX9GYcnpZ0j/oaVXljQNPueEl4B0ai4UNgt+g8XMK8EdAbsbOFtt/AxyUtKzYdQ2wu4cllfUacKWks4ormmsYgJvIbTxD4weA4vuf9LCW0iStAr4EfM72T3tdT1m2d9r+iO3FRRvQceCK4udgEGwCPgUg6WPAXAZnNctDwD8tfv0p4Idd+V1t56vEF3A5UAdeofEXaUGvaypZ9+8Be4FXgf8C/Fyva+pQ7+M07ie8RSNgbgL+Po2nbX5YfP/5XtdZsu4x4CCwvfh6qNd1lq29ZfyvgPN6Xecp/LnPBb5Z/J1/GfhUr+s8hdr/MbAV2AG8CPxSN14rSyBERFRcpm4iIiouQR8RUXEJ+oiIikvQR0RUXII+IqLiEvQRERWXoI+IqLj/DzT4u0q1AdNnAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(Y,X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.optimize import curve_fit\n",
    "def F(x,a,b):\n",
    "    return a*x+b\n",
    "\n",
    "params = curve_fit(F, xdata=Y, ydata=X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 13027.72090425, 774762.85246468]),\n",
       " array([[  149783.38253452, -1722508.90516021],\n",
       "        [-1722508.90516021, 22242832.44913608]]))"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "z=[i for i in range(20)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "can't multiply sequence by non-int of type 'numpy.float64'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-139-51ab28e4d22b>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mY\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mz\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mz\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mparams\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m**\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mparams\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mz\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mparams\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m: can't multiply sequence by non-int of type 'numpy.float64'"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEFCAYAAADzHRw3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXwV9b3/8dcHwr6EJewQQoCwbyEsoigoUkSr1ZaqdUNRtPZ6FZdWa3ut91YvVVzqhiIiVdS6XKu1ioDI5sIuyBpICJCwBQhLFrJ/f3/k2B9NCVnOSeack/fz8fDhycwc5s1J5s1klu+Ycw4REQl9dbwOICIigaFCFxEJEyp0EZEwoUIXEQkTKnQRkTARUZMri4qKcjExMTW5ShGRkLdu3bojzrk25S1Xo4UeExPD2rVra3KVIiIhz8z2VGQ5HXIREQkTKnQRkTChQhcRCRMqdBGRMKFCFxEJE+UWupnNMbN0M9t82rTBZrbSzDaY2VozG169MUVEpDwV2UOfC0woNe0J4FHn3GDgv3xfi4iIh8otdOfcciCj9GSgue91JLA/wLlERMJCbkERf/j7FvYdP1Xt66rqMfR7gCfNLBWYATxU1oJmNtV3WGbt4cOHq7g6EZHQ9JdvdjP3m92kZuRU+7qqWui/BKY557oA04DXylrQOTfLOZfgnEto06bcO1dFRMLGydwCZi5L5vy4NoyMbV3t66tqod8EfOh7/T6gk6IiIqW8unwXx3MK+PWPetXI+qpa6PuBC3yvLwR2BiaOiEh4OJyZx2tfpXDpwA707xRZI+ssd3AuM3sHGANEmVka8AhwG/BnM4sAcoGp1RlSRCTUvLgkibzCYu67OK7G1lluoTvnri1j1tAAZxERCQtpx3J4e9VeJg3tTGybpjW2Xt0pKiISYM9+sRMM7h7Xs0bXq0IXEQmgnYcy+XB9GjeO7EqHyEY1um4VuohIAM1YmEjj+hHcObZHja9bhS4iEiAbUo+zYMshbhsdS6sm9Wt8/Sp0EZEAeXLBdlo3qc+U0d08Wb8KXUQkAL5OOsLXSUe5c2wPmjao0cc1/5MKXUTET845nliQSMfIhlw3ItqzHCp0ERE/LdhyiI2px7lnXBwN69X1LIcKXUTED0XFjhkLE+nepglXxXfyNIsKXUTED88t3klSehb3j+9FRF1vK1WFLiJSRYu2HuLPi3dyVXwnJvRv73UcFbqISFUkpWcx7d0NDOgUyeNXDsDMvI6kQhcRqazM3AJuf3Mt9SPq8PINQz09EXo6by6WFBEJUcXFjnvf28juoznMmzKCTi1qdryWs9EeuohIJbywJIlFWw/x8MQ+nNO9+h8rVxkqdBGRClq87RDPfLGDK4d04uZzY7yO829U6CIiFbDrcBb3/HUDfTs0D5qToKWp0EVEypGVV8jtb64joq7xyg1DaVQ/OE6ClqaToiIiZ1Fc7LjvvQ3sOpLNm7cMp3PLxl5HKpP20EVEzmLmsmQWbDnEQ5f0ZlSPKK/jnJUKXUSkDEsS05mxMJErBndkynnejHFeGeUWupnNMbN0M9tcavpdZpZoZlvM7InqiygiUvN2H8nm7ne+o0/75ky/amBQngQtrSJ76HOBCadPMLOxwBXAQOdcP2BG4KOJiHgjO6+QqW+upU6d4D4JWlq5he6cWw5klJr8S2C6cy7Pt0x6NWQTEalxzjke+GAjSelZvHBtPF1aBe9J0NKqegw9DhhtZqvMbJmZDQtkKBERLzjn+OOn2/hs00EevKQ35/UM7pOgpVX1ssUIoCUwEhgGvGdmsc45V3pBM5sKTAWIjvbu0UwiImdTXOz4/cebeWvVXiaPiuG20bFeR6q0qu6hpwEfuhKrgWLgjP+UOedmOecSnHMJbdq0qWpOEZFqU1hUzP0fbOStVXv55ZjuPPLjviFxErS0qhb6R8CFAGYWB9QHjgQqlIhITckvLObuv27gw/X7uH98HL+Z0DskyxwqcMjFzN4BxgBRZpYGPALMAeb4LmXMB2460+EWEZFglltQxK/eWs/i7en87tI+3BqCh1lOV26hO+euLWPW9QHOIiJSY3LyC5n6xjq+SjrCH3/Sn+tHdvU6kt80louI1DqZuQXcMncN6/Yc46lJg/jp0M5eRwoIFbqI1CrHc/K5cc5qtu4/yfPXxnPpwA5eRwoYFbqI1BpHsvK4fvYqdh3O5uXrhzKubzuvIwWUCl1EaoWDJ3L5xeyV7D9+itcmJzC6Z/hdRq1CF5Gwl5qRw3WzV5GRnc8bt4xgeLdWXkeqFip0EQlruw5ncd3sVeTkFzHv1hEM7tLC60jVRoUuImEr8WAm181ehXOOv04dSZ8Ozb2OVK1U6CISljalneDGOauoH1GHt249hx5tm3odqdqp0EUk7Kzbk8HkOWto3qgeb982gq6tm3gdqUao0EUkbBQWFTNv5R6eWJBIu+YNeevWEXRs0cjrWDVGhS4iYeGbpCM8+slWEg9lcl6PKJ6+ehBtmzX0OlaNUqGLSEhLzcjh8c+2MX/zQbq0asQrNwxlfN92ITtioj9U6CISkk7lFzFzWTKvLEumjhn3j4/j1tGxNKwXGs//rA4qdBEJKc45Pt10gMc/3cb+E7lcPqgjD03sTYfI2nOsvCwqdBEJGVv3n+TRT7awKiWDvh2a8+w1Q8L2rs+qUKGLSNA7lp3PU4sSeXvVXiIb1eOxK/tzzbBo6tapfcfJz0aFLiJBq7ComHdW72XGwh1k5RVy4zkxTBsXR2Tjel5HC0oqdBEJSt8mH+XRT7aw/WAmo7q35pEf96NX+2ZexwpqKnQRCSppx3L438+28+mmA3Ru2YiXr4/nR/3a18rLECtLhS4iQeFUfhGvLE9m5tJkzODei+OYen7tvgyxslToIuIp5xzzNx/ksU+3se/4KS4b2IGHJvahUy26ZT9QVOgi4pntB0/yh79vYeWuDPp0aM7TPx/EiNjWXscKWeUWupnNAS4D0p1z/UvNux94EmjjnDtSPRFFJNwcz8nn6UU7mLdyD80b1eOPP+nPtcN1GaK/KrKHPhd4AXjj9Ilm1gW4GNgb+FgiEo6Kih1vr97LUwsTOXmqgBtGdmXaxXG0aFzf62hhodxCd84tN7OYM8x6Bvg18HGAM4lIGPpq5xEe+2wb2w6c5JzY1jxyeV96tw/vJwjVtCodQzezy4F9zrmN5V1KZGZTgakA0dHRVVmdiISwLftPMH3+dlbsPEKnFo146bp4LumvyxCrQ6UL3cwaAw8D4yuyvHNuFjALICEhwVV2fSISmlIzcnhqYSIfbdhPi8b1+N2lfbjhnK40iNBliNWlKnvo3YFuwA97552B9WY23Dl3MJDhRCT0HMvO54UlSbz57R7M4M4x3bn9gu5ENtLt+tWt0oXunNsEtP3hazPbDSToKheR2i23oIg5X6cwc2ky2XmFTBrahWkXx9E+snY9NchLFbls8R1gDBBlZmnAI86516o7mIiEhqJixwfrUnlm0U4OnsxlXJ+2/HpCb+LaadyVmlaRq1yuLWd+TMDSiEjIcM6xeFs6f/p8OzvTsxgS3YLnrtX45F7SnaIiUmnr9x5j+mfbWb07g9ioJhpAK0io0EWkwnYdzuLJBYnM33yQqKYN+ONP+nP1sC7Uq1vH62iCCl1EKiA9M5c/f7GTv65JpWFEHaaNi+PW0d1o0kAVEkz03RCRMmXlFTJr+S5mr9hFfmEx14+I5q6LehLVtIHX0eQMVOgi8m8KfI9+e27xTo5k5XPpwA48ML4XMVFNvI4mZ6FCF5F/cs7x6aYDzFiQyO6jOYyMbcVrN/VhUJcWXkeTClChiwhQ8gzP6fO3sTHtBL3aNeP1ycMY06uNrlwJISp0kVpu+8GT/Gn+dpYkHqZDZEOe/NlArorvrLHJQ5AKXaSW2nM0m+e/TOL/1qfRrEEED17Sm8mjYvQMzxCmQhepZTbvO8HMZcnM33SAiLp1uG10LHeO6a6HTIQBFbpILeCc4+uko7y8LJmvko7QrEEEU8/vzi3nxtC2uQbPChcqdJEwVlTsmL/5AK8s28WmfSdo06wBD17Sm1+MiKZ5Qw1nG25U6CJhKLegiA/WpfHqil3sOZpDbFQTpl81gCvjO+kBE2FMhS4SRk6cKmDeyj28/nUKR7LyGdSlBQ9d0puL+7bXVSu1gApdJAwcPJHLnK9TeHvVXrLyCjk/rg13XBDLObGtdR15LaJCFwlhSelZzFqezN++20dRseOygR25/YJY+nWM9DqaeECFLhKC1u89xstLk1m07RD169bh2uHR3DY6li6tGnsdTTykQhcJEc45liYeZuayZFanZBDZqB53je3BTaNiaK3RDwUVukjQKygq5h/f7+eVZbvYfjCTjpEN+f1lfblmWBeNRy7/Qj8NIkEqJ7+Qd9ekMntFCvuOn6Jn26Y8NWkQlw/uqCcEyRmp0EWCTEZ2Pn/5ZjdvfLubYzkFJHRtyaOX9+PC3m2po0sP5SzKLXQzmwNcBqQ75/r7pj0J/BjIB5KBm51zx6szqEi4SzuWw+wVKby7JpVTBUWM69OWOy7oTkJMK6+jSYioyB76XOAF4I3Tpi0CHnLOFZrZn4CHgN8EPp5I+Nt24CSvLEvmk+8PYMAVgztx+wWxxLVr5nU0CTHlFrpzbrmZxZSatvC0L1cCPwtsLJHwt2Z3Bi8tSWJJ4mEa16/L5FExTDmvGx1bNPI6moSoQBxDvwV4t6yZZjYVmAoQHR0dgNWJhLbUjBwe+3Qbn285SOsm9bnv4jhuOKerhq8Vv/lV6Gb2MFAIvFXWMs65WcAsgISEBOfP+kRCWXZeIS8tTeLVFSnUNeP+8XFMOS+WRvU1WJYERpUL3cxuouRk6UXOORW1SBmKix0fbdjH9PnbSc/M48ohnfjNhN60j9Q45BJYVSp0M5tAyUnQC5xzOYGNJBI+vtt7jEc/2cqG1OMM6hzJzOuHMrRrS69jSZiqyGWL7wBjgCgzSwMeoeSqlgbAIt9Ibiudc3dUY06RkHLoZC5/+nw7H67fR5tmDZgxaRBXDemk68ilWlXkKpdrzzD5tWrIIhLycguKeO2rFF5ckkRhkeOXY7rzq7E9aKpb9KUG6KdMJACccyzYcojHPttKasYpxvdtx8OX9qFr6yZeR5NaRIUu4qftB0/y359s5Zvko8S1a8q8KSM4r2eU17GkFlKhi1TRsex8nvliB/NW7qFZw3r89xX9+MXwaCI0cJZ4RIUuUkkFRcW8tXIPz3yxk6y8Qm4Y2ZV7xsXRsoluDBJvqdBFKuGrnUd49JMt7EzP4twerfmvy/rRq73GXJHgoEIXqYDdR7J57LNtLNp6iOhWjXnlhqGM79tOD2CWoKJCFzmLrLxCXvgyiTlfpRBR1/j1hF7ccm43GtbT7foSfFToImdQXOz4v/VpPLEgkcOZefw0vjO/ntCLds11u74ELxW6SCnr9hzj0U+28H3aCYZEt+DVGxMY3KWF17FEyqVCF/HZuv8kz36xg4VbD9GueQOeuXoQVwzS7foSOlToUuvtOJTJs1/s4LNNB2nWIIJ7xvXkttGxNNHt+hJi9BMrtVZSehZ/XryTf3y/n8b16nLXhT249bxYIhvX8zqaSJWo0KXW2X0km+cW7+SjDftoWK8ud1zQnamjY3VjkIQ8FbrUGnuP5vD8lzv58Lt91Ktr3Do6lqnnxxLVtIHX0UQCQoUuYS/tWA4vLkni/bVp1Klj3HRODHeMiaVtM12CKOFFhS5h68CJU7y4JIl316RiGNeNiObOsT10LbmELRW6hJ30k7m8tDSZt1ftxeH4eUIXfjW2Bx1bNPI6mki1UqFL2DicmcfLy5KZt3IPhcWOSUM786uxPejSqrHX0URqhApdQt7RrDxmLd/FX77dTX5hMVfFd+auC3voaUFS66jQJWQdy87n1RW7mPvNbk4VFPGTwZ2468IexLZp6nU0EU+o0CXknDhVwGsrdjHn691k5xdy2cCO3H1RD3q01bjkUrup0CVkZOYW8PrXu3l1xS4ycwuZOKA9d18UpwdMiPiUW+hmNge4DEh3zvX3TWsFvAvEALuBnzvnjlVfTKnNsvMKmftNSZEfzylgfN923DMujr4dm3sdTSSoVGQPfS7wAvDGadMeBBY756ab2YO+r38T+HhSm+XkF/Lmt3t4ZfkuMrLzuah3W+4ZF8eAzpFeRxMJSuUWunNuuZnFlJp8BTDG9/ovwFJU6BIguQVFvL1qLy8tTeJIVj4XxLVh2sVxGpNcpBxVPYbezjl3AMA5d8DM2pa1oJlNBaYCREdHV3F1UhsUFBXzwbo0nlu8kwMnchnVvTUvXx9HQkwrr6OJhIRqPynqnJsFzAJISEhw1b0+CT1FxY5PNu7nmS92sOdoDkOiW/DUpEGM6hHldTSRkFLVQj9kZh18e+cdgPRAhpLawTnHgi2HeHpRIjsOZdGnQ3NeuymBC3u3xUxPCRKprKoW+t+Bm4Dpvv9/HLBEEvaccyzfeYSnFibyfdoJYts04YVfDGFi/w563JuIHypy2eI7lJwAjTKzNOARSor8PTObAuwFJlVnSAkfq1MymLEgkdW7M+jcshFP/mwgVw7pRETdOl5HEwl5FbnK5doyZl0U4CwSxr5PO86MhTtYvuMwbZs14H+u6MfVw6KpH6EiFwkU3Skq1WrHoUyeWpjIgi2HaNm4Hr+d2JsbRsbQqH5dr6OJhB0VulSL3UeyefaLHXy8cT9N60cwbVwct5wXQ7OGegCzSHVRoUtA7T9+iue/3Ml7a9OoV9e4/fzu3H6+HsAsUhNU6BIQhzPzeGlpEm+t2gsObhjZlTvHdtdzO0VqkApd/HIip4BXlifz+te7yS8q5mfxnbnroh50bqmnBInUNBW6VElWXiGvf5XCrBW7yMor5McDO3LPuJ56uISIh1ToUim5BUXMW7mHl5Ymk5Gdz8V923Hf+Dh6t9dQtiJeU6FLheQXFvPe2lRe+DKJgydzGd0zivvG99IIiCJBRIUuZ1VU7Ph4wz6e+WIHqRmnSOjakmevGczI2NZeRxORUlTockbFxY7Ptxzk6UU7SErPol/H5rx+c3/GxLXRwFkiQUqFLv/COceSxHSeWriDLftP0rNtU2ZeF8+E/u1V5CJBToUu//RN8hFmLEhk/d7jRLdqzDNXD+LyQZ2oqxEQRUKCCl1Yv/cYTy1M5Ouko3SIbMjjVw5gUkJn6mkERJGQokKvxbbsP8HTC3eweHs6rZvU5/eX9eW6EdE0rKeBs0RCkQq9FkpKz+KZRTv4dNMBmjeM4IEf9WLyqBiaNNCPg0go0xZci6Rm5PDsFzv523dpNKpXl/+8sAdTRscS2UgjIIqEAxV6LXDwRC7Pf7mTd9ekUreOMeW8btxxQXdaN23gdTQRCSAVehg7mpXHzKXJvLlyD8XOcc3wLvzH2J60j9QIiCLhSIUehk6cKuDV5buY83UKuQVFXBXfmbsv6kmXVhoBUSScqdDDSHZeIXO/2c0ry5I5mVvIpQM7MG1cHD3aagREkdpAhR4GfhgBcebSZI5m53NR77bcOz6Ofh0jvY4mIjXIr0I3s2nArYADNgE3O+dyAxFMyldQVDIC4vOLS0ZAPLdHa+4b34v46JZeRxMRD1S50M2sE/CfQF/n3Ckzew+4BpgboGxShqJix0ff7ePPi3eyNyOH+OgWPH31IEZ1j/I6moh4yN9DLhFAIzMrABoD+/2PJGVxzvH55oM8dfoIiJOHMaaXRkAUET8K3Tm3z8xmAHuBU8BC59zC0suZ2VRgKkB0dHRVV1frrd2dwWOfbeO7vcfp0bYpL10Xz4R+7amjgbNExMefQy4tgSuAbsBx4H0zu945N+/05Zxzs4BZAAkJCc6PrLVS8uEsnvh8Owu2HKJtswb86acD+Gl8ZyI0cJaIlOLPIZdxQIpz7jCAmX0IjALmnfVdUiGHM/N4bvFO3l69l4YRdbjv4jimjO5G4/q6MElEzsyfdtgLjDSzxpQccrkIWBuQVLVYTn4hs1ek8MqyZHILi/nF8GjuHteTKN2mLyLl8OcY+ioz+wBYDxQC3+E7tCKVV1hUzAfr0nh60Q7SM/OY0K89D0zoRfc2uilIRCrGr9/fnXOPAI8EKEut9MMj36bP386OQ1nER7fgpeviSYhp5XU0EQkxOiDroe/TjvO/n23n211HiWndWM/uFBG/qNA9kJqRw5MLEvn7xv20alKfRy/vxy9GROuRbyLiFxV6DTqek88LXybxxrd7qFMH/mNsD26/IJZmDfWACRHxnwq9BuQWFPHGt7t54cskMvMKmTS0M9MujqNDZCOvo4lIGFGhV6PiYsffN+7nyQWJ7Dt+ijG92vDgJb3p3b6519FEJAyp0KvJN0lHeHz+NjbvO0m/js154mcDObeHBs8SkeqjQg+wxIOZ/O/8bSxNPEynFo149urBXD6oo8ZcEZFqp0IPkIMncnl6USIfrEujaYMIfjuxNzeeE0PDenW9jiYitYQK3U9ZeYW8vDSZ2V/torgYbjm3G/9xYQ9aNK7vdTQRqWVU6H5YnZLBve9tIO3YKS4f1JEHftRLD2IWEc+o0Ksgv7CYZ77YwcvLkunSsjEf3HGObtUXEc+p0Ctpx6FM7vnrBrYeOMk1w7rwu8v60rSBPkYR8Z6aqIKKix1zv9nN9M+306xBBLNuGMr4fu29jiUi8k8q9Ao4cOIUD7z/PV8lHeGi3m2Z/tOBtGmm8clFJLio0Mvxycb9PPy3TRQUOR6/cgDXDu+i0RBFJCip0Mtw4lQB//XxZj7esJ/BXVrw7NWDiYlq4nUsEZEyqdDP4JukI9z3/kbSM/O49+I47hzTXQ9lFpGgp0I/TW5BETMWJDL7qxRio5rw4S9HMahLC69jiYhUiArdZ+v+k0x7dwOJhzK5YWRXfjuxD43q67Z9EQkdtb7Qi4ods1fs4qmFO4hsXI/Xbx7G2F5tvY4lIlJptbrQ047lcO97G1mdksGEfu15/KoBtGqiMVhEJDTVykJ3zvHh+n384e9bcMCMSYP4aXwnXY4oIiHNr0I3sxbAbKA/4IBbnHPfBiJYdTmWnc/DH23is00HGRbTkqd/PlgDaolIWPB3D/3PwOfOuZ+ZWX0gqJtx2Y7DPPD+Ro7l5PObCb2Zen4sdfXgCREJE1UudDNrDpwPTAZwzuUD+YGJFVjOOZ5bnMQzX+ygZ9umzJk8jP6dIr2OJSISUP7soccCh4HXzWwQsA642zmXffpCZjYVmAoQHR3tx+qqpqComN/9bTPvrk3lqvhOPH7lAD1FSETCkj+3P0YA8cBM59wQIBt4sPRCzrlZzrkE51xCmzZt/Fhd5WXnFXLbG2t5d20q/3lhD56aNEhlLiJhy5899DQgzTm3yvf1B5yh0L2SnpnLLXPXsO1AJtOvGsA1w2v+twMRkZpU5UJ3zh00s1Qz6+WcSwQuArYGLlrVJaVnMfn11RzNymf2jQmM7a0bhUQk/Pl7lctdwFu+K1x2ATf7H8k/a3ZncOtf1lKvrvHu7SMZ2FljsYhI7eBXoTvnNgAJAcrit/mbDnD3uxvo3KIRc28eTnTroL6KUkQkoMLmTtE5X6XwP59uJT66JbNvTKClbuEXkVom5Au9uNjx2GfbeO2rFCb0a8+z1wzWlSwiUiuFdKHnFhRx33sb+XTTASaPiuH3l/XVnZ8iUmuFbKEfz8nntjfWsmb3MX53aR+mnNdNg2uJSK0WkoWempHD5NdXk5pxiuevHcKPB3X0OpKIiOdCrtA37zvBzXPXkFdQxJtThjMitrXXkUREgkJIFfrSxHTufGs9LRvX5+1bR9CzXTOvI4mIBI2QKfT31qTy0N820atdM16/eRjtmjf0OpKISFAJiUJ/cUkSTy5IZHTPKGZeP5SmDUIitohIjQqJZuwW1YSfJ3TmsSsHUK+uPwNEioiEr5Ao9IkDOjBxQAevY4iIBDXt7oqIhAkVuohImFChi4iECRW6iEiYUKGLiIQJFbqISJhQoYuIhAkVuohImDDnXM2tzOwwsKcSb4kCjlRTnEAI5nzKVnXBnC+Ys0Fw5wvlbF2dc23K+0NqtNAry8zWOueC5iHUpQVzPmWrumDOF8zZILjz1YZsOuQiIhImVOgiImEi2At9ltcByhHM+ZSt6oI5XzBng+DOF/bZgvoYuoiIVFyw76GLiEgFqdBFRMKEZ4VuZhPMLNHMkszswTPMjzSzT8xso5ltMbObS82va2bfmdk/gimbmbUwsw/MbLuZbTOzc4Io2zTftM1m9o6ZBfzBrBXI19LM/mZm35vZajPrX9H3epXNzLqY2RLf93OLmd0d6Gz+5DttvpfbxNm+r15vE2fLVq3bhJnNMbN0M9tcxnwzs+d82b83s/iK/r3OyDlX4/8BdYFkIBaoD2wE+pZa5rfAn3yv2wAZQP3T5t8LvA38I5iyAX8BbvW9rg+0CIZsQCcgBWjkm/ceMNmDz+5J4BHf697A4oq+18NsHYB43+tmwI5AZvM3X5BsE2VmC4Jtoqzva01sE+cD8cDmMuZPBOYDBowEVlX073Wm/7zaQx8OJDnndjnn8oG/AleUWsYBzczMgKaUFFMhgJl1Bi4FZgdTNjNrTsk38DUA51y+c+54MGTzzYsAGplZBNAY2B/AbBXN1xdYDOCc2w7EmFm7Cr7Xk2zOuQPOufW+6ZnANkrKIJD8+eyCYZs4Y7Yg2SbK/Nyo5m3CObeckm2wLFcAb7gSK4EWZtaBKm4PXhV6JyD1tK/T+PcN5AWgDyUf8CbgbudcsW/es8CvgWICz59sscBh4HXfr76zzaxJMGRzzu0DZgB7gQPACefcwgBmq2i+jcBVAGY2HOgKdK7ge73K9k9mFgMMAVYFMFsg8nm9TZSVLRi2iTNmq6Ftojxl5a/S9uBVodsZppW+fvJHwAagIzAYeMHMmpvZZUC6c25dsGWj5F/7eGCmc24IkA0E8liwP59bS0r+he/mm9fEzK4PYLaK5psOtDSzDcBdwHeU/AZRkfd6la3kDzBrCvwfcI9z7mQAs/mVL0i2ibI+u2DYJsr63GpimyhPWfmrtD1E+B2natKALqd93Zl//1XnZmC6KzmglGRmKZQc/zoXuNzMJgINgeZmNs85F4/vZTkAAALBSURBVKhvhD/Z9gJpzrkf9t4+ILA/vP5k6wqkOOcOA5jZh8AoYF5N5vMV4c2+DEbJMcwUSn7dLe/v5lU2zKweJWX+lnPuwwDmCkS+a/B4myjn++rpNnGWbD+i+reJ8pSVv34Z088ukCcAKnGiIALYRcm/jD8c8O9XapmZwB98r9sB+4CoUsuMIfAngPzKBqwAevle/wF4MhiyASOALZRsYEbJiaq7PPjsWvD/TyDfRsnxwwq918NsBrwBPOvxNnHGfEGyTZSZLQi2ibK+r9W+TfjWE0PZJ0Uv5V9Piq72Z3uolh/OCv4lJ1JytUAy8LBv2h3AHb7XHYGFlBwH3gxcXxM/vP5mo+Qwx1rge+AjoGUQZXsU2O6b/ibQwIPP7hxgpy/Hh6d/Pmd6bzBkA86j5Nfd7yk5nLUBmBgs+YJkmzjb99XrbeJs2ap1mwDeoeT4fAEle+NTSmUz4EVf9k1Agj/bg279FxEJE7pTVEQkTKjQRUTChApdRCRMqNBFRMKECl1EpJqUNzjXGZb/uZlt9Q0Y9nal16erXEREqoeZnQ9kUXLte/9ylu1JyQBhFzrnjplZW+dcemXWpz10EZFq4s4wOJeZdTezz81snZmtMLPevlm3AS8654753lupMgcVuohITZtFyR2pQ4H7gZd80+OAODP72sxWmtmEyv7BXo3lIiJS6/gGeBsFvF8yrAwADXz/jwB6UnK3b2dghZn1d5UYbliFLiJSc+oAx51zg88wLw1Y6ZwrAFLMLJGSgl9TmT9cRERqgCsZ+THFzCbBPx9BN8g3+yNgrG96FCWHYHZV5s9XoYuIVBMzewf4FuhlZmlmNgW4DphiZhspGe3xhycRLQCOmtlWYAnwgHPuaKXWp8sWRUTCg/bQRUTChApdRCRMqNBFRMKECl1EJEyo0EVEwoQKXUQkTKjQRUTCxP8DNVmr7EjiTpMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(X,Y)\n",
    "plt.plot(z,z*params[0][0]**2+params[0][1]*z+params[0][2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Code checking if there are indeed 19 instances per city\n",
    "for v in filtered_data2['METROREG'].unique():\n",
    "    print(filtered_data2[filtered_data2['METROREG'] == v])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[0,0,0].count(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def treat_missing(k,t): #t is a threshold for the number of zeros (mising value we allow)\n",
    "    L1 = list(k.employed_persons)\n",
    "    L2 = list(k.gdp)\n",
    "    L3 = list(k.population) \n",
    "    \n",
    "    if L1.count(0)>t,\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "#need to check if there are zeros in subdataframe\n",
    "def treat_missing(k):\n",
    "    #nested helper function\n",
    "    #needed because .index() doesn't retrieve position for more than one value\n",
    "    #eg: for l=[0,0,2], l.index(0) returns 0 and not (0,1)\n",
    "    def get_zeros(k):\n",
    "        z=[]\n",
    "        for i in range(len(k)):\n",
    "            if k[i]==0:\n",
    "                z.append(i)\n",
    "        return z  \n",
    "    #turns the dataframe into lists\n",
    "    L1 = list(k.employed_persons)\n",
    "    L2 = list(k.gdp)\n",
    "    L3 = list(k.population) \n",
    "    z1=get_zeros(L1)\n",
    "    z2=get_zeros(L2)\n",
    "    z3=get_zeros(L3)\n",
    "    #for missing single values at either ends \n",
    "    \n",
    "    #employed_persons\n",
    "    if len(z1) == 1 and z1[0] == 0:\n",
    "        L1[0] = L1[1] - abs(L1[1]-L1[2])\n",
    "        k.employed_persons = L1\n",
    "    if len(z1) == 1 and z1[0] == 18:\n",
    "        L1[18] = L1[17] + abs(L1[17]-L1[16])\n",
    "        k.employed_persons = L1\n",
    "    \n",
    "    #gdp\n",
    "    if len(z2) == 1 and z2[0] == 0:\n",
    "        L2[0] = L2[1] - abs(L2[1]-L2[2])\n",
    "        k.gdp = L2\n",
    "    if len(z2) == 1 and z2[0] == 18:\n",
    "        L2[18] = L2[17] + abs(L2[17]-L2[16])\n",
    "        k.gdp = L2\n",
    "    \n",
    "    #population\n",
    "    if len(z3) == 1 and z3[0] == 0:\n",
    "        L3[0] = L3[1] - abs(L3[1]-L3[2])\n",
    "        k.population = L3\n",
    "    if len(z3) == 1 and z3[0] == 18:\n",
    "        L3[18] = L3[17] + abs(L3[17]-L3[16])\n",
    "        k.population = L3\n",
    "    #For missing values at ends (more than one)\n",
    "    #nested function\n",
    "    #[0,1]->True\n",
    "    #[0,2]->False\n",
    "    def evenly_spaced(l):\n",
    "        for i in range(len(l)-1):\n",
    "            if abs(l[i]-l[i+1])!=1:\n",
    "                return False\n",
    "        return True\n",
    "\n",
    "    #employed_persons\n",
    "    if (len(z1) >1) and evenly_spaced(z1) == True and z1[0] == 0:\n",
    "        for i in reversed(z1):\n",
    "            L1[i] = L1[i+1] - abs(L1[i+1] - L1[i+2])\n",
    "        k.employed_persons = L1\n",
    "    \n",
    "    #gdp\n",
    "    if (len(z2) >1) and evenly_spaced(z2) == True and z2[0] == 0:\n",
    "        for i in reversed(z2):\n",
    "            L2[i] = L2[i+1] - abs(L2[i+1] - L2[i+2])\n",
    "        k.gdp = L2\n",
    "        \n",
    "    #population\n",
    "    if (len(z3) >1) and evenly_spaced(z3) == True and z3[0] == 0:\n",
    "        for i in reversed(z3):\n",
    "            L3[i] = L3[i+1] - abs(L3[i+1] - L3[i+2])\n",
    "        k.population = L3\n",
    "        \n",
    "    #other end \n",
    "    #employed_persons\n",
    "    if (len(z1) >1) and evenly_spaced(z1) == True and z1[-1] == 18:\n",
    "        for i in z1:\n",
    "            L1[i] = L1[i-1] + abs(L1[i-1] - L1[i-2])\n",
    "        k.employed_persons = L1\n",
    "    #gdp\n",
    "    if (len(z2) >1) and evenly_spaced(z2) == True and z2[-1] == 18:\n",
    "        for i in z2:\n",
    "            L2[i] = L2[i-1] + abs(L2[i-1] - L2[i-2])\n",
    "        k.gdp = L2\n",
    "    #population\n",
    "    if (len(z3) >1) and evenly_spaced(z3) == True and z3[-1] == 18:\n",
    "        for i in z3:\n",
    "            L3[i] = L3[i-1] + abs(L3[i-1] - L3[i-2])\n",
    "        k.population = L3\n",
    "    #If values are missing in the middle of the dataframe: -> apply what Simon showed on excel\n",
    "    #employed_persons\n",
    "    if ((len(z1) >1) and evenly_spaced(z1) == True) and (z1[-1] != 18 and z1[0] != 0):\n",
    "        p = L1[z1[0]-1] - L1[z1[-1]+1]\n",
    "        add = p/(len(z1)+1)\n",
    "        for i in z1:\n",
    "            L1[i]=L1[i-1]+add\n",
    "        k.employed_persons=L1\n",
    "    #gdp\n",
    "    if ((len(z2) >1) and evenly_spaced(z2) == True) and (z2[-1] != 18 and z2[0] != 0):\n",
    "        print(L2)\n",
    "        print(z2)\n",
    "        p = L2[z2[0]-1] - L2[z2[-1]+1]\n",
    "        add = p/(len(z2)+1)\n",
    "        for i in z2:\n",
    "            L2[i]=L2[i-1]+add\n",
    "        k.gdp=L2\n",
    "    #population\n",
    "    if ((len(z3) >1) and evenly_spaced(z3) == True) and (z3[-1] != 18 and z3[0] != 0):\n",
    "        p = L3[z3[0]-1] - L3[z3[-1]+1]\n",
    "        add = p/(len(z3)+1)\n",
    "        for i in z3:\n",
    "            L3[i]=L3[i-1]+add\n",
    "        k.population=L3\n",
    "    #more complicated cases\n",
    "    #0xxx000\n",
    "    #000xxx0\n",
    "    #000xxx000\n",
    "    #3rd helper function for \n",
    "    def get_sep(l):\n",
    "        for i in range(len(l)-1):\n",
    "            if abs(l[i]-l[i+1])!=1:\n",
    "                t=(l[i],l[i+1])\n",
    "        return t\n",
    "    #0xxx000\n",
    "    if evenly_spaced(z1) == False and len(z1)>1 and z1[0]==0:\n",
    "        L1[0] = L1[1] - abs(L1[1]-L1[2])\n",
    "        p = L1[z1[0]-1] - L1[z1[-1]+1]\n",
    "        add = p/(len(z1))\n",
    "        for i in z1[get_sep(z1)[1]:]:\n",
    "            L1[i]=L1[i-1]+add\n",
    "        k.employed_persons=L1\n",
    "    \n",
    "    #gdp\n",
    "    if evenly_spaced(z2) == False and len(z2)>1 and z1[0]==0:\n",
    "        L2[0] = L2[1] - abs(L2[1]-L2[2])\n",
    "        p = L2[z2[0]-1] - L2[z2[-1]+1]\n",
    "        add = p/(len(z2))\n",
    "        for i in z2[get_sep(z2)[1]:]:\n",
    "            L2[i]=L2[i-1]+add\n",
    "        k.gdp=L2\n",
    "    \n",
    "    #population\n",
    "    if evenly_spaced(z3) == False and len(z3)>1 and z3[0]==0:\n",
    "        L3[0] = L3[1] - abs(L3[1]-L3[2])\n",
    "        p = L3[z3[0]-1] - L3[z3[-1]+1]\n",
    "        add = p/(len(z3))\n",
    "        for i in z3[get_sep(z3)[1]:]:\n",
    "            L3[i]=L3[i-1]+add\n",
    "        k.population=L3\n",
    "    \n",
    "    #000xxx0\n",
    "    if evenly_spaced(z1) == False and len(z1)>1 and z1[-1]==18:\n",
    "        L1[18] = L1[17] + abs(L1[17]-L1[16])\n",
    "        p = L1[z1[0]-1] - L1[z1[-1]+1]\n",
    "        add = p/(len(z1))\n",
    "        for i in z1[:get_sep(z1)[1]]:\n",
    "            L1[i]=L1[i-1]+add\n",
    "        k.employed_persons=L1\n",
    "    \n",
    "    #gdp\n",
    "    if evenly_spaced(z2) == False and len(z2)>1 and z1[-1]==18:\n",
    "        L2[0] = L2[1] - abs(L2[1]-L2[2])\n",
    "        p = L2[z2[0]-1] - L2[z2[-1]+1]\n",
    "        add = p/(len(z2))\n",
    "        for i in z2[get_sep(z2)[1]:]:\n",
    "            L2[i]=L2[i-1]+add\n",
    "        k.gdp=L2\n",
    "    \n",
    "    #population\n",
    "    if evenly_spaced(z3) == False and len(z3)>1 and z3[-1]==18:\n",
    "        L3[0] = L3[1] - abs(L3[1]-L3[2])\n",
    "        p = L3[z3[0]-1] - L3[z3[-1]+1]\n",
    "        add = p/(len(z3))\n",
    "        for i in z3[get_sep(z3)[1]:]:\n",
    "            L3[i]=L3[i-1]+add\n",
    "        k.population=L3\n",
    "    \n",
    "    \n",
    "        \n",
    "        \n",
    "        \n",
    "        \n",
    "        \n",
    "                \n",
    "        \n",
    "        \n",
    "    \n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_sep([1,2,4])[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_sep(l):\n",
    "    for i in range(len(l)-1):\n",
    "        if abs(l[i]-l[i+1])!=1:\n",
    "            t=(l[i],l[i+1])\n",
    "    return t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    " Yo=filtered_data2[filtered_data2['METROREG'] == 'Bergen']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "treat_missing(Yo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Yo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for v in filtered_data2['METROREG'].unique():\n",
    "    treat_missing(filtered_data2[filtered_data2['METROREG'] == v])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_data2[20:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_data3 = filtered_data2[(filtered_data2['TIME']>=2000) & (filtered_data2['population']>500000) & (filtered_data2['gdp']>40000)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_data3 = filtered_data3[filtered_data3['METROREG'].apply(lambda x: len(x) < 27)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import geopy\n",
    "from geopy.geocoders import Nominatim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "geolocator = Nominatim(user_agent=\"Master-Student\")\n",
    "def get_coord(k):\n",
    "    assert type(k)==str\n",
    "    return (geolocator.geocode(k).latitude,geolocator.geocode(k).longitude)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "latlong=[0 for i in range(filtered_data3.shape[0])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_data3['latitude'] = latlong\n",
    "filtered_data3['longitude'] = latlong"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_data3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [],
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "d={}\n",
    "for i in sorted(set(list(filtered_data3.METROREG))):\n",
    "    print(i)\n",
    "    d[i]=get_coord(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for i in range(len(list(d.keys()))):\n",
    "    for idx, row in filtered_data3.iterrows():\n",
    "            filtered_data3.loc[idx,'latitude'] = list(d.values())[i][0]\n",
    "            filtered_data3.loc[idx,'longitude'] = list(d.values())[i][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "d2=dict(filtered_data3['METROREG'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "d2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in d2.keys():\n",
    "    if d2[i]<10:\n",
    "        filtered_data3=filtered_data3[filtered_data3.METROREG!=i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_data3"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}