languages-new.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JjO3l4i3Ky68"
   },
   "source": [
    "## Create language data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import dependencies\n",
    "import pandas as pd\n",
    "import regex as re\n",
    "import numpy as np\n",
    "\n",
    "# Path\n",
    "path = \"data/glottolog/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26416\n"
     ]
    }
   ],
   "source": [
    "# Load in linguoid dataset\n",
    "df = pd.read_csv(path + 'languoid.csv', delimiter = ',', encoding = \"utf-8\")\n",
    "\n",
    "# Extract desired columns\n",
    "df = df[['id', 'name', 'iso639P3code', 'level', 'family_id', 'country_ids', 'latitude', 'longitude']]\n",
    "\n",
    "# Rename columns\n",
    "df.columns = ['glcode', 'name', 'iso', 'level', 'family', 'country', 'lat', 'lon']\n",
    "\n",
    "# From\n",
    "df = df.assign(source='glottolog')\n",
    "\n",
    "print(df.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21746\n"
     ]
    }
   ],
   "source": [
    "# Load in geodataset for macroarea\n",
    "df_geo = pd.read_csv(path + 'languages_and_dialects_geo.csv', delimiter=',', encoding=\"utf-8\")\n",
    "\n",
    "# Rename columns\n",
    "df_geo.columns = ['glcode', 'name', 'iso', 'level', 'macroarea', 'lat', 'lon']\n",
    "\n",
    "# From\n",
    "df_geo = df_geo.assign(source='glottolog')\n",
    "print(df_geo.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26416\n"
     ]
    }
   ],
   "source": [
    "# Merge\n",
    "df = pd.merge(df, df_geo, on=[\"name\", \"iso\", \"level\", \"id\", \"lat\", \"lon\", \"source\"], how='outer')\n",
    "\n",
    "# Drop duplicates\n",
    "df.drop_duplicates(subset=['id', 'name'], keep='first', inplace=True, ignore_index=True)\n",
    "print(df.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25590\n"
     ]
    }
   ],
   "source": [
    "# Drop sign languages, bookkeeping, unattested, artificial\n",
    "df = df[df['family'] != 'book1242']\n",
    "df = df[df['family'] != 'sign1238']\n",
    "df = df[df['family'] != 'unat1236']\n",
    "df = df[df['family'] != 'arti1236']\n",
    "\n",
    "print(df.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Manually name families?\n",
    "df['family'] = [re.sub('sino1245', \"Sino-Tibetan\", str(x)) for x in df['family']]\n",
    "df['family'] = [re.sub('drav1251', \"Dravidian\", str(x)) for x in df['family']]\n",
    "df['family'] = [re.sub('atla1278', \"Atlantic-Congo\", str(x)) for x in df['family']]\n",
    "df['family'] = [re.sub('aust1307', \"Austronesian\", str(x)) for x in df['family']]\n",
    "df['family'] = [re.sub('indo1319', \"Indo-European\", str(x)) for x in df['family']]\n",
    "df['family'] = [re.sub('afro1255', \"Afro-Asiatic\", str(x)) for x in df['family']]\n",
    "df['family'] = [re.sub('aust1305', \"Austroasiatic\", str(x)) for x in df['family']]\n",
    "df['family'] = [re.sub('nucl1709', \"Nuclear Trans New Guinea\", str(x)) for x in df['family']]\n",
    "df['family'] = [re.sub('pama1250', \"Pama-Nyungan\", str(x)) for x in df['family']]\n",
    "df['family'] = [re.sub('pama1250', \"Pama-Nyungan\", str(x)) for x in df['family']]\n",
    "df['family'] = [re.sub('tupi1275', \"Tupian\", str(x)) for x in df['family']]\n",
    "df['family'] = [re.sub('tuuu1241', \"Tuu\", str(x)) for x in df['family']]\n",
    "df['family'] = [re.sub('ural1272', \"Uralic\", str(x)) for x in df['family']]\n",
    "df['family'] = [re.sub('turk1311', \"Turkic\", str(x)) for x in df['family']]\n",
    "df['family'] = [re.sub('khoe1240', \"Khoe-Kwadi\", str(x)) for x in df['family']]\n",
    "df['family'] = [re.sub('lake1255', \"Lakes Plain\", str(x)) for x in df['family']]\n",
    "df['family'] = [re.sub('japo1237', \"Japonic\", str(x)) for x in df['family']]\n",
    "df['family'] = [re.sub('kxaa1236', \"Kxa\", str(x)) for x in df['family']]\n",
    "df['family'] = [re.sub('utoa1244', \"Uto-Aztecan\", str(x)) for x in df['family']]\n",
    "df['family'] = [re.sub('mong1349', \"Mongolic-Khitan\", str(x)) for x in df['family']]\n",
    "df['family'] = [re.sub('koia1260', \"Koiarian\", str(x)) for x in df['family']]\n",
    "df['family'] = [re.sub('zapa1251', \"Zaparoan\", str(x)) for x in df['family']]\n",
    "df['family'] = [re.sub('jara1244', \"Jarawa-Onge\", str(x)) for x in df['family']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop if no code\n",
    "# df = df[df['iso'].notna()]\n",
    "# print(df.shape)\n",
    "\n",
    "# Drop dialects\n",
    "# df = df[df['level'] != 'dialect']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>name</th>\n",
       "      <th>iso</th>\n",
       "      <th>level</th>\n",
       "      <th>family</th>\n",
       "      <th>macroarea</th>\n",
       "      <th>country</th>\n",
       "      <th>lat</th>\n",
       "      <th>lon</th>\n",
       "      <th>source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3adt1234</td>\n",
       "      <td>3Ad-Tekles</td>\n",
       "      <td>NaN</td>\n",
       "      <td>dialect</td>\n",
       "      <td>Afro-Asiatic</td>\n",
       "      <td>Africa</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>glottolog</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>aala1237</td>\n",
       "      <td>Aalawa</td>\n",
       "      <td>NaN</td>\n",
       "      <td>dialect</td>\n",
       "      <td>Austronesian</td>\n",
       "      <td>Papunesia</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>glottolog</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>aant1238</td>\n",
       "      <td>Aantantara</td>\n",
       "      <td>NaN</td>\n",
       "      <td>dialect</td>\n",
       "      <td>Nuclear Trans New Guinea</td>\n",
       "      <td>Papunesia</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>glottolog</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>aari1238</td>\n",
       "      <td>Aari-Gayil</td>\n",
       "      <td>aiz</td>\n",
       "      <td>family</td>\n",
       "      <td>sout2845</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>glottolog</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>aari1239</td>\n",
       "      <td>Aari</td>\n",
       "      <td>aiw</td>\n",
       "      <td>language</td>\n",
       "      <td>sout2845</td>\n",
       "      <td>Africa</td>\n",
       "      <td>ET</td>\n",
       "      <td>5.95034</td>\n",
       "      <td>36.5721</td>\n",
       "      <td>glottolog</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26411</th>\n",
       "      <td>zuti1239</td>\n",
       "      <td>Guajajára of Zutiua</td>\n",
       "      <td>NaN</td>\n",
       "      <td>dialect</td>\n",
       "      <td>Tupian</td>\n",
       "      <td>South America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>glottolog</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26412</th>\n",
       "      <td>zuwa1238</td>\n",
       "      <td>Zuwadza</td>\n",
       "      <td>NaN</td>\n",
       "      <td>dialect</td>\n",
       "      <td>Koiarian</td>\n",
       "      <td>Papunesia</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>glottolog</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26413</th>\n",
       "      <td>zwal1238</td>\n",
       "      <td>Zwall</td>\n",
       "      <td>NaN</td>\n",
       "      <td>dialect</td>\n",
       "      <td>Atlantic-Congo</td>\n",
       "      <td>Africa</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>glottolog</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26414</th>\n",
       "      <td>zyph1238</td>\n",
       "      <td>Zyphe</td>\n",
       "      <td>zyp</td>\n",
       "      <td>language</td>\n",
       "      <td>Sino-Tibetan</td>\n",
       "      <td>Eurasia</td>\n",
       "      <td>IN MM</td>\n",
       "      <td>22.52400</td>\n",
       "      <td>93.2640</td>\n",
       "      <td>glottolog</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26415</th>\n",
       "      <td>zyud1238</td>\n",
       "      <td>Zyuzdin</td>\n",
       "      <td>NaN</td>\n",
       "      <td>dialect</td>\n",
       "      <td>Uralic</td>\n",
       "      <td>Eurasia</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>glottolog</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>25590 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             id                 name  iso     level                    family  \\\n",
       "0      3adt1234           3Ad-Tekles  NaN   dialect              Afro-Asiatic   \n",
       "1      aala1237               Aalawa  NaN   dialect              Austronesian   \n",
       "2      aant1238           Aantantara  NaN   dialect  Nuclear Trans New Guinea   \n",
       "3      aari1238           Aari-Gayil  aiz    family                  sout2845   \n",
       "4      aari1239                 Aari  aiw  language                  sout2845   \n",
       "...         ...                  ...  ...       ...                       ...   \n",
       "26411  zuti1239  Guajajára of Zutiua  NaN   dialect                    Tupian   \n",
       "26412  zuwa1238              Zuwadza  NaN   dialect                  Koiarian   \n",
       "26413  zwal1238                Zwall  NaN   dialect            Atlantic-Congo   \n",
       "26414  zyph1238                Zyphe  zyp  language              Sino-Tibetan   \n",
       "26415  zyud1238              Zyuzdin  NaN   dialect                    Uralic   \n",
       "\n",
       "           macroarea country       lat      lon     source  \n",
       "0             Africa     NaN       NaN      NaN  glottolog  \n",
       "1          Papunesia     NaN       NaN      NaN  glottolog  \n",
       "2          Papunesia     NaN       NaN      NaN  glottolog  \n",
       "3                NaN     NaN       NaN      NaN  glottolog  \n",
       "4             Africa      ET   5.95034  36.5721  glottolog  \n",
       "...              ...     ...       ...      ...        ...  \n",
       "26411  South America     NaN       NaN      NaN  glottolog  \n",
       "26412      Papunesia     NaN       NaN      NaN  glottolog  \n",
       "26413         Africa     NaN       NaN      NaN  glottolog  \n",
       "26414        Eurasia   IN MM  22.52400  93.2640  glottolog  \n",
       "26415        Eurasia     NaN       NaN      NaN  glottolog  \n",
       "\n",
       "[25590 rows x 10 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reorder columns\n",
    "df = df[['id', 'name', 'iso', 'level', 'family', 'macroarea', 'country', 'lat', 'lon', 'source']]\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add coordinates by hand\n",
    "df.loc[df.name == 'Arabic', ['lat', 'lon']] = 27.96, 43.85 # Coordinates of Standard Arabic, OK\n",
    "df.loc[df.name == 'West Germanic', ['lat', 'lon']] = 48, 9\n",
    "df.loc[df.name == 'Semitic', ['lat', 'lon']] = 29, 40\n",
    "df.loc[df.name == 'Anglo-Norman', ['lat', 'lon']] = 51, -1\n",
    "df.loc[df.name == 'Romance', ['lat', 'lon']] = 44, 12\n",
    "df.loc[df.name == 'Medieval Latin', ['lat', 'lon']] = 46.2, 1.3 # Western Europe coordinates\n",
    "df.loc[df.name == 'Late Latin', ['lat', 'lon']] = 46.2, 1.3 # Western Europe coordinates\n",
    "df.loc[df.name == 'Germanic', ['lat', 'lon']] = 48.25, 10.60 #Middle High German\n",
    "df.loc[df.name == 'New Latin', ['lat', 'lon']] = 46.2, 1.3 # Western Europe coordinates\n",
    "df.loc[df.name == 'Slavic', ['lat', 'lon']] = 43.72, 22.84 #Church Slavic???\n",
    "df.loc[df.name == 'Brunei Malay', ['lat', 'lon']] = 4.53, 114.72 #Coordinates of Brunei, OK?\n",
    "df.loc[df.name == 'Fante', ['lat', 'lon']] = 6.35, -1.33 #Coordinates of hyperglot Akan, OK\n",
    "df.loc[df.name == 'Twi', ['lat', 'lon']] = 6.35, -1.33 #Coordinates of hyperglot Akan, OK\n",
    "df.loc[df.name == 'Mongolian', ['lat', 'lon']] = 48.32, 106.29 #Coordinates of hypoglot, Halh Mongolian (most spoken dialect) OK\n",
    "df.loc[df.name == 'Volapük', ['lat', 'lon']] = 49, 8.24 #Coordinates of Karlsrühe, Baden OK\n",
    "df.loc[df.name == 'Luxemburgish', ['lat', 'lon']] = 49.68, 6.15 #Coordinates of hyperglot, Moselle Franconian OK\n",
    "df.loc[df.name == 'Armenian', ['lat', 'lon']] = 49.68, 6.15 #Coordinates of hyperglot, Moselle Franconian OK\n",
    "df.loc[df.name == 'Dravidian', ['lat', 'lon']] = 11, 78 #Coordinates of Old Tamil\n",
    "df.loc[df.name == 'Aramaic', ['lat', 'lon']] =33.91, 42.19 # Coordinates of Imperial Aramaic (700-300 BCE)\n",
    "df.loc[df.name == 'Turkic', ['lat', 'lon']] =55.49, 47.16 # Coordinates of Chuvash, cognate of bors.\n",
    "df.loc[df.name == 'Iranian', ['lat', 'lon']] =39.70, 66.98 # Coordinates of Sogdian, cognate bors\n",
    "\n",
    "# Modify names\n",
    "df.loc[df.name == 'Old English (ca. 450-1100)', ['name']] = 'Old English'\n",
    "df.loc[df.name == 'Old French (842-ca. 1400)', ['name']] = 'Old French'\n",
    "df.loc[df.name == 'Jewish Babylonian Aramaic (ca. 200-1200 CE)', ['name']] = 'Jewish Babylonian Aramaic'\n",
    "# df.loc[df.name == 'Pahlavi', ['name']] = 'Middle Persian'\n",
    "# Late Middle Indo-Aryan in the new instead of Middle Indo Aryan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>name</th>\n",
       "      <th>iso</th>\n",
       "      <th>level</th>\n",
       "      <th>family</th>\n",
       "      <th>macroarea</th>\n",
       "      <th>country</th>\n",
       "      <th>lat</th>\n",
       "      <th>lon</th>\n",
       "      <th>source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gane1238</td>\n",
       "      <td>!Gã!ne</td>\n",
       "      <td>NaN</td>\n",
       "      <td>language</td>\n",
       "      <td>Tuu</td>\n",
       "      <td>Africa</td>\n",
       "      <td>ZA</td>\n",
       "      <td>-31.3200</td>\n",
       "      <td>28.7500</td>\n",
       "      <td>glottolog</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>kwii1241</td>\n",
       "      <td>!Ui</td>\n",
       "      <td>NaN</td>\n",
       "      <td>family</td>\n",
       "      <td>Tuu</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>glottolog</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>abda1238</td>\n",
       "      <td>'Abd Al-Kuri</td>\n",
       "      <td>NaN</td>\n",
       "      <td>dialect</td>\n",
       "      <td>Afro-Asiatic</td>\n",
       "      <td>Africa</td>\n",
       "      <td>YE</td>\n",
       "      <td>12.1959</td>\n",
       "      <td>52.2282</td>\n",
       "      <td>glottolog</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>aden1242</td>\n",
       "      <td>'Aden</td>\n",
       "      <td>NaN</td>\n",
       "      <td>dialect</td>\n",
       "      <td>Afro-Asiatic</td>\n",
       "      <td>Eurasia</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>glottolog</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>alga1234</td>\n",
       "      <td>'Algaden</td>\n",
       "      <td>NaN</td>\n",
       "      <td>dialect</td>\n",
       "      <td>Afro-Asiatic</td>\n",
       "      <td>Africa</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>glottolog</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25588</th>\n",
       "      <td>oita1237</td>\n",
       "      <td>Ōita-ben</td>\n",
       "      <td>NaN</td>\n",
       "      <td>dialect</td>\n",
       "      <td>Japonic</td>\n",
       "      <td>Eurasia</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>glottolog</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25589</th>\n",
       "      <td>omut1237</td>\n",
       "      <td>Ōmuta-ben</td>\n",
       "      <td>NaN</td>\n",
       "      <td>dialect</td>\n",
       "      <td>Japonic</td>\n",
       "      <td>Eurasia</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>glottolog</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25590</th>\n",
       "      <td>osak1237</td>\n",
       "      <td>Ōsaka</td>\n",
       "      <td>NaN</td>\n",
       "      <td>dialect</td>\n",
       "      <td>Japonic</td>\n",
       "      <td>Eurasia</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>glottolog</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25591</th>\n",
       "      <td>suma1275</td>\n",
       "      <td>Šumadija-Vojvodina</td>\n",
       "      <td>NaN</td>\n",
       "      <td>dialect</td>\n",
       "      <td>Indo-European</td>\n",
       "      <td>Eurasia</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>glottolog</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25592</th>\n",
       "      <td>huaa1249</td>\n",
       "      <td>ǂHua</td>\n",
       "      <td>NaN</td>\n",
       "      <td>dialect</td>\n",
       "      <td>Kxa</td>\n",
       "      <td>Africa</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>glottolog</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>25593 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             id                name  iso     level         family macroarea  \\\n",
       "0      gane1238             !Gã!ne  NaN  language            Tuu    Africa   \n",
       "1      kwii1241                 !Ui  NaN    family            Tuu       NaN   \n",
       "2      abda1238        'Abd Al-Kuri  NaN   dialect   Afro-Asiatic    Africa   \n",
       "3      aden1242               'Aden  NaN   dialect   Afro-Asiatic   Eurasia   \n",
       "4      alga1234            'Algaden  NaN   dialect   Afro-Asiatic    Africa   \n",
       "...         ...                 ...  ...       ...            ...       ...   \n",
       "25588  oita1237            Ōita-ben  NaN   dialect        Japonic   Eurasia   \n",
       "25589  omut1237           Ōmuta-ben  NaN   dialect        Japonic   Eurasia   \n",
       "25590  osak1237               Ōsaka  NaN   dialect        Japonic   Eurasia   \n",
       "25591  suma1275  Šumadija-Vojvodina  NaN   dialect  Indo-European   Eurasia   \n",
       "25592  huaa1249                ǂHua  NaN   dialect            Kxa    Africa   \n",
       "\n",
       "      country      lat      lon     source  \n",
       "0          ZA -31.3200  28.7500  glottolog  \n",
       "1         NaN      NaN      NaN  glottolog  \n",
       "2          YE  12.1959  52.2282  glottolog  \n",
       "3         NaN      NaN      NaN  glottolog  \n",
       "4         NaN      NaN      NaN  glottolog  \n",
       "...       ...      ...      ...        ...  \n",
       "25588     NaN      NaN      NaN  glottolog  \n",
       "25589     NaN      NaN      NaN  glottolog  \n",
       "25590     NaN      NaN      NaN  glottolog  \n",
       "25591     NaN      NaN      NaN  glottolog  \n",
       "25592     NaN      NaN      NaN  glottolog  \n",
       "\n",
       "[25593 rows x 10 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Add language\n",
    "df.loc[-1] = [np.NaN, \"Proto-Dravidian\", np.NaN, \"proto\", \"Dravidian\", \"Asia\", np.NaN, 11.00, 78.00, \"manual\"]  # adding a row\n",
    "df.index = df.index + 1  # shifting index\n",
    "df.loc[-1] = [np.NaN, \"Proto-Iranian\", np.NaN, \"proto\", \"Indo-European\", \"Asia\", np.NaN, 45, 65, \"manual\"]  # Loc of Andronovo culture\n",
    "df.index = df.index + 1  # shifting index\n",
    "df.loc[-1] = [np.NaN, \"Proto-Sino-Tibetan\", np.NaN, \"proto\", \"Sino-Tibetan\", \"Asia\", np.NaN, 25, 103, \"manual\"]  # Loc of Yunnan\n",
    "df.index = df.index + 1  # shifting index\n",
    "\n",
    "# Sort by alphabetical order\n",
    "df = df.sort_values(['name', 'family'])\n",
    "df.reset_index(inplace=True, drop=True)\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns\n",
    "df = df.rename(columns={\"name\": \"language\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "58KLauIoKy7I"
   },
   "outputs": [],
   "source": [
    "# Save\n",
    "path = \"data/languages/\"\n",
    "\n",
    "# Write data out\n",
    "languages = df.copy()\n",
    "languages.to_csv(path + 'languages.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13 (main, Aug 25 2022, 23:51:50) [MSC v.1916 64 bit (AMD64)]"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "18c4d285bf73f8d33795481156cb8e20c5329867a6af52097d4f49f8abbf9900"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}