scripts/1_2_py_MILESTONES_research.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rhVcWnJpXEYo"
   },
   "source": [
    "#  Milestone type of inscription exploration\n",
    "\n",
    "**Research idea & domain expertise:** Petra Hermankova, Aarhus University\n",
    "\n",
    "**Script & technical solution:** Vojtech Kase, Aarhus University\n",
    "\n",
    "Source: https://github.com/sdam-au/social_diversity/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "x-R0Vdcrn1fq",
    "outputId": "80166604-e1a0-4341-fe32-b73be0a04cdd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting sddk\n",
      "  Using cached sddk-2.8.2-py3-none-any.whl (11 kB)\n",
      "Collecting numpy\n",
      "  Downloading numpy-1.20.1-cp37-cp37m-manylinux2010_x86_64.whl (15.3 MB)\n",
      "\u001b[K     |████████████████████████████████| 15.3 MB 4.7 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting pyarrow\n",
      "  Downloading pyarrow-3.0.0-cp37-cp37m-manylinux2014_x86_64.whl (20.7 MB)\n",
      "\u001b[K     |████████████████████████████████| 20.7 MB 34.0 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting pandas\n",
      "  Downloading pandas-1.2.2-cp37-cp37m-manylinux1_x86_64.whl (9.9 MB)\n",
      "\u001b[K     |████████████████████████████████| 9.9 MB 36.1 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting requests\n",
      "  Downloading requests-2.25.1-py2.py3-none-any.whl (61 kB)\n",
      "\u001b[K     |████████████████████████████████| 61 kB 507 kB/s  eta 0:00:01\n",
      "\u001b[?25hCollecting matplotlib\n",
      "  Downloading matplotlib-3.3.4-cp37-cp37m-manylinux1_x86_64.whl (11.5 MB)\n",
      "\u001b[K     |████████████████████████████████| 11.5 MB 86.1 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting beautifulsoup4\n",
      "  Downloading beautifulsoup4-4.9.3-py3-none-any.whl (115 kB)\n",
      "\u001b[K     |████████████████████████████████| 115 kB 94.0 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting pytz>=2017.3\n",
      "  Downloading pytz-2021.1-py2.py3-none-any.whl (510 kB)\n",
      "\u001b[K     |████████████████████████████████| 510 kB 28.5 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting python-dateutil>=2.7.3\n",
      "  Downloading python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)\n",
      "\u001b[K     |████████████████████████████████| 227 kB 83.4 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting chardet<5,>=3.0.2\n",
      "  Downloading chardet-4.0.0-py2.py3-none-any.whl (178 kB)\n",
      "\u001b[K     |████████████████████████████████| 178 kB 90.6 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting idna<3,>=2.5\n",
      "  Downloading idna-2.10-py2.py3-none-any.whl (58 kB)\n",
      "\u001b[K     |████████████████████████████████| 58 kB 2.1 MB/s  eta 0:00:01\n",
      "\u001b[?25hCollecting certifi>=2017.4.17\n",
      "  Downloading certifi-2020.12.5-py2.py3-none-any.whl (147 kB)\n",
      "\u001b[K     |████████████████████████████████| 147 kB 36.2 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting urllib3<1.27,>=1.21.1\n",
      "  Downloading urllib3-1.26.3-py2.py3-none-any.whl (137 kB)\n",
      "\u001b[K     |████████████████████████████████| 137 kB 38.3 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting pillow>=6.2.0\n",
      "  Downloading Pillow-8.1.0-cp37-cp37m-manylinux1_x86_64.whl (2.2 MB)\n",
      "\u001b[K     |████████████████████████████████| 2.2 MB 35.9 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting kiwisolver>=1.0.1\n",
      "  Downloading kiwisolver-1.3.1-cp37-cp37m-manylinux1_x86_64.whl (1.1 MB)\n",
      "\u001b[K     |████████████████████████████████| 1.1 MB 14.1 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting cycler>=0.10\n",
      "  Downloading cycler-0.10.0-py2.py3-none-any.whl (6.5 kB)\n",
      "Collecting pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.3\n",
      "  Downloading pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)\n",
      "\u001b[K     |████████████████████████████████| 67 kB 1.1 MB/s  eta 0:00:01\n",
      "\u001b[?25hCollecting soupsieve>1.2; python_version >= \"3.0\"\n",
      "  Downloading soupsieve-2.2-py3-none-any.whl (33 kB)\n",
      "Collecting six>=1.5\n",
      "  Downloading six-1.15.0-py2.py3-none-any.whl (10 kB)\n",
      "Installing collected packages: numpy, pyarrow, pytz, six, python-dateutil, pandas, chardet, idna, certifi, urllib3, requests, pillow, kiwisolver, cycler, pyparsing, matplotlib, soupsieve, beautifulsoup4, sddk\n",
      "\u001b[31mERROR: After October 2020 you may experience errors when installing or updating packages. This is because pip will change the way that it resolves dependency conflicts.\n",
      "\n",
      "We recommend you use --use-feature=2020-resolver to test your packages with the new resolver before it becomes the default.\n",
      "\n",
      "rpy2 3.1.0 requires pytest, which is not installed.\u001b[0m\n",
      "Successfully installed beautifulsoup4-4.9.3 certifi-2020.12.5 chardet-4.0.0 cycler-0.10.0 idna-2.10 kiwisolver-1.3.1 matplotlib-3.3.4 numpy-1.20.1 pandas-1.2.2 pillow-8.1.0 pyarrow-3.0.0 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2021.1 requests-2.25.1 sddk-2.8.2 six-1.15.0 soupsieve-2.2 urllib3-1.26.3\n"
     ]
    }
   ],
   "source": [
    "### REQUIREMENTS - will install the libraries\n",
    "import numpy as np\n",
    "import math\n",
    "import pandas as pd\n",
    "import sys\n",
    "import requests\n",
    "from urllib.request import urlopen \n",
    "from bs4 import BeautifulSoup\n",
    "import io\n",
    "\n",
    "# to avoid errors, we sometime use time.sleep(N) before retrying a request\n",
    "import time\n",
    "\n",
    "# the input data have typically a json structure\n",
    "import json\n",
    "import getpass\n",
    "\n",
    "import datetime as dt\n",
    "\n",
    "#!pip install sddk ### our own package under construction, always install to have an up-to-date version\n",
    "!pip install --ignore-installed sddk\n",
    "import sddk\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "2cK0r3WVvOH3"
   },
   "source": [
    "## Establishing connection to the Sciencedata.dk, connecting GoogleSheets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "8uTwTHVboJdw",
    "outputId": "c442da59-a2ff-4397-8af3-e6691eec6beb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sciencedata.dk username (format '123456@au.dk'): 648560@au.dk\n",
      "sciencedata.dk password: ··········\n",
      "connection with shared folder established with you as its ordinary user\n",
      "endpoint variable has been configured to: https://sciencedata.dk/sharingout/648597%40au.dk/SDAM_root/\n"
     ]
    }
   ],
   "source": [
    "# to access gsheet, you need Google Service Account key json file\n",
    "# I have mine located in my personal space on sciencedata.dk, so I read it from there:\n",
    "conf = sddk.configure()\n",
    "\n",
    "# (1) read the file and parse its content\n",
    "file_data = conf[0].get(conf[1] + \"ServiceAccountsKey.json\").json()\n",
    "# (2) transform the content into crendentials object\n",
    "credentials = service_account.Credentials.from_service_account_info(file_data)\n",
    "# (3) specify your usage of the credentials\n",
    "scoped_credentials = credentials.with_scopes(['https://spreadsheets.google.com/feeds', 'https://www.googleapis.com/auth/drive'])\n",
    "# (4) use the constrained credentials for authentication of gspread package\n",
    "gc = gspread.Client(auth=scoped_credentials)\n",
    "# (5) establish connection with spreadsheets specified by their url\n",
    "terms = gc.open_by_url(\"https://docs.google.com/spreadsheets/d/1tdtjPCoHY61FSZB0CxAdZXN9xDgl76KU-ObMp4uNG2A/edit#gid=0\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "pUU9syAHQEgs"
   },
   "source": [
    "## Connecting to the preprocessed and enriched JSON files from sciencedata.dk\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 578
    },
    "colab_type": "code",
    "id": "t7-hpGOCvBUM",
    "outputId": "5793d302-423b-4142-9825-e93395a5a16d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "reading file located in a public folder\n",
      "reading file located in a public folder\n"
     ]
    }
   ],
   "source": [
    "# read datasets in public folders\n",
    "\n",
    "publicfolder = \"66cbabddae0e02c6ae6c15be9746990c/\"\n",
    "\n",
    "EDH = sddk.read_file(\"EDH_terms_2021-02-26.json\", \"df\", publicfolder)\n",
    "EDCS = sddk.read_file(\"EDCS_terms_2021-02-26.json\", \"df\", publicfolder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "iPyRxbu17mpZ",
    "outputId": "a10fc526-af67-41a6-dba8-170d781592ad"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "81476"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(EDH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "rwPxDL6MRO6u",
    "outputId": "8556af58-f65b-4abc-d22d-d555dcd11ae6"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(81476, 76)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Inspect how many rows and columns we have\n",
    "EDH.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "EU_VoZxXa3Hv"
   },
   "source": [
    "## Subsetting the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 272
    },
    "colab_type": "code",
    "id": "5AICYwYMRSsh",
    "outputId": "56cec3f4-5095-4c35-cdc3-1712f67d7786"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['epitaph', 'honorific inscription', 'votive inscription',\n",
       "       'defixio', 'owner/artist inscription', 'owner/artist inscription?',\n",
       "       'mile-/leaguestone', 'acclamation', 'boundary inscription',\n",
       "       'building/dedicatory inscription', None, 'votive inscription?',\n",
       "       'military diploma', 'building/dedicatory inscription?', 'epitaph?',\n",
       "       'honorific inscription?', 'identification inscription',\n",
       "       'public legal inscription', 'private legal inscription',\n",
       "       'boundary inscription?', 'label', 'label?', 'list',\n",
       "       'private legal inscription?', 'calendar',\n",
       "       'identification inscription?', 'list?', 'seat inscription',\n",
       "       'elogium', 'assignation inscription', 'seat inscription?',\n",
       "       'elogium?', 'prayer', 'acclamation?', 'defixio?', 'calendar?',\n",
       "       'letter', 'mile-/leaguestone?', 'adnuntiatio',\n",
       "       'public legal inscription?', 'prayer?', 'letter?',\n",
       "       'assignation inscription?', 'military diploma?'], dtype=object)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Inspect all unique values within \"type_of_inscription\"\n",
    "EDH[\"type_of_inscription\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "fdCSK6FuSATJ",
    "outputId": "9c0e934e-d1fa-496d-e9b0-6861839a4a09"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1730"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Example how to subset the dataset, this time based on a specific string in the type of inscription\n",
    "EDH_miles = EDH[EDH[\"type_of_inscription\"].str.startswith(\"mile-/lea\", na=False)]\n",
    "len(EDH_miles) ### shows how many records in the dataset fulfils the condition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 296
    },
    "colab_type": "code",
    "id": "wU0PH9TgSfVd",
    "outputId": "5c945243-a315-4e0e-a16c-54cdcfddbbe2"
   },
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'NotebookFormatter' object has no attribute 'get_result'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/IPython/core/formatters.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    343\u001b[0m             \u001b[0mmethod\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_real_method\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprint_method\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    344\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mmethod\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 345\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    346\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    347\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m_repr_html_\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    796\u001b[0m         \u001b[0mmax_cols\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_option\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"display.max_columns\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    797\u001b[0m         \u001b[0mmax_colwidth\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_option\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"display.max_colwidth\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 798\u001b[0;31m         \u001b[0mshow_dimensions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_option\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"display.show_dimensions\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    799\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mget_option\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"display.expand_frame_repr\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    800\u001b[0m             \u001b[0mwidth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconsole\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_console_size\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/pandas/io/formats/format.py\u001b[0m in \u001b[0;36mto_html\u001b[0;34m(self, buf, encoding, classes, notebook, border)\u001b[0m\n\u001b[1;32m    986\u001b[0m         \u001b[0mnotebook\u001b[0m \u001b[0;34m:\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptional\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdefault\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    987\u001b[0m             \u001b[0mWhether\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mgenerated\u001b[0m \u001b[0mHTML\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mIPython\u001b[0m \u001b[0mNotebook\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 988\u001b[0;31m         \u001b[0mborder\u001b[0m \u001b[0;34m:\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    989\u001b[0m             \u001b[0mA\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mborder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mborder\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0mattribute\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mincluded\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mopening\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    990\u001b[0m             \u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m<\u001b[0m\u001b[0mtable\u001b[0m\u001b[0;34m>\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0mtag\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefault\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdisplay\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhtml\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mborder\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'NotebookFormatter' object has no attribute 'get_result'"
     ]
    },
    {
     "data": {
      "text/plain": [
       "    responsible_individual type_of_inscription letter_size not_after  \\\n",
       "23                 Feraudi   mile-/leaguestone     7-10 cm      0375   \n",
       "176                Feraudi   mile-/leaguestone        8 cm      0300   \n",
       "\n",
       "                                            literature  work_status    height  \\\n",
       "23   AE 1983, 0575.; L. Dos Santos - P. Le Roux - A...  provisional  (107) cm   \n",
       "176  AE 1983, 0572.; L. Dos Santos - P. Le Roux - A...  provisional   (22) cm   \n",
       "\n",
       "                                       diplomatic_text  \\\n",
       "23   D N / VALENTIN[ ] / VICTORI AC TRIVMPHATORI [ ...   \n",
       "176                             ]AV[ ] / [ ] VIII CON[   \n",
       "\n",
       "                                                people depth  ...  \\\n",
       "23   [{'name': 'Valentin[iano]', 'person_id': '1', ...  None  ...   \n",
       "176  [{'nomen': '[---]', 'cognomen': '[---]', 'gend...  None  ...   \n",
       "\n",
       "                      clean_text_interpretive_sentence  \\\n",
       "23   Domino nostro Valentiniano victori ac triumpha...   \n",
       "176                           AV imperator VIII consul   \n",
       "\n",
       "                            findspot year_of_find    present_location  \\\n",
       "23                              None         None  Mus. Pio XII Braga   \n",
       "176  Pfarrkirche, sekundär verwendet         None  Mus. Pio XII Braga   \n",
       "\n",
       "    religion geography social_economic_legal_history military terms terms_N  \n",
       "23      None      None                data available     None    []       0  \n",
       "176     None      None                          None     None    []       0  \n",
       "\n",
       "[2 rows x 76 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "EDH_miles.head(2) # shows the first (2) rows of the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "69IYFnBNTqWl",
    "outputId": "fffbee70-f912-40e2-bf62-a5dd8b212d9d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1726"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# how to show only the dated ones\n",
    "EDH_miles_date = EDH_miles[EDH_miles[\"origdate_text\"].str.startswith(\"\", na=False)]\n",
    "len(EDH_miles_date) ### how long it is?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9i-ETMSMwPr3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1730"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# with geolocations\n",
    "len(EDH_miles[EDH_miles[\"coordinates\"].notnull()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "Bi4UTq7mSSBv",
    "outputId": "e13793d4-38e2-4ebf-ff9c-d71e1718405e"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# selects only the milestones in the province Sardinia\n",
    "EDH_miles_sardinia = EDH_miles[EDH_miles[\"province_label\"].str.startswith(\"Sardinia\", na=False)]\n",
    "len(EDH_miles_sardinia)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "aFEVlXTEbATU"
   },
   "source": [
    "### Saving the subset as CSV file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "s743pomQA0F2"
   },
   "outputs": [],
   "source": [
    "# If you need to save the subset into a CSV and save it into a local computer\n",
    "from google.colab import files\n",
    "EDH_miles.to_csv('EDH_milestones.csv') \n",
    "files.download('EDH_milestones.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xHaarC5NUd5_"
   },
   "outputs": [],
   "source": [
    "# prints as CSV into a local computer\n",
    "from google.colab import files\n",
    "EDH_miles_sardinia.to_csv('EDH_milestones_sardinia.csv') \n",
    "files.download('EDH_milestones_sardinia.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "2f6_SrIVQKB-"
   },
   "source": [
    "## Inscriptions from one province (Example of Sardinia)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "6SrMnZtuQQhg"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Latium et Campania (Regio I)', 'Roma', 'Baetica', 'Britannia',\n",
       "       'Aemilia (Regio VIII)', 'Hispania citerior', 'unknown',\n",
       "       'Alpes Maritimae', 'Apulia et Calabria (Regio II)', 'Narbonensis',\n",
       "       'Lusitania', 'Africa Proconsularis', 'Samnium (Regio IV)',\n",
       "       'Etruria (Regio VII)', 'Raetia', 'Pannonia superior',\n",
       "       'Lugdunensis', 'Moesia inferior', 'Dalmatia', 'Belgica',\n",
       "       'Umbria (Regio VI)', 'Germania inferior', 'Germania superior',\n",
       "       'Dacia', 'Aquitania', 'Arabia', 'Mauretania Caesariensis',\n",
       "       'Noricum', 'Numidia', 'Pannonia inferior',\n",
       "       'Venetia et Histria (Regio X)', 'Barbaricum',\n",
       "       'Transpadana (Regio XI)', 'Sardinia', 'Aegyptus',\n",
       "       'Mauretania Tingitana', 'Asia', 'Syria', 'Bithynia et Pontus',\n",
       "       'Cyrene', 'Moesia superior', 'Macedonia',\n",
       "       'Bruttium et Lucania (Regio III)', 'Picenum (Regio V)', 'Epirus',\n",
       "       'Alpes Poeninae', 'Galatia', 'Liguria (Regio IX)',\n",
       "       'Sicilia, Melita', 'Iudaea', 'Corsica', 'Achaia',\n",
       "       'Lycia et Pamphylia', 'Cappadocia', 'Mesopotamia', 'Alpes Graiae',\n",
       "       'Regnum Bospori', 'Creta', 'Tuscia et Umbria', 'Cyprus', 'Thracia',\n",
       "       'Cilicia', 'Alpes Cottiae', 'Armenia', 'Macedonia, Epirus'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "EDH[\"province_label_clean\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "XPhhnAYKZZO7"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "228"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# subset based on the name of province \n",
    "EDH_sardinia = EDH[EDH[\"province_label\"].str.startswith(\"Sardinia\", na=False)]\n",
    "len(EDH_sardinia) ### how long it is?"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "2_py_MILESTONES_research.ipynb",
   "provenance": []
  },
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}