summarise-index-details.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Summarise index details\n",
    "\n",
    "This notebook counts the number of rows in each index and calculates the total for the whole repository. It formats the results in nice HTML and Markdown tables for easy browsing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from urllib.parse import urljoin\n",
    "\n",
    "import pandas as pd\n",
    "from IPython.display import HTML, display\n",
    "from tabulate import tabulate"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add links and totals to the list of indexes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Load the index data\n",
    "df = pd.read_csv(\"indexes.csv\").sort_values(by=\"title\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def make_download_link(url):\n",
    "    \"\"\"\n",
    "    Create a link to download the CSV file from GitHub\n",
    "    \"\"\"\n",
    "    slug = url.strip(\"/\").split(\"/\")[-1]\n",
    "    filename = f\"{slug}.csv\"\n",
    "    url = urljoin(\n",
    "        \"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/\",\n",
    "        filename,\n",
    "    )\n",
    "    link = '<a href=\"{}\">CSV file</a>'.format(url)\n",
    "    return link\n",
    "\n",
    "\n",
    "# Create a HTML link to the index data on the NSWSA site\n",
    "df[\"web\"] = df[\"url\"].apply(lambda x: '<a href=\"{}\">Browse index</a>'.format(x))\n",
    "\n",
    "# Create a HTML link to download the CSV file from GitHub\n",
    "df[\"download\"] = df[\"url\"].apply(lambda x: make_download_link(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def count_rows(url):\n",
    "    \"\"\"\n",
    "    Count the number of rows in a CSV file.\n",
    "    \"\"\"\n",
    "    slug = url.strip(\"/\").split(\"/\")[-1]\n",
    "    url = urljoin(\n",
    "        \"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/\",\n",
    "        f\"{slug}.csv\",\n",
    "    )\n",
    "    df = pd.read_csv(url, dtype=object)\n",
    "    return df.shape[0]\n",
    "\n",
    "\n",
    "# Add number of rows in the CSV\n",
    "df[\"rows\"] = df[\"url\"].apply(lambda x: count_rows(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2481881"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# How many rows in the whole repository?\n",
    "df[\"rows\"].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "title                           Deceased estates index 1880-1958\n",
       "url            https://mhnsw.au/indexes/deceased-estates/dece...\n",
       "description    Researching deceased estates files is a comple...\n",
       "category                                        Deceased estates\n",
       "web            <a href=\"https://mhnsw.au/indexes/deceased-est...\n",
       "download       <a href=\"https://media.githubusercontent.com/m...\n",
       "rows                                                      577891\n",
       "Name: 26, dtype: object"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Which index has the most number of rows?\n",
    "df.loc[df[\"rows\"].idxmax()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summarise the results of the harvest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Currently: 75 indexes harvested with 2,481,881 rows of data.'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"Currently: {} indexes harvested with {:,} rows of data.\".format(\n",
    "    df.shape[0], df[\"rows\"].sum()\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Make a nicely formatted table in both HTML and Markdown."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>Title                                                                   </th><th style=\"text-align: right;\">  Number of rows</th><th>Download data                                                                                                                                                 </th><th>View at State Archives                                                                                                                 </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>Aboriginal People in the Register of Aboriginal Reserves 1875-1904      </td><td style=\"text-align: right;\">              78</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/aboriginal-people-in-the-register-of-aboriginal-reserves.csv\">CSV file</a></td><td><a href=\"https://mhnsw.au/indexes/first-nations/aboriginal-people-in-the-register-of-aboriginal-reserves/\">Browse index</a>            </td></tr>\n",
       "<tr><td>Assisted Immigrants Index 1839-1896                                     </td><td style=\"text-align: right;\">         200,000</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/assisted-immigrants-index.csv\">CSV file</a>                               </td><td><a href=\"https://mhnsw.au/indexes/immigration-and-shipping/assisted-immigrants-index/\">Browse index</a>                                </td></tr>\n",
       "<tr><td>Australian Railway Supply Detachment 1914                               </td><td style=\"text-align: right;\">              65</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/australian-railway-supply-detachment.csv\">CSV file</a>                    </td><td><a href=\"https://mhnsw.au/indexes/wwi/australian-railway-supply-detachment/\">Browse index</a>                                          </td></tr>\n",
       "<tr><td>Bankruptcy index 1888-1929                                              </td><td style=\"text-align: right;\">          30,000</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/bankruptcy-index.csv\">CSV file</a>                                        </td><td><a href=\"https://mhnsw.au/indexes/bankruptcy-and-insolvency/bankruptcy-index/\">Browse index</a>                                        </td></tr>\n",
       "<tr><td>Bench of Magistrates Index 1788-1820                                    </td><td style=\"text-align: right;\">           4,442</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/bench-of-magistrates-index.csv\">CSV file</a>                              </td><td><a href=\"https://mhnsw.au/indexes/courts-lower/bench-of-magistrates-index/\">Browse index</a>                                           </td></tr>\n",
       "<tr><td>Botanic Gardens and government domains employees                        </td><td style=\"text-align: right;\">             916</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/botanic-gardens-and-government-domains-employees-index.csv\">CSV file</a>  </td><td><a href=\"https://mhnsw.au/indexes/professions-and-occupations/botanic-gardens-and-government-domains-employees-index/\">Browse index</a></td></tr>\n",
       "<tr><td>Bubonic plague index 1900-1908                                          </td><td style=\"text-align: right;\">             567</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/bubonic-plague-index.csv\">CSV file</a>                                    </td><td><a href=\"https://mhnsw.au/indexes/bubonic-plague/bubonic-plague-index/\">Browse index</a>                                               </td></tr>\n",
       "<tr><td>Census - 1841                                                           </td><td style=\"text-align: right;\">           9,355</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/census-1841.csv\">CSV file</a>                                             </td><td><a href=\"https://mhnsw.au/indexes/census-and-musters/census-1841/\">Browse index</a>                                                    </td></tr>\n",
       "<tr><td>Chemists, druggists and pharmacists index 1876-1920                     </td><td style=\"text-align: right;\">           2,967</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/chemists-druggists-and-pharmacists-index.csv\">CSV file</a>                </td><td><a href=\"https://mhnsw.au/indexes/professions-and-occupations/chemists-druggists-and-pharmacists-index/\">Browse index</a>              </td></tr>\n",
       "<tr><td>Child care and protection index 1817-1942                               </td><td style=\"text-align: right;\">          21,292</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/child-care-and-protection-index.csv\">CSV file</a>                         </td><td><a href=\"https://mhnsw.au/indexes/child-care-and-protection/child-care-and-protection-index/\">Browse index</a>                         </td></tr>\n",
       "<tr><td>Colonial (Government) Architect index 1837-1970                         </td><td style=\"text-align: right;\">           2,373</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/colonial-architect-index.csv\">CSV file</a>                                </td><td><a href=\"https://mhnsw.au/indexes/architecture-and-design/colonial-architect-index/\">Browse index</a>                                  </td></tr>\n",
       "<tr><td>Colonial Secretary Letters Received, 1826-1896                          </td><td style=\"text-align: right;\">         205,863</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/colonial-secretary-letters-received-1826-1896.csv\">CSV file</a>           </td><td><a href=\"https://mhnsw.au/indexes/colonial-secretary/colonial-secretary-letters-received-1826-1896/\">Browse index</a>                  </td></tr>\n",
       "<tr><td>Colonial Secretary's Papers 1788-1825                                   </td><td style=\"text-align: right;\">         144,572</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/colonial-secretarys-papers-1788-1825.csv\">CSV file</a>                    </td><td><a href=\"https://mhnsw.au/indexes/colonial-secretary/colonial-secretarys-papers-1788-1825/\">Browse index</a>                           </td></tr>\n",
       "<tr><td>Colonial Secretary's letters relating to land 1826-1856                 </td><td style=\"text-align: right;\">          20,000</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/colonial-secretarys-letters-re-land.csv\">CSV file</a>                     </td><td><a href=\"https://mhnsw.au/indexes/land/colonial-secretarys-letters-re-land/\">Browse index</a>                                          </td></tr>\n",
       "<tr><td>Colonial Secretary's main series of letters received                    </td><td style=\"text-align: right;\">           7,638</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/colonial-secretarys-main-series-of-letters-received.csv\">CSV file</a>     </td><td><a href=\"https://mhnsw.au/indexes/colonial-secretary/colonial-secretarys-main-series-of-letters-received/\">Browse index</a>            </td></tr>\n",
       "<tr><td>Convict assignments index 1821-1825                                     </td><td style=\"text-align: right;\">           6,156</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/convict-assignments-index.csv\">CSV file</a>                               </td><td><a href=\"https://mhnsw.au/indexes/convicts/convict-assignments-index/\">Browse index</a>                                                </td></tr>\n",
       "<tr><td>Convict exiles index 1849-1850                                          </td><td style=\"text-align: right;\">           3,004</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/convict-exiles-index.csv\">CSV file</a>                                    </td><td><a href=\"https://mhnsw.au/indexes/convicts/convict-exiles-index/\">Browse index</a>                                                     </td></tr>\n",
       "<tr><td>Convict indents (digitised) index 1788-1801                             </td><td style=\"text-align: right;\">          20,000</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/convict-indents-index.csv\">CSV file</a>                                   </td><td><a href=\"https://mhnsw.au/indexes/convicts/convict-indents-index/\">Browse index</a>                                                    </td></tr>\n",
       "<tr><td>Convicts applications to marry 1825-1851                                </td><td style=\"text-align: right;\">          14,327</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/convicts-applications-to-marry.csv\">CSV file</a>                          </td><td><a href=\"https://mhnsw.au/indexes/convicts/convicts-applications-to-marry/\">Browse index</a>                                           </td></tr>\n",
       "<tr><td>Convicts index 1791-1873                                                </td><td style=\"text-align: right;\">         150,000</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/convicts-index.csv\">CSV file</a>                                          </td><td><a href=\"https://mhnsw.au/indexes/convicts/convicts-index/\">Browse index</a>                                                           </td></tr>\n",
       "<tr><td>Coroners' inquests index 1796-1824                                      </td><td style=\"text-align: right;\">             808</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/coroners-inquests-index.csv\">CSV file</a>                                 </td><td><a href=\"https://mhnsw.au/indexes/inquests-and-coronial-inquiries/coroners-inquests-index/\">Browse index</a>                           </td></tr>\n",
       "<tr><td>Court of Civil Jurisdiction index 1799-1814                             </td><td style=\"text-align: right;\">           2,876</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/court-of-civil-jurisdiction-index.csv\">CSV file</a>                       </td><td><a href=\"https://mhnsw.au/indexes/courts-lower/court-of-civil-jurisdiction-index/\">Browse index</a>                                    </td></tr>\n",
       "<tr><td>Court of Claims (Land) index 1833-1922                                  </td><td style=\"text-align: right;\">           2,966</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/court-of-claims-index.csv\">CSV file</a>                                   </td><td><a href=\"https://mhnsw.au/indexes/land/court-of-claims-index/\">Browse index</a>                                                        </td></tr>\n",
       "<tr><td>Crew and passengers 1828-1841                                           </td><td style=\"text-align: right;\">           2,560</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/crew-and-passengers-index-1828-1841.csv\">CSV file</a>                     </td><td><a href=\"https://mhnsw.au/indexes/immigration-and-shipping/crew-and-passengers-index-1828-1841/\">Browse index</a>                      </td></tr>\n",
       "<tr><td>Criminal court records index 1788-1833                                  </td><td style=\"text-align: right;\">           5,028</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/criminal-court-records-index.csv\">CSV file</a>                            </td><td><a href=\"https://mhnsw.au/indexes/criminal-courts/criminal-court-records-index/\">Browse index</a>                                      </td></tr>\n",
       "<tr><td>Criminal depositions (Deposition Books) index 1849-1949                 </td><td style=\"text-align: right;\">         117,508</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/criminal-depositions-index.csv\">CSV file</a>                              </td><td><a href=\"https://mhnsw.au/indexes/criminal-courts/criminal-depositions-index/\">Browse index</a>                                        </td></tr>\n",
       "<tr><td>Criminal indictments index 1863-1919                                    </td><td style=\"text-align: right;\">          20,000</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/criminal-indictments-index.csv\">CSV file</a>                              </td><td><a href=\"https://mhnsw.au/indexes/criminal-courts/criminal-indictments-index/\">Browse index</a>                                        </td></tr>\n",
       "<tr><td>Deceased estates index 1880-1958                                        </td><td style=\"text-align: right;\">         577,891</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/deceased-estates-index.csv\">CSV file</a>                                  </td><td><a href=\"https://mhnsw.au/indexes/deceased-estates/deceased-estates-index/\">Browse index</a>                                           </td></tr>\n",
       "<tr><td>Depasturing licenses index 1837-1851                                    </td><td style=\"text-align: right;\">           7,449</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/depasturing-licenses-index.csv\">CSV file</a>                              </td><td><a href=\"https://mhnsw.au/indexes/land/depasturing-licenses-index/\">Browse index</a>                                                   </td></tr>\n",
       "<tr><td>Dependent children registers 1883-1923                                  </td><td style=\"text-align: right;\">          28,910</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/dependent-children-registers.csv\">CSV file</a>                            </td><td><a href=\"https://mhnsw.au/indexes/child-care-and-protection/dependent-children-registers/\">Browse index</a>                            </td></tr>\n",
       "<tr><td>Devonshire Street Cemetery reinterment index                            </td><td style=\"text-align: right;\">           9,559</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/devonshire-street-cemetery-reinterment-index.csv\">CSV file</a>            </td><td><a href=\"https://mhnsw.au/indexes/cemeteries-and-burials/devonshire-street-cemetery-reinterment-index/\">Browse index</a>               </td></tr>\n",
       "<tr><td>Divorce records index 1873-1923                                         </td><td style=\"text-align: right;\">          21,239</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/divorce-records-index.csv\">CSV file</a>                                   </td><td><a href=\"https://mhnsw.au/indexes/divorce/divorce-records-index/\">Browse index</a>                                                     </td></tr>\n",
       "<tr><td>Fire Commissioners Personnel                                            </td><td style=\"text-align: right;\">           3,767</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/fire-commissioners-personnel.csv\">CSV file</a>                            </td><td><a href=\"https://mhnsw.au/indexes/firefighters-fires-and-fire-brigades/fire-commissioners-personnel/\">Browse index</a>                 </td></tr>\n",
       "<tr><td>Gaol inmates & prisoners photos index 1870-1930                         </td><td style=\"text-align: right;\">          52,055</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/gaol-inmates-prisoners-photos-index.csv\">CSV file</a>                     </td><td><a href=\"https://mhnsw.au/indexes/gaol-inmates-and-prisoners/gaol-inmates-prisoners-photos-index/\">Browse index</a>                    </td></tr>\n",
       "<tr><td>Gold (auriferous) lease registers 1874-1953                             </td><td style=\"text-align: right;\">          60,000</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/gold-lease-registers.csv\">CSV file</a>                                    </td><td><a href=\"https://mhnsw.au/indexes/goldmining/gold-lease-registers/\">Browse index</a>                                                   </td></tr>\n",
       "<tr><td>Indigenous colonial court cases 1788-1838                               </td><td style=\"text-align: right;\">              65</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/indigenous-colonial-court-cases.csv\">CSV file</a>                         </td><td><a href=\"https://mhnsw.au/indexes/first-nations/indigenous-colonial-court-cases/\">Browse index</a>                                     </td></tr>\n",
       "<tr><td>Infirm & destitute (Government) asylums index 1880-1896                 </td><td style=\"text-align: right;\">          20,000</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/infirm-destitute-asylums-index.csv\">CSV file</a>                          </td><td><a href=\"https://mhnsw.au/indexes/asylums/infirm-destitute-asylums-index/\">Browse index</a>                                            </td></tr>\n",
       "<tr><td>Inquest index 1942-1963                                                 </td><td style=\"text-align: right;\">          45,547</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/inquest-index.csv\">CSV file</a>                                           </td><td><a href=\"https://mhnsw.au/indexes/inquests-and-coronial-inquiries/inquest-index/\">Browse index</a>                                     </td></tr>\n",
       "<tr><td>Insolvency index 1842-1887                                              </td><td style=\"text-align: right;\">          23,108</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/insolvency-index.csv\">CSV file</a>                                        </td><td><a href=\"https://mhnsw.au/indexes/bankruptcy-and-insolvency/insolvency-index/\">Browse index</a>                                        </td></tr>\n",
       "<tr><td>Intestate estates index 1821-1913                                       </td><td style=\"text-align: right;\">          30,000</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/intestate-estates-index.csv\">CSV file</a>                                 </td><td><a href=\"https://mhnsw.au/indexes/intestates/intestate-estates-index/\">Browse index</a>                                                </td></tr>\n",
       "<tr><td>Land grants and leases (registers) 1792-1865                            </td><td style=\"text-align: right;\">           5,627</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/land-grants-and-leases.csv\">CSV file</a>                                  </td><td><a href=\"https://mhnsw.au/indexes/land/land-grants-and-leases/\">Browse index</a>                                                       </td></tr>\n",
       "<tr><td>Letters re migration to NSW 1838-1857                                   </td><td style=\"text-align: right;\">          22,771</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/letters-re-migration-to-nsw-1838-1857.csv\">CSV file</a>                   </td><td><a href=\"https://mhnsw.au/indexes/immigration-and-shipping/letters-re-migration-to-nsw-1838-1857/\">Browse index</a>                    </td></tr>\n",
       "<tr><td>Maintenance registers - Metropolitan Children's Court 1915-1917         </td><td style=\"text-align: right;\">           1,372</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/maintenance-registers-metropolitan-childrens-court.csv\">CSV file</a>      </td><td><a href=\"https://mhnsw.au/indexes/child-care-and-protection/maintenance-registers-metropolitan-childrens-court/\">Browse index</a>      </td></tr>\n",
       "<tr><td>Miscellaneous immigrants index 1828-1843                                </td><td style=\"text-align: right;\">           8,821</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/miscellaneous-immigrants-index.csv\">CSV file</a>                          </td><td><a href=\"https://mhnsw.au/indexes/immigration-and-shipping/miscellaneous-immigrants-index/\">Browse index</a>                           </td></tr>\n",
       "<tr><td>NSW Government employees granted military leave                         </td><td style=\"text-align: right;\">          20,000</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/nsw-government-employees-granted-military-leave.csv\">CSV file</a>         </td><td><a href=\"https://mhnsw.au/indexes/military-and-war/nsw-government-employees-granted-military-leave/\">Browse index</a>                  </td></tr>\n",
       "<tr><td>NSW King’s / Queen’s Counsel appointment correspondence                 </td><td style=\"text-align: right;\">           2,083</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/nsw-kings-queens-counsel-appointment-correspondence.csv\">CSV file</a>     </td><td><a href=\"https://mhnsw.au/indexes/professions-and-occupations/nsw-kings-queens-counsel-appointment-correspondence/\">Browse index</a>   </td></tr>\n",
       "<tr><td>Naturalization index 1834-1903                                          </td><td style=\"text-align: right;\">           9,860</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/naturalization-index.csv\">CSV file</a>                                    </td><td><a href=\"https://mhnsw.au/indexes/naturalisation-and-citizenship/naturalization-index/\">Browse index</a>                               </td></tr>\n",
       "<tr><td>Nominal Roll of the First Railway Section (AIF)                         </td><td style=\"text-align: right;\">             416</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/nominal-roll-of-the-first-railway-section-aif.csv\">CSV file</a>           </td><td><a href=\"https://mhnsw.au/indexes/wwi/nominal-roll-of-the-first-railway-section-aif/\">Browse index</a>                                 </td></tr>\n",
       "<tr><td>Norfolk Island special bundles index 1794-1813                          </td><td style=\"text-align: right;\">             216</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/norfolk-island-special-bundles-index.csv\">CSV file</a>                    </td><td><a href=\"https://mhnsw.au/indexes/norfolk-island/norfolk-island-special-bundles-index/\">Browse index</a>                               </td></tr>\n",
       "<tr><td>Nurses index 1926-1954                                                  </td><td style=\"text-align: right;\">          46,499</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/nurses-index.csv\">CSV file</a>                                            </td><td><a href=\"https://mhnsw.au/indexes/nurses-and-midwives/nurses-index/\">Browse index</a>                                                  </td></tr>\n",
       "<tr><td>Police service registers 1852-1913                                      </td><td style=\"text-align: right;\">          20,000</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/police-service-registers.csv\">CSV file</a>                                </td><td><a href=\"https://mhnsw.au/indexes/police/police-service-registers/\">Browse index</a>                                                   </td></tr>\n",
       "<tr><td>Port Macquarie Small Debts Register, 1845-1887                          </td><td style=\"text-align: right;\">           2,036</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/port-macquarie-small-debts-register-1845-1887.csv\">CSV file</a>           </td><td><a href=\"https://mhnsw.au/indexes/courts-lower/port-macquarie-small-debts-register-1845-1887/\">Browse index</a>                        </td></tr>\n",
       "<tr><td>Probate records - supplementary index 1790-1875                         </td><td style=\"text-align: right;\">           1,626</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/probate-records-supplementary-index.csv\">CSV file</a>                     </td><td><a href=\"https://mhnsw.au/indexes/probates-and-wills/probate-records-supplementary-index/\">Browse index</a>                            </td></tr>\n",
       "<tr><td>Public Works Salary Registers                                           </td><td style=\"text-align: right;\">             523</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/public-works-salary-registers.csv\">CSV file</a>                           </td><td><a href=\"https://mhnsw.au/indexes/professions-and-occupations/public-works-salary-registers/\">Browse index</a>                         </td></tr>\n",
       "<tr><td>Publicans' licenses index 1830-1861                                     </td><td style=\"text-align: right;\">          20,000</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/publicans-licenses-index.csv\">CSV file</a>                                </td><td><a href=\"https://mhnsw.au/indexes/publicans-hoteliers-innkeepers/publicans-licenses-index/\">Browse index</a>                           </td></tr>\n",
       "<tr><td>Quarter sessions cases 1824-1837                                        </td><td style=\"text-align: right;\">           6,232</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/quarter-sessions-cases-1824-1837.csv\">CSV file</a>                        </td><td><a href=\"https://mhnsw.au/indexes/criminal-courts/quarter-sessions-cases-1824-1837/\">Browse index</a>                                  </td></tr>\n",
       "<tr><td>Railway employment records 1856-1917                                    </td><td style=\"text-align: right;\">             763</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/railway-employment-records.csv\">CSV file</a>                              </td><td><a href=\"https://mhnsw.au/indexes/railways-and-railway-workers/railway-employment-records/\">Browse index</a>                           </td></tr>\n",
       "<tr><td>Railways and Tramways Roll of Honour                                    </td><td style=\"text-align: right;\">           1,214</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/railways-and-tramways-roll-of-honour.csv\">CSV file</a>                    </td><td><a href=\"https://mhnsw.au/indexes/military-and-war/railways-and-tramways-roll-of-honour/\">Browse index</a>                             </td></tr>\n",
       "<tr><td>Register of Firms index 1903-1922                                       </td><td style=\"text-align: right;\">          50,000</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/register-of-firms-index.csv\">CSV file</a>                                 </td><td><a href=\"https://mhnsw.au/indexes/business-and-company-records/register-of-firms-index/\">Browse index</a>                              </td></tr>\n",
       "<tr><td>School teachers' rolls 1869-1908                                        </td><td style=\"text-align: right;\">          20,000</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/teachers-rolls.csv\">CSV file</a>                                          </td><td><a href=\"https://mhnsw.au/indexes/teachers/teachers-rolls/\">Browse index</a>                                                           </td></tr>\n",
       "<tr><td>Schools and related records 1876-1979                                   </td><td style=\"text-align: right;\">          30,181</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/schools-and-related-records.csv\">CSV file</a>                             </td><td><a href=\"https://mhnsw.au/indexes/schools-and-education/schools-and-related-records/\">Browse index</a>                                 </td></tr>\n",
       "<tr><td>Soldier (Closer) Settlement - Returned Soldiers Transfer files 1907-1951</td><td style=\"text-align: right;\">           9,656</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/soldier-settlement-returned-soldiers-transfer-files.csv\">CSV file</a>     </td><td><a href=\"https://mhnsw.au/indexes/soldier-settlement/soldier-settlement-returned-soldiers-transfer-files/\">Browse index</a>            </td></tr>\n",
       "<tr><td>Soldier (Closer) Settlement transfer registers 1919-1925                </td><td style=\"text-align: right;\">           4,957</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/soldier-settlement-transfer-registers.csv\">CSV file</a>                   </td><td><a href=\"https://mhnsw.au/indexes/soldier-settlement/soldier-settlement-transfer-registers/\">Browse index</a>                          </td></tr>\n",
       "<tr><td>Soldier (Closer) settlement promotion files index 1913-1958             </td><td style=\"text-align: right;\">           4,354</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/soldier-settlement-promotion-files-index.csv\">CSV file</a>                </td><td><a href=\"https://mhnsw.au/indexes/soldier-settlement/soldier-settlement-promotion-files-index/\">Browse index</a>                       </td></tr>\n",
       "<tr><td>Soldier Settlement loan files index 1906-1960                           </td><td style=\"text-align: right;\">           7,642</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/soldier-settlement-loan-files-index.csv\">CSV file</a>                     </td><td><a href=\"https://mhnsw.au/indexes/soldier-settlement/soldier-settlement-loan-files-index/\">Browse index</a>                            </td></tr>\n",
       "<tr><td>Soldier Settlement miscellaneous files index 1916                       </td><td style=\"text-align: right;\">           1,050</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/soldier-settlement-miscellaneous-files-index.csv\">CSV file</a>            </td><td><a href=\"https://mhnsw.au/indexes/soldier-settlement/soldier-settlement-miscellaneous-files-index/\">Browse index</a>                   </td></tr>\n",
       "<tr><td>Soldier Settlement purchases index 1905-1937                            </td><td style=\"text-align: right;\">           9,776</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/soldier-settlement-purchases-index.csv\">CSV file</a>                      </td><td><a href=\"https://mhnsw.au/indexes/soldier-settlement/soldier-settlement-purchases-index/\">Browse index</a>                             </td></tr>\n",
       "<tr><td>Squatters and graziers index 1837-1849                                  </td><td style=\"text-align: right;\">           9,003</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/squatters-and-graziers-index.csv\">CSV file</a>                            </td><td><a href=\"https://mhnsw.au/indexes/squatters-and-graziers/squatters-and-graziers-index/\">Browse index</a>                               </td></tr>\n",
       "<tr><td>Surveyor General's crown plans 1792-1886                                </td><td style=\"text-align: right;\">           5,455</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/surveyor-generals-crown-plans.csv\">CSV file</a>                           </td><td><a href=\"https://mhnsw.au/indexes/maps-and-plans/surveyor-generals-crown-plans/\">Browse index</a>                                      </td></tr>\n",
       "<tr><td>Surveyors' field books 1794-1860                                        </td><td style=\"text-align: right;\">             813</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/surveyors-field-books.csv\">CSV file</a>                                   </td><td><a href=\"https://mhnsw.au/indexes/maps-and-plans/surveyors-field-books/\">Browse index</a>                                              </td></tr>\n",
       "<tr><td>Surveyors’ letters 1822-1855                                            </td><td style=\"text-align: right;\">             157</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/surveyors-letters-1822-1855.csv\">CSV file</a>                             </td><td><a href=\"https://mhnsw.au/indexes/surveyor-general/surveyors-letters-1822-1855/\">Browse index</a>                                      </td></tr>\n",
       "<tr><td>Tramway employees 1879-1911                                             </td><td style=\"text-align: right;\">          10,606</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/tramway-employees.csv\">CSV file</a>                                       </td><td><a href=\"https://mhnsw.au/indexes/tramways/tramway-employees/\">Browse index</a>                                                        </td></tr>\n",
       "<tr><td>Unassisted immigrants index 1842-1855                                   </td><td style=\"text-align: right;\">         140,000</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/unassisted-immigrants-index.csv\">CSV file</a>                             </td><td><a href=\"https://mhnsw.au/indexes/immigration-and-shipping/unassisted-immigrants-index/\">Browse index</a>                              </td></tr>\n",
       "<tr><td>Unemployed in Sydney 1866                                               </td><td style=\"text-align: right;\">           3,222</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/unemployed-in-sydney.csv\">CSV file</a>                                    </td><td><a href=\"https://mhnsw.au/indexes/unemployment/unemployed-in-sydney/\">Browse index</a>                                                 </td></tr>\n",
       "<tr><td>Vessels arrived in Sydney 1837-1925                                     </td><td style=\"text-align: right;\">         129,999</td><td><a href=\"https://media.githubusercontent.com/media/wragge/srnsw-indexes/master/data/vessels-arrived-in-sydney.csv\">CSV file</a>                               </td><td><a href=\"https://mhnsw.au/indexes/immigration-and-shipping/vessels-arrived-in-sydney/\">Browse index</a>                                </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Select the columns that we want\n",
    "columns = df[[\"title\", \"rows\", \"download\", \"web\"]]\n",
    "\n",
    "# Create a list of headers\n",
    "headers = [\"Title\", \"Number of rows\", \"Download data\", \"View at State Archives\"]\n",
    "\n",
    "# Use Tabulate to generate a HTML table\n",
    "display(\n",
    "    HTML(\n",
    "        tabulate(\n",
    "            columns, headers=headers, showindex=False, tablefmt=\"unsafehtml\", intfmt=\",\"\n",
    "        )\n",
    "    )\n",
    ")\n",
    "\n",
    "# Write a GitHub Markdown formatted version of the table to a file\n",
    "with open(\"indexes.md\", \"w\") as md_file:\n",
    "    md_file.write(\n",
    "        tabulate(\n",
    "            columns, headers=headers, showindex=False, tablefmt=\"github\", intfmt=\",\"\n",
    "        )\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----\n",
    "\n",
    "Created by [Tim Sherratt](https://timsherratt.org/) for the [GLAM Workbench](https://glam-workbench.net/) project."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}