list_datasets_with_occurrence_count.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import requests\n",
    "import json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gets a list of all the occurrence datasets published in GBIF\n",
    "\n",
    "By using the registry and the occurrence API, this scripst get a list of all the occurrence datasets published on GBIF and get the number of occurrences they contain.\n",
    "Running the script might take a while.\n",
    "\n",
    "Alternatively, if you would like to get all the datasets with occurrences (which also include sampling-events and some checklists), you can simply use the folowing API call:\n",
    "\n",
    "```\n",
    "https://api.gbif.org/v1/occurrence/search?facet=datasetKey&limit=0&facetLimit=50000\n",
    "```\n",
    "(this is much faster)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "GBIF_API = \"http://api.gbif.org/v1/\"\n",
    "root_URL = \"https://www.gbif.org/dataset/\"\n",
    "step = 500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets_with_count = pd.DataFrame()\n",
    "\n",
    "endOfRecords = False\n",
    "offset = 0\n",
    "while not endOfRecords:\n",
    "    param = {\n",
    "        \"offset\": offset,\n",
    "        \"limit\": step,\n",
    "        \"type\": \"OCCURRENCE\"\n",
    "    }\n",
    "    # Get dataset\n",
    "    response = requests.get(GBIF_API + \"dataset\", param)\n",
    "    if response.ok:\n",
    "        dataset_list = response.json()\n",
    "        for dataset in dataset_list[\"results\"]:\n",
    "            datasets_with_count.at[dataset[\"key\"], \"URL\"] = root_URL + dataset[\"key\"]\n",
    "            datasets_with_count.at[dataset[\"key\"], \"title\"] = dataset[\"title\"]\n",
    "            if \"description\" in dataset:\n",
    "                datasets_with_count.at[dataset[\"key\"], \"description\"] = dataset[\"description\"]\n",
    "            \n",
    "            # Get occurrence count\n",
    "            response_occ_count = requests.get(GBIF_API + \"occurrence/count\", {\"datasetKey\": dataset[\"key\"]})\n",
    "            if response_occ_count.ok:\n",
    "                datasets_with_count.at[dataset[\"key\"], \"count_occ\"] = response_occ_count.json()\n",
    "            else:\n",
    "                print(response_occ_count)\n",
    "                print(dataset[\"key\"])\n",
    "        offset += step\n",
    "        endOfRecords = dataset_list[\"endOfRecords\"]\n",
    "    else:\n",
    "        print(response)\n",
    "        endOfRecords = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets_with_count.to_csv('occurrence_datasets_with_count.txt', sep = '\\t', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>URL</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>count_occ</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0d7c6a1a-0aab-47dc-8256-f23fefac69cd</td>\n",
       "      <td>https://www.gbif.org/dataset/0d7c6a1a-0aab-47d...</td>\n",
       "      <td>Wild bees of Belgium</td>\n",
       "      <td>Multidiciplinary assessment of BELgian wild BE...</td>\n",
       "      <td>236585.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>a0a4d131-f53f-43b2-a1ba-254473b8a006</td>\n",
       "      <td>https://www.gbif.org/dataset/a0a4d131-f53f-43b...</td>\n",
       "      <td>Marine Invertebrate voucher specimens at the F...</td>\n",
       "      <td>This dataset is generated from the records for...</td>\n",
       "      <td>99609.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>73416d1c-e675-43b3-bbe4-672d5a9f9e8b</td>\n",
       "      <td>https://www.gbif.org/dataset/73416d1c-e675-43b...</td>\n",
       "      <td>Monitoreo de Flora y Fauna de los Acuerdos de ...</td>\n",
       "      <td>Estos datos dan cuenta de los inventarios de b...</td>\n",
       "      <td>2413.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>36cd5465-6625-4bf5-929d-e51a57583c9c</td>\n",
       "      <td>https://www.gbif.org/dataset/36cd5465-6625-4bf...</td>\n",
       "      <td>Caracterización de la avifauna asociada a la c...</td>\n",
       "      <td>El Atrato, es considerado  como uno de los río...</td>\n",
       "      <td>313.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4bef41c6-daf5-48b0-9868-0548976dead0</td>\n",
       "      <td>https://www.gbif.org/dataset/4bef41c6-daf5-48b...</td>\n",
       "      <td>Caracterización de la fauna reptiliana asociad...</td>\n",
       "      <td>El Atrato, es considerado  como uno de los río...</td>\n",
       "      <td>60.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>85727f1e-f762-11e1-a439-00145eb45e9a</td>\n",
       "      <td>https://www.gbif.org/dataset/85727f1e-f762-11e...</td>\n",
       "      <td>Herbarium Willing at Herbarium Berolinense, Be...</td>\n",
       "      <td>The “Herbarium Willing” is a private Phaneroga...</td>\n",
       "      <td>111899.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>85739778-f762-11e1-a439-00145eb45e9a</td>\n",
       "      <td>https://www.gbif.org/dataset/85739778-f762-11e...</td>\n",
       "      <td>Lichens at Herbarium Berolinense, Berlin (B)</td>\n",
       "      <td>This database contains label information for a...</td>\n",
       "      <td>129129.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8575f23e-f762-11e1-a439-00145eb45e9a</td>\n",
       "      <td>https://www.gbif.org/dataset/8575f23e-f762-11e...</td>\n",
       "      <td>PonTaurus collection</td>\n",
       "      <td>Plant specimens gathered in the Toroslar mount...</td>\n",
       "      <td>1534.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>85771146-f762-11e1-a439-00145eb45e9a</td>\n",
       "      <td>https://www.gbif.org/dataset/85771146-f762-11e...</td>\n",
       "      <td>Staatliches Museum für Naturkunde Stuttgart, H...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>27351.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>857846ba-f762-11e1-a439-00145eb45e9a</td>\n",
       "      <td>https://www.gbif.org/dataset/857846ba-f762-11e...</td>\n",
       "      <td>Desmidiaceae Rostock</td>\n",
       "      <td>Culture collection of Desmidiaceae contained i...</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>19129 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                    URL  \\\n",
       "0d7c6a1a-0aab-47dc-8256-f23fefac69cd  https://www.gbif.org/dataset/0d7c6a1a-0aab-47d...   \n",
       "a0a4d131-f53f-43b2-a1ba-254473b8a006  https://www.gbif.org/dataset/a0a4d131-f53f-43b...   \n",
       "73416d1c-e675-43b3-bbe4-672d5a9f9e8b  https://www.gbif.org/dataset/73416d1c-e675-43b...   \n",
       "36cd5465-6625-4bf5-929d-e51a57583c9c  https://www.gbif.org/dataset/36cd5465-6625-4bf...   \n",
       "4bef41c6-daf5-48b0-9868-0548976dead0  https://www.gbif.org/dataset/4bef41c6-daf5-48b...   \n",
       "...                                                                                 ...   \n",
       "85727f1e-f762-11e1-a439-00145eb45e9a  https://www.gbif.org/dataset/85727f1e-f762-11e...   \n",
       "85739778-f762-11e1-a439-00145eb45e9a  https://www.gbif.org/dataset/85739778-f762-11e...   \n",
       "8575f23e-f762-11e1-a439-00145eb45e9a  https://www.gbif.org/dataset/8575f23e-f762-11e...   \n",
       "85771146-f762-11e1-a439-00145eb45e9a  https://www.gbif.org/dataset/85771146-f762-11e...   \n",
       "857846ba-f762-11e1-a439-00145eb45e9a  https://www.gbif.org/dataset/857846ba-f762-11e...   \n",
       "\n",
       "                                                                                  title  \\\n",
       "0d7c6a1a-0aab-47dc-8256-f23fefac69cd                               Wild bees of Belgium   \n",
       "a0a4d131-f53f-43b2-a1ba-254473b8a006  Marine Invertebrate voucher specimens at the F...   \n",
       "73416d1c-e675-43b3-bbe4-672d5a9f9e8b  Monitoreo de Flora y Fauna de los Acuerdos de ...   \n",
       "36cd5465-6625-4bf5-929d-e51a57583c9c  Caracterización de la avifauna asociada a la c...   \n",
       "4bef41c6-daf5-48b0-9868-0548976dead0  Caracterización de la fauna reptiliana asociad...   \n",
       "...                                                                                 ...   \n",
       "85727f1e-f762-11e1-a439-00145eb45e9a  Herbarium Willing at Herbarium Berolinense, Be...   \n",
       "85739778-f762-11e1-a439-00145eb45e9a       Lichens at Herbarium Berolinense, Berlin (B)   \n",
       "8575f23e-f762-11e1-a439-00145eb45e9a                               PonTaurus collection   \n",
       "85771146-f762-11e1-a439-00145eb45e9a  Staatliches Museum für Naturkunde Stuttgart, H...   \n",
       "857846ba-f762-11e1-a439-00145eb45e9a                               Desmidiaceae Rostock   \n",
       "\n",
       "                                                                            description  \\\n",
       "0d7c6a1a-0aab-47dc-8256-f23fefac69cd  Multidiciplinary assessment of BELgian wild BE...   \n",
       "a0a4d131-f53f-43b2-a1ba-254473b8a006  This dataset is generated from the records for...   \n",
       "73416d1c-e675-43b3-bbe4-672d5a9f9e8b  Estos datos dan cuenta de los inventarios de b...   \n",
       "36cd5465-6625-4bf5-929d-e51a57583c9c  El Atrato, es considerado  como uno de los río...   \n",
       "4bef41c6-daf5-48b0-9868-0548976dead0  El Atrato, es considerado  como uno de los río...   \n",
       "...                                                                                 ...   \n",
       "85727f1e-f762-11e1-a439-00145eb45e9a  The “Herbarium Willing” is a private Phaneroga...   \n",
       "85739778-f762-11e1-a439-00145eb45e9a  This database contains label information for a...   \n",
       "8575f23e-f762-11e1-a439-00145eb45e9a  Plant specimens gathered in the Toroslar mount...   \n",
       "85771146-f762-11e1-a439-00145eb45e9a                                                NaN   \n",
       "857846ba-f762-11e1-a439-00145eb45e9a  Culture collection of Desmidiaceae contained i...   \n",
       "\n",
       "                                      count_occ  \n",
       "0d7c6a1a-0aab-47dc-8256-f23fefac69cd   236585.0  \n",
       "a0a4d131-f53f-43b2-a1ba-254473b8a006    99609.0  \n",
       "73416d1c-e675-43b3-bbe4-672d5a9f9e8b     2413.0  \n",
       "36cd5465-6625-4bf5-929d-e51a57583c9c      313.0  \n",
       "4bef41c6-daf5-48b0-9868-0548976dead0       60.0  \n",
       "...                                         ...  \n",
       "85727f1e-f762-11e1-a439-00145eb45e9a   111899.0  \n",
       "85739778-f762-11e1-a439-00145eb45e9a   129129.0  \n",
       "8575f23e-f762-11e1-a439-00145eb45e9a     1534.0  \n",
       "85771146-f762-11e1-a439-00145eb45e9a    27351.0  \n",
       "857846ba-f762-11e1-a439-00145eb45e9a        6.0  \n",
       "\n",
       "[19129 rows x 4 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datasets_with_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}