create_github_issues_grscicoll_inst.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import json\n",
    "import requests\n",
    "import time\n",
    "import sys\n",
    "# import psycopg2\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "grscicoll_api = 'http://api.gbif.org/v1/grscicoll/'\n",
    "occurrence_api = 'https://api.gbif.org/v1/occurrence/search'\n",
    "organization_api = 'https://api.gbif.org/v1/organization'\n",
    "dataset_prefix = \"https://www.gbif.org/dataset/\"\n",
    "# GitHub\n",
    "gh_username = \"\"\n",
    "github_header = {'Accept': \"application/vnd.github.v3+json\"}\n",
    "gh_token = \"\"\n",
    "owner = \"gbif\"\n",
    "repo = \"collection-mobilization\"\n",
    "gh_api = \"https://api.github.com/repos/\"+owner+\"/\"+repo+\"/issues\"\n",
    "link_readme = \"https://github.com/\"+owner+\"/\"+repo+\"#readme\"\n",
    "\n",
    "# Scope\n",
    "region = \"ASIA\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test_json = {\n",
    "#     \"title\": \"title\"\n",
    "# }\n",
    "\n",
    "# post = requests.post(gh_api,\n",
    "#                      data=json.dumps(test_json),\n",
    "#                      auth=(gh_username, gh_token),\n",
    "#                      headers=github_header)\n",
    "# print(gh_api)\n",
    "# post"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_body_issue(inst, country, steps_to_check,\n",
    "                      gbif_prefix='https://www.gbif.org/',\n",
    "                      grscicoll_prefix='https://www.gbif.org/grscicoll/'):\n",
    "    \"\"\"\n",
    "    Use institution information and find related GBIF records then format all of it in a GH issue\n",
    "    \"\"\"\n",
    "    markdown_body = \"## \"+inst[\"name\"]+\"\\n\"\n",
    "    markdown_body += \"**GRSciColl URL**: \"+grscicoll_prefix+\"institution/\"+inst[\"key\"]+\"\\n\"\n",
    "    if inst[\"masterSource\"] != \"GRSCICOLL\":\n",
    "        markdown_body += \"**Synchronized with** `\"+inst[\"masterSource\"]+\"`\\n\"\n",
    "    if \"code\" in inst:\n",
    "        markdown_body += \"**Code**: `\"+inst[\"code\"]+\"`\\n\"\n",
    "    if \"homepage\" in inst:\n",
    "        markdown_body += \"**Homepage**: \"+ inst[\"homepage\"]+\"\\n\"\n",
    "    \n",
    "    markdown_body += \"**Fuzzy name search in GBIF publisher**:\\n\"\n",
    "    param_org = {\n",
    "        \"q\": inst[\"name\"],\n",
    "        \"limit\": 30,\n",
    "        \"country\": country\n",
    "    }\n",
    "    inst_name_search = requests.get(organization_api, param_org)\n",
    "    if inst_name_search.ok:\n",
    "        organization_match = inst_name_search.json()\n",
    "        organization_names = []\n",
    "        organization_UUIDs = []\n",
    "        if organization_match[\"results\"] == []:\n",
    "            markdown_body += \"`None`\\n\"\n",
    "        else:\n",
    "            for org in organization_match[\"results\"]:\n",
    "                markdown_body += \"* [\"+org[\"title\"]+\"](\"+gbif_prefix+\"publisher/\"+org[\"key\"]+\")\\n\"\n",
    "    \n",
    "    markdown_body += \"\\n\\n\"\n",
    "    \n",
    "    linked_occurrences = 0\n",
    "    param = {\n",
    "        \"institution_key\": inst[\"key\"],\n",
    "        \"limit\": 0,\n",
    "        \"facet\": \"publishingOrg\"\n",
    "    }\n",
    "    inst_occ = requests.get(occurrence_api, param)\n",
    "    if inst_occ.ok:\n",
    "        inst_occ_dict = inst_occ.json()\n",
    "        linked_occurrences = inst_occ_dict[\"count\"]\n",
    "        markdown_body += \"**Number of linked occurrences**: `\"+str(linked_occurrences)+\"` (\"+gbif_prefix+\"occurrence/search?advanced=1&institution_key=\"+inst[\"key\"]+\")\\n\"\n",
    "        if linked_occurrences != 0:\n",
    "            markdown_body += \"**Publishing organizations for linked occurrences**:\\n\"\n",
    "            for publisher in inst_occ_dict[\"facets\"][0][\"counts\"]:\n",
    "                markdown_body += \"* \"+gbif_prefix+\"publisher/\"+publisher[\"name\"]+\"\\n\"\n",
    "    \n",
    "    markdown_body += steps_to_check\n",
    "    return markdown_body"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def post_issue(data, gh_api, gh_username, gh_token,github_header):\n",
    "    post = requests.post(gh_api,\n",
    "                         data=json.dumps(data),\n",
    "                         auth=(gh_username, gh_token),\n",
    "                         headers=github_header)\n",
    "    if not post.ok:\n",
    "        print(\"couldn't create issue for \", data)\n",
    "        print(post)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_issue_for_institution_based_on_json(inst, country, gh_api, gh_username, gh_token, github_header, steps_to_check):\n",
    "    \"\"\"\n",
    "    Create a GitHub issue for an institution entry:\n",
    "    * inst: is an institution JSON as returned by the GRSciColl API.\n",
    "    * country: country of the institution\n",
    "    \"\"\"\n",
    "    issue = {}\n",
    "    issue[\"title\"] = inst[\"name\"]\n",
    "    issue[\"body\"] = create_body_issue(inst, country, steps_to_follow)\n",
    "    issue[\"labels\"] = [country]\n",
    "    post_issue(issue, gh_api, gh_username, gh_token,github_header)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_issue_for_institution_per_country(country, already_issued, grscicoll_api, gh_api, gh_username, gh_token, github_header, steps_to_check, step=500):\n",
    "    \"\"\"\n",
    "    For a given country, create one GitHub issue per GRSciColl institution entry. If an issue already exists, don't create a new issue\n",
    "    \"\"\"\n",
    "    query = {\n",
    "        \"country\": country,\n",
    "        \"limit\": step,\n",
    "        \"offset\" : 0\n",
    "    }\n",
    "    endOfRecords = False\n",
    "    while not endOfRecords:\n",
    "        institutions = requests.get(grscicoll_api + 'institution/', query)\n",
    "\n",
    "        if institutions.ok:\n",
    "            institution_page = institutions.json()\n",
    "            endOfRecords = institution_page[\"endOfRecords\"]\n",
    "            query[\"offset\"] += step\n",
    "\n",
    "            for inst in institution_page[\"results\"]:\n",
    "                if inst[\"key\"] not in already_issued:\n",
    "                    create_issue_for_institution_based_on_json(inst, country, gh_api, gh_username, gh_token, github_header, steps_to_check)\n",
    "                    time.sleep(3)\n",
    "        else:\n",
    "            print(institutions)\n",
    "            endOfRecords = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_issue_for_institution(key, country, gh_api, gh_username, gh_token, github_header, steps_to_check):\n",
    "    \"\"\"\n",
    "    Create a GitHub issue for an institution entry:\n",
    "    * key: is an institution key in GRSciColl\n",
    "    * country: country of the institution\n",
    "    \"\"\"\n",
    "    institution = requests.get(grscicoll_api + 'institution/'+ key)\n",
    "    if institution.ok:\n",
    "        create_issue_for_institution_based_on_json(institution.json(), country, gh_api, gh_username, gh_token, github_header, steps_to_check)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "steps_to_follow = \"\"\"\\n\\n---\n",
    "1. **Find out if the information is complete and up to date**:\n",
    "\n",
    "- [ ]  Check the homepage if available or google the institution name in the country. Check if all collections for that institution are represented in GRSciColl.\n",
    "    - Check whether it is an independent entry. If duplicated merge with the selected and entry and close this issue.\n",
    "- [ ]  Add missing collections/information to the institution on GRSciColl directly in the registry. If some collections are digitized, put the information in the GRSciColl `Notes` field.\n",
    "\n",
    "2. **Check if the data is also in GBIF**:\n",
    "\n",
    "- [ ]  If there are GBIF occurrence records linked, check from which dataset/publisher they come. Is the institution a registered publisher? Or do the records come from a third party publisher? Are all the collections in GRSciColl also in GBIF? (add comments to the issue)\n",
    "- [ ]  If no record is linked to GRSciColl, look for the institution name on the GBIF list of publishers. Is there any corresponding publisher? Have they published any data? (add comments to the issue)\n",
    "- [ ]  If data has been published on GBIF but isn’t linked to GRSciColl, notify Marie (tag ManonGros), she can link the data.\n",
    "- [ ]  Translate outcome of your checks into labels. See guidelines here: \"\"\" + link_readme"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "## If you want to query all the countries for a region\n",
    "# countries = []\n",
    "# all_countries = requests.get(\"https://api.gbif.org/v1/enumeration/country\")\n",
    "# if all_countries.ok:\n",
    "#     all_countries = all_countries.json()\n",
    "#     for country in all_countries:\n",
    "#         if country[\"gbifRegion\"] == region:\n",
    "#             countries.append(country[\"iso2\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "countries = [\n",
    "    \"VE\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "## IF you want to have a list of all the issues already created\n",
    "# page = 1\n",
    "# end = False\n",
    "# already_issued = []\n",
    "\n",
    "# while not end:\n",
    "#     issues = requests.get(gh_api, {\"page\":page}, auth=(gh_username, gh_token)).json()\n",
    "#     page += 1\n",
    "#     if len(issues) < 30:\n",
    "#         end = True\n",
    "        \n",
    "#     for issue in issues:\n",
    "#         search_res = re.search('https://www.gbif.org/grscicoll/institution/.+\\n', issue[\"body\"])\n",
    "#         if search_res is not None:\n",
    "#             already_issued.append(search_res.group(0).replace(\"\\n\",\"\").replace(\"\\r\", \"\").replace(\"https://www.gbif.org/grscicoll/institution/\",\"\"))\n",
    "#         else:\n",
    "#             print(issue[\"body\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "VE\n"
     ]
    }
   ],
   "source": [
    "for country in countries:\n",
    "    print(country)\n",
    "    create_issue_for_institution_per_country(country, already_issued, grscicoll_api, gh_api, gh_username, gh_token, github_header, steps_to_follow)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "## For just one institution\n",
    "create_issue_for_institution(\"a50c1ed5-76c3-478c-be0b-e781e7cb04eb\", \"LT\", gh_api, gh_username, gh_token, github_header, steps_to_follow)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}