obo/3_cross_refs/xrefs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [],
   "source": [
    "import json\n",
    "import yaml\n",
    "import re\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "df_skos = pd.read_json('results/xrefs_skos.json')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "def classify_mapping(value: str):\n",
    "    if value.startswith('http://') or value.startswith('https://'):\n",
    "        if value.startswith('http://purl.obolibrary.org/obo/'):\n",
    "            foreign_ns_id = value.replace('http://purl.obolibrary.org/obo/', '')\n",
    "            ref_type = 'uri_obo'\n",
    "        else:\n",
    "            foreign_ns_id = re.sub('.*[/#]', '', value)\n",
    "            ref_type = 'uri'\n",
    "    elif re.match('^t\\d+$', value):\n",
    "        ref_type = 'bnode'\n",
    "        foreign_ns_id = ''\n",
    "    else:\n",
    "        foreign_ns_id = value\n",
    "        ref_type = 'id'\n",
    "\n",
    "    namespace = ''\n",
    "    ns_sep = ''\n",
    "    foreign_id = ''\n",
    "    match = re.match('([a-zA-Z_\\d.]+)([:_])([.A-Z\\d]+)', foreign_ns_id)\n",
    "    if not match:\n",
    "        match = re.match('([a-zA-Z_\\d.]+)([:])(.+)', foreign_ns_id)\n",
    "\n",
    "    if match:\n",
    "        namespace = match.group(1)\n",
    "        ns_sep = match.group(2)\n",
    "        foreign_id = match.group(3)\n",
    "\n",
    "    return ref_type, foreign_ns_id, namespace, ns_sep, foreign_id\n",
    "\n",
    "def analyze_mappings(df: pd.DataFrame):\n",
    "    df1 = df.copy()\n",
    "    df1['ref_type'], df1['foreign_ns_id'], df1['namespace'], df1['ns_sep'], df1['foreign_id'] = \\\n",
    "        zip(*df1['o'].apply(classify_mapping))\n",
    "    agg1 = df1[['o', 'ref_type', 'namespace']].groupby(['ref_type', 'namespace']).agg('count')\n",
    "    agg2 = df1[['o', 'ref_type']].groupby(['ref_type']).agg('count')\n",
    "    return df1, agg1, agg2"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "source": [
    "df1_skos, agg1_skos, agg2_skos = analyze_mappings(df_skos)\n",
    "agg1_skos"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": 4,
   "outputs": [
    {
     "data": {
      "text/plain": "                      o\nref_type namespace     \nid                    1\n         ICD10CM      1\n         MESH       246\n         NCI          5\n         ORDO         2\n         UMLS_CUI     1\nuri_obo  MI           3",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th>o</th>\n    </tr>\n    <tr>\n      <th>ref_type</th>\n      <th>namespace</th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th rowspan=\"6\" valign=\"top\">id</th>\n      <th></th>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>ICD10CM</th>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>MESH</th>\n      <td>246</td>\n    </tr>\n    <tr>\n      <th>NCI</th>\n      <td>5</td>\n    </tr>\n    <tr>\n      <th>ORDO</th>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>UMLS_CUI</th>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>uri_obo</th>\n      <th>MI</th>\n      <td>3</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "data": {
      "text/plain": "            o\nref_type     \nid        256\nuri_obo     3",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>o</th>\n    </tr>\n    <tr>\n      <th>ref_type</th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>id</th>\n      <td>256</td>\n    </tr>\n    <tr>\n      <th>uri_obo</th>\n      <td>3</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg2_skos"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "data": {
      "text/plain": ":    255\n_      3\n       1\nName: ns_sep, dtype: int64"
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1_skos['ns_sep'].value_counts()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [],
   "source": [
    "df_alt_id = pd.read_json('results/xrefs_altId.json')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "source": [
    "df1_alt, agg1_alt, agg2_alt = analyze_mappings(df_alt_id)\n",
    "agg1_alt"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": 8,
   "outputs": [
    {
     "data": {
      "text/plain": "                         o\nref_type namespace        \nid                      17\n         BSPOTEMP        1\n         BTO             3\n         CHEBI       18502\n         CL             73\n         CMECS          26\n         DOID         1696\n         EHDAA2        157\n         EMAPA           1\n         ENVO            7\n         EO            574\n         FBbt          242\n         FMA             1\n         GAZ          4012\n         GO           3501\n         HDAA2           4\n         HP           3693\n         LTER           12\n         MESH            4\n         METAR           1\n         MP            684\n         NBO            10\n         NCBITaxon   65446\n         NLCD           20\n         OBO             1\n         OBO_REL         3\n         PATO           84\n         PO            204\n         SYMP            6\n         UBERON        322\n         UBERONTEMP      9\n         XCO             1\n         ZFA             3\nuri                     17",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th>o</th>\n    </tr>\n    <tr>\n      <th>ref_type</th>\n      <th>namespace</th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th rowspan=\"33\" valign=\"top\">id</th>\n      <th></th>\n      <td>17</td>\n    </tr>\n    <tr>\n      <th>BSPOTEMP</th>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>BTO</th>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>CHEBI</th>\n      <td>18502</td>\n    </tr>\n    <tr>\n      <th>CL</th>\n      <td>73</td>\n    </tr>\n    <tr>\n      <th>CMECS</th>\n      <td>26</td>\n    </tr>\n    <tr>\n      <th>DOID</th>\n      <td>1696</td>\n    </tr>\n    <tr>\n      <th>EHDAA2</th>\n      <td>157</td>\n    </tr>\n    <tr>\n      <th>EMAPA</th>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>ENVO</th>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>EO</th>\n      <td>574</td>\n    </tr>\n    <tr>\n      <th>FBbt</th>\n      <td>242</td>\n    </tr>\n    <tr>\n      <th>FMA</th>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>GAZ</th>\n      <td>4012</td>\n    </tr>\n    <tr>\n      <th>GO</th>\n      <td>3501</td>\n    </tr>\n    <tr>\n      <th>HDAA2</th>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>HP</th>\n      <td>3693</td>\n    </tr>\n    <tr>\n      <th>LTER</th>\n      <td>12</td>\n    </tr>\n    <tr>\n      <th>MESH</th>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>METAR</th>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>MP</th>\n      <td>684</td>\n    </tr>\n    <tr>\n      <th>NBO</th>\n      <td>10</td>\n    </tr>\n    <tr>\n      <th>NCBITaxon</th>\n      <td>65446</td>\n    </tr>\n    <tr>\n      <th>NLCD</th>\n      <td>20</td>\n    </tr>\n    <tr>\n      <th>OBO</th>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>OBO_REL</th>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>PATO</th>\n      <td>84</td>\n    </tr>\n    <tr>\n      <th>PO</th>\n      <td>204</td>\n    </tr>\n    <tr>\n      <th>SYMP</th>\n      <td>6</td>\n    </tr>\n    <tr>\n      <th>UBERON</th>\n      <td>322</td>\n    </tr>\n    <tr>\n      <th>UBERONTEMP</th>\n      <td>9</td>\n    </tr>\n    <tr>\n      <th>XCO</th>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>ZFA</th>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>uri</th>\n      <th></th>\n      <td>17</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [
    {
     "data": {
      "text/plain": "              o\nref_type       \nid        99320\nuri          17",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>o</th>\n    </tr>\n    <tr>\n      <th>ref_type</th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>id</th>\n      <td>99320</td>\n    </tr>\n    <tr>\n      <th>uri</th>\n      <td>17</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg2_alt"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [],
   "source": [
    "df_obo = pd.read_json('results/xrefs_altId.json')\\\n",
    "    .append(pd.read_json('results/xrefs_dbxref_1.json'))\\\n",
    "    .append(pd.read_json('results/xrefs_dbxref_2.json'))\\\n",
    "    .append(pd.read_json('results/xrefs_dbxref_3.json'))\\\n",
    "    .append(pd.read_json('results/xrefs_dbxref_4.json'))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [],
   "source": [
    "df1_obo, agg1_obo, agg2_obo = analyze_mappings(df_obo)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "source": [
    "agg1_obo"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": 12,
   "outputs": [
    {
     "data": {
      "text/plain": "                                      o\nref_type namespace                     \nbnode                               214\nid                               374092\n         .answers.com                 2\n         .thefreedictionary.com       2\n         AAO                       2065\n...                                 ...\nuri_obo  NCBITaxon                   98\n         NCIT                         6\n         OMIT                         1\n         TAXRANK                      1\n         TXPO                         1\n\n[1109 rows x 1 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th>o</th>\n    </tr>\n    <tr>\n      <th>ref_type</th>\n      <th>namespace</th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>bnode</th>\n      <th></th>\n      <td>214</td>\n    </tr>\n    <tr>\n      <th rowspan=\"4\" valign=\"top\">id</th>\n      <th></th>\n      <td>374092</td>\n    </tr>\n    <tr>\n      <th>.answers.com</th>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>.thefreedictionary.com</th>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>AAO</th>\n      <td>2065</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <th>...</th>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th rowspan=\"5\" valign=\"top\">uri_obo</th>\n      <th>NCBITaxon</th>\n      <td>98</td>\n    </tr>\n    <tr>\n      <th>NCIT</th>\n      <td>6</td>\n    </tr>\n    <tr>\n      <th>OMIT</th>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>TAXRANK</th>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>TXPO</th>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n<p>1109 rows × 1 columns</p>\n</div>"
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [
    {
     "data": {
      "text/plain": "                o\nref_type         \nbnode         214\nid        3856416\nuri         51953\nuri_obo       169",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>o</th>\n    </tr>\n    <tr>\n      <th>ref_type</th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>bnode</th>\n      <td>214</td>\n    </tr>\n    <tr>\n      <th>id</th>\n      <td>3856416</td>\n    </tr>\n    <tr>\n      <th>uri</th>\n      <td>51953</td>\n    </tr>\n    <tr>\n      <th>uri_obo</th>\n      <td>169</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg2_obo"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [
    {
     "data": {
      "text/plain": "                       o\nref_type ns_sep         \nbnode                214\nid                374092\n         :       3439229\n         _         43095\nuri                51379\n         :            49\n         _           525\nuri_obo               57\n         _           112",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th>o</th>\n    </tr>\n    <tr>\n      <th>ref_type</th>\n      <th>ns_sep</th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>bnode</th>\n      <th></th>\n      <td>214</td>\n    </tr>\n    <tr>\n      <th rowspan=\"3\" valign=\"top\">id</th>\n      <th></th>\n      <td>374092</td>\n    </tr>\n    <tr>\n      <th>:</th>\n      <td>3439229</td>\n    </tr>\n    <tr>\n      <th>_</th>\n      <td>43095</td>\n    </tr>\n    <tr>\n      <th rowspan=\"3\" valign=\"top\">uri</th>\n      <th></th>\n      <td>51379</td>\n    </tr>\n    <tr>\n      <th>:</th>\n      <td>49</td>\n    </tr>\n    <tr>\n      <th>_</th>\n      <td>525</td>\n    </tr>\n    <tr>\n      <th rowspan=\"2\" valign=\"top\">uri_obo</th>\n      <th></th>\n      <td>57</td>\n    </tr>\n    <tr>\n      <th>_</th>\n      <td>112</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_obo_sep_agg = df1_obo[['ref_type', 'ns_sep', 'o']].groupby(['ref_type', 'ns_sep']).agg('count')\n",
    "df_obo_sep_agg"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [],
   "source": [
    "test = df1_obo[df1_obo.ns_sep == '_']"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "Attempt to resolve OBO Foundry prefixes"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [],
   "source": [
    "with open('data/ontologies.yml') as fp:\n",
    "    ontologies = yaml.safe_load(fp)['ontologies']"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [],
   "source": [
    "df_ontos = pd.DataFrame.from_records(ontologies, columns=['id', 'preferredPrefix', 'alternatePrefix'])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "outputs": [],
   "source": [
    "prefixes = set()\n",
    "for _, r in df_ontos.iterrows():\n",
    "    prefixes.add(r['id'])\n",
    "    if isinstance(r['preferredPrefix'], str):\n",
    "        prefixes.add(r['preferredPrefix'].lower())"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "outputs": [],
   "source": [
    "agg_valid_prefixes = agg1_obo.reset_index()\n",
    "agg_valid_prefixes = agg_valid_prefixes[agg_valid_prefixes.namespace.apply(lambda x: x.lower() in prefixes)]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [
    {
     "data": {
      "text/plain": "               o      \n             sum count\nref_type              \nid        187167    75\nuri_obo      112     8",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead tr th {\n        text-align: left;\n    }\n\n    .dataframe thead tr:last-of-type th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr>\n      <th></th>\n      <th colspan=\"2\" halign=\"left\">o</th>\n    </tr>\n    <tr>\n      <th></th>\n      <th>sum</th>\n      <th>count</th>\n    </tr>\n    <tr>\n      <th>ref_type</th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>id</th>\n      <td>187167</td>\n      <td>75</td>\n    </tr>\n    <tr>\n      <th>uri_obo</th>\n      <td>112</td>\n      <td>8</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg_valid_prefixes[['ref_type', 'o']].groupby('ref_type').agg(['sum', 'count'])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "outputs": [
    {
     "data": {
      "text/plain": "               o      \n             sum count\nref_type              \nid        187167    75\nuri_obo      112     8",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead tr th {\n        text-align: left;\n    }\n\n    .dataframe thead tr:last-of-type th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr>\n      <th></th>\n      <th colspan=\"2\" halign=\"left\">o</th>\n    </tr>\n    <tr>\n      <th></th>\n      <th>sum</th>\n      <th>count</th>\n    </tr>\n    <tr>\n      <th>ref_type</th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>id</th>\n      <td>187167</td>\n      <td>75</td>\n    </tr>\n    <tr>\n      <th>uri_obo</th>\n      <td>112</td>\n      <td>8</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg_valid_prefixes[['ref_type', 'o']].groupby(['ref_type']).agg(['sum', 'count'])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "outputs": [],
   "source": [
    "all_valid_pref = df1_obo[df1_obo.namespace.apply(lambda x: x.lower() in prefixes)]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "outputs": [
    {
     "data": {
      "text/plain": "                      o\nref_type ns_sep        \nid       :       187148\n         _           19\nuri_obo  _          112",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th>o</th>\n    </tr>\n    <tr>\n      <th>ref_type</th>\n      <th>ns_sep</th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th rowspan=\"2\" valign=\"top\">id</th>\n      <th>:</th>\n      <td>187148</td>\n    </tr>\n    <tr>\n      <th>_</th>\n      <td>19</td>\n    </tr>\n    <tr>\n      <th>uri_obo</th>\n      <th>_</th>\n      <td>112</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_valid_pref[['ref_type', 'ns_sep', 'o']].groupby(['ref_type', 'ns_sep']).agg('count')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "outputs": [
    {
     "data": {
      "text/plain": "                                                        p  \\\n397952  http://www.geneontology.org/formats/oboInOwl#h...   \n684507  http://www.geneontology.org/formats/oboInOwl#h...   \n693346  http://www.geneontology.org/formats/oboInOwl#h...   \n693347  http://www.geneontology.org/formats/oboInOwl#h...   \n693348  http://www.geneontology.org/formats/oboInOwl#h...   \n693349  http://www.geneontology.org/formats/oboInOwl#h...   \n693350  http://www.geneontology.org/formats/oboInOwl#h...   \n693351  http://www.geneontology.org/formats/oboInOwl#h...   \n704736  http://www.geneontology.org/formats/oboInOwl#h...   \n704737  http://www.geneontology.org/formats/oboInOwl#h...   \n704738  http://www.geneontology.org/formats/oboInOwl#h...   \n704739  http://www.geneontology.org/formats/oboInOwl#h...   \n704740  http://www.geneontology.org/formats/oboInOwl#h...   \n704741  http://www.geneontology.org/formats/oboInOwl#h...   \n704742  http://www.geneontology.org/formats/oboInOwl#h...   \n704743  http://www.geneontology.org/formats/oboInOwl#h...   \n704744  http://www.geneontology.org/formats/oboInOwl#h...   \n704745  http://www.geneontology.org/formats/oboInOwl#h...   \n704746  http://www.geneontology.org/formats/oboInOwl#h...   \n\n                                                    s               o  \\\n397952  http://purl.obolibrary.org/obo/UBERON_0001572    MFMO_0000064   \n684507                                       t8572278  UBERON_0003707   \n693346                                       t8857485      CL_0000598   \n693347                                       t8900135      CL_0000740   \n693348                                       t8919740      CL_0001070   \n693349                                       t8939918      CL_0001070   \n693350                                       t8854625      CL_0002060   \n693351                                       t8936546      CL_1001571   \n704736                                       t8806170  UBERON_0002313   \n704737                                       t8804620  UBERON_0005368   \n704738                                       t8880341  UBERON_0005368   \n704739                                       t8900189  UBERON_0005368   \n704740                                       t8814331  UBERON_0005371   \n704741                                       t8844677  UBERON_0005371   \n704742                                       t8858250  UBERON_0005371   \n704743                                       t8880967  UBERON_0005371   \n704744                                       t8893399  UBERON_0005372   \n704745                                       t8924874  UBERON_0005372   \n704746                                       t8928599  UBERON_0005372   \n\n       ref_type   foreign_ns_id namespace ns_sep foreign_id  \n397952       id    MFMO_0000064      MFMO      _    0000064  \n684507       id  UBERON_0003707    UBERON      _    0003707  \n693346       id      CL_0000598        CL      _    0000598  \n693347       id      CL_0000740        CL      _    0000740  \n693348       id      CL_0001070        CL      _    0001070  \n693349       id      CL_0001070        CL      _    0001070  \n693350       id      CL_0002060        CL      _    0002060  \n693351       id      CL_1001571        CL      _    1001571  \n704736       id  UBERON_0002313    UBERON      _    0002313  \n704737       id  UBERON_0005368    UBERON      _    0005368  \n704738       id  UBERON_0005368    UBERON      _    0005368  \n704739       id  UBERON_0005368    UBERON      _    0005368  \n704740       id  UBERON_0005371    UBERON      _    0005371  \n704741       id  UBERON_0005371    UBERON      _    0005371  \n704742       id  UBERON_0005371    UBERON      _    0005371  \n704743       id  UBERON_0005371    UBERON      _    0005371  \n704744       id  UBERON_0005372    UBERON      _    0005372  \n704745       id  UBERON_0005372    UBERON      _    0005372  \n704746       id  UBERON_0005372    UBERON      _    0005372  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>p</th>\n      <th>s</th>\n      <th>o</th>\n      <th>ref_type</th>\n      <th>foreign_ns_id</th>\n      <th>namespace</th>\n      <th>ns_sep</th>\n      <th>foreign_id</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>397952</th>\n      <td>http://www.geneontology.org/formats/oboInOwl#h...</td>\n      <td>http://purl.obolibrary.org/obo/UBERON_0001572</td>\n      <td>MFMO_0000064</td>\n      <td>id</td>\n      <td>MFMO_0000064</td>\n      <td>MFMO</td>\n      <td>_</td>\n      <td>0000064</td>\n    </tr>\n    <tr>\n      <th>684507</th>\n      <td>http://www.geneontology.org/formats/oboInOwl#h...</td>\n      <td>t8572278</td>\n      <td>UBERON_0003707</td>\n      <td>id</td>\n      <td>UBERON_0003707</td>\n      <td>UBERON</td>\n      <td>_</td>\n      <td>0003707</td>\n    </tr>\n    <tr>\n      <th>693346</th>\n      <td>http://www.geneontology.org/formats/oboInOwl#h...</td>\n      <td>t8857485</td>\n      <td>CL_0000598</td>\n      <td>id</td>\n      <td>CL_0000598</td>\n      <td>CL</td>\n      <td>_</td>\n      <td>0000598</td>\n    </tr>\n    <tr>\n      <th>693347</th>\n      <td>http://www.geneontology.org/formats/oboInOwl#h...</td>\n      <td>t8900135</td>\n      <td>CL_0000740</td>\n      <td>id</td>\n      <td>CL_0000740</td>\n      <td>CL</td>\n      <td>_</td>\n      <td>0000740</td>\n    </tr>\n    <tr>\n      <th>693348</th>\n      <td>http://www.geneontology.org/formats/oboInOwl#h...</td>\n      <td>t8919740</td>\n      <td>CL_0001070</td>\n      <td>id</td>\n      <td>CL_0001070</td>\n      <td>CL</td>\n      <td>_</td>\n      <td>0001070</td>\n    </tr>\n    <tr>\n      <th>693349</th>\n      <td>http://www.geneontology.org/formats/oboInOwl#h...</td>\n      <td>t8939918</td>\n      <td>CL_0001070</td>\n      <td>id</td>\n      <td>CL_0001070</td>\n      <td>CL</td>\n      <td>_</td>\n      <td>0001070</td>\n    </tr>\n    <tr>\n      <th>693350</th>\n      <td>http://www.geneontology.org/formats/oboInOwl#h...</td>\n      <td>t8854625</td>\n      <td>CL_0002060</td>\n      <td>id</td>\n      <td>CL_0002060</td>\n      <td>CL</td>\n      <td>_</td>\n      <td>0002060</td>\n    </tr>\n    <tr>\n      <th>693351</th>\n      <td>http://www.geneontology.org/formats/oboInOwl#h...</td>\n      <td>t8936546</td>\n      <td>CL_1001571</td>\n      <td>id</td>\n      <td>CL_1001571</td>\n      <td>CL</td>\n      <td>_</td>\n      <td>1001571</td>\n    </tr>\n    <tr>\n      <th>704736</th>\n      <td>http://www.geneontology.org/formats/oboInOwl#h...</td>\n      <td>t8806170</td>\n      <td>UBERON_0002313</td>\n      <td>id</td>\n      <td>UBERON_0002313</td>\n      <td>UBERON</td>\n      <td>_</td>\n      <td>0002313</td>\n    </tr>\n    <tr>\n      <th>704737</th>\n      <td>http://www.geneontology.org/formats/oboInOwl#h...</td>\n      <td>t8804620</td>\n      <td>UBERON_0005368</td>\n      <td>id</td>\n      <td>UBERON_0005368</td>\n      <td>UBERON</td>\n      <td>_</td>\n      <td>0005368</td>\n    </tr>\n    <tr>\n      <th>704738</th>\n      <td>http://www.geneontology.org/formats/oboInOwl#h...</td>\n      <td>t8880341</td>\n      <td>UBERON_0005368</td>\n      <td>id</td>\n      <td>UBERON_0005368</td>\n      <td>UBERON</td>\n      <td>_</td>\n      <td>0005368</td>\n    </tr>\n    <tr>\n      <th>704739</th>\n      <td>http://www.geneontology.org/formats/oboInOwl#h...</td>\n      <td>t8900189</td>\n      <td>UBERON_0005368</td>\n      <td>id</td>\n      <td>UBERON_0005368</td>\n      <td>UBERON</td>\n      <td>_</td>\n      <td>0005368</td>\n    </tr>\n    <tr>\n      <th>704740</th>\n      <td>http://www.geneontology.org/formats/oboInOwl#h...</td>\n      <td>t8814331</td>\n      <td>UBERON_0005371</td>\n      <td>id</td>\n      <td>UBERON_0005371</td>\n      <td>UBERON</td>\n      <td>_</td>\n      <td>0005371</td>\n    </tr>\n    <tr>\n      <th>704741</th>\n      <td>http://www.geneontology.org/formats/oboInOwl#h...</td>\n      <td>t8844677</td>\n      <td>UBERON_0005371</td>\n      <td>id</td>\n      <td>UBERON_0005371</td>\n      <td>UBERON</td>\n      <td>_</td>\n      <td>0005371</td>\n    </tr>\n    <tr>\n      <th>704742</th>\n      <td>http://www.geneontology.org/formats/oboInOwl#h...</td>\n      <td>t8858250</td>\n      <td>UBERON_0005371</td>\n      <td>id</td>\n      <td>UBERON_0005371</td>\n      <td>UBERON</td>\n      <td>_</td>\n      <td>0005371</td>\n    </tr>\n    <tr>\n      <th>704743</th>\n      <td>http://www.geneontology.org/formats/oboInOwl#h...</td>\n      <td>t8880967</td>\n      <td>UBERON_0005371</td>\n      <td>id</td>\n      <td>UBERON_0005371</td>\n      <td>UBERON</td>\n      <td>_</td>\n      <td>0005371</td>\n    </tr>\n    <tr>\n      <th>704744</th>\n      <td>http://www.geneontology.org/formats/oboInOwl#h...</td>\n      <td>t8893399</td>\n      <td>UBERON_0005372</td>\n      <td>id</td>\n      <td>UBERON_0005372</td>\n      <td>UBERON</td>\n      <td>_</td>\n      <td>0005372</td>\n    </tr>\n    <tr>\n      <th>704745</th>\n      <td>http://www.geneontology.org/formats/oboInOwl#h...</td>\n      <td>t8924874</td>\n      <td>UBERON_0005372</td>\n      <td>id</td>\n      <td>UBERON_0005372</td>\n      <td>UBERON</td>\n      <td>_</td>\n      <td>0005372</td>\n    </tr>\n    <tr>\n      <th>704746</th>\n      <td>http://www.geneontology.org/formats/oboInOwl#h...</td>\n      <td>t8928599</td>\n      <td>UBERON_0005372</td>\n      <td>id</td>\n      <td>UBERON_0005372</td>\n      <td>UBERON</td>\n      <td>_</td>\n      <td>0005372</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_valid_pref[(all_valid_pref.ref_type == 'id') & (all_valid_pref.ns_sep == '_')]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "outputs": [],
   "source": [
    "df_uris = df1_obo[df1_obo.ref_type.isin(['uri', 'uri_obo'])].copy()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "outputs": [],
   "source": [
    "df_uris['domain'] = df_uris.o.apply(lambda x: re.match('https?://?(.*?)(/|$)', x).group(1))\n",
    "df_uris['domain2'] = df_uris.domain.apply(lambda x: re.sub('^www?\\d?\\.', '', x))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "outputs": [
    {
     "data": {
      "text/plain": "                            o\ndomain2                      \nen.wikipedia.org        17365\norcid.org                8085\nlangual.org              5592\nsnomedbrowser.com        4140\nlinkedlifedata.com       3092\n...                       ...\nhowmed.net                  1\nhindawi.com                 1\nhelda.helsinki.fi           1\nheartandmetabolism.org      1\niucnredlist.org             1\n\n[651 rows x 1 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>o</th>\n    </tr>\n    <tr>\n      <th>domain2</th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>en.wikipedia.org</th>\n      <td>17365</td>\n    </tr>\n    <tr>\n      <th>orcid.org</th>\n      <td>8085</td>\n    </tr>\n    <tr>\n      <th>langual.org</th>\n      <td>5592</td>\n    </tr>\n    <tr>\n      <th>snomedbrowser.com</th>\n      <td>4140</td>\n    </tr>\n    <tr>\n      <th>linkedlifedata.com</th>\n      <td>3092</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>howmed.net</th>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>hindawi.com</th>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>helda.helsinki.fi</th>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>heartandmetabolism.org</th>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>iucnredlist.org</th>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n<p>651 rows × 1 columns</p>\n</div>"
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_uris[['o', 'domain2']].groupby('domain2').agg('count').sort_values('o', ascending=False)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "outputs": [],
   "source": [
    "df_ids_unknown = df1_obo[(df1_obo.ref_type == 'id') & df1_obo.namespace.apply(lambda x: x.lower() not in prefixes)].copy()\n",
    "agg_ids_unknown = df_ids_unknown[['o', 'namespace']].groupby('namespace').agg('count')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "outputs": [
    {
     "data": {
      "text/plain": "3669249"
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg_ids_unknown.o.sum()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "Try Bioregistry for the still unresolved entities"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "outputs": [],
   "source": [
    "with open('data/bioregistry.yml') as fp:\n",
    "    bioregistry = yaml.safe_load(fp)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "outputs": [],
   "source": [
    "bioregistry_prefixes = dict()\n",
    "for key, ontology in bioregistry.items():\n",
    "    bioregistry_prefixes[ontology['preferred_prefix'].lower()] = key\n",
    "\n",
    "    if 'synonyms' in ontology:\n",
    "        for syn in ontology['synonyms']:\n",
    "            bioregistry_prefixes[syn.lower()] = key"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "outputs": [],
   "source": [
    "df_ids_unknown['ont_id'] = df_ids_unknown['namespace'].apply(lambda x: bioregistry_prefixes.get(x.lower(), ''))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "outputs": [],
   "source": [
    "agg_ids_unknown_found = df_ids_unknown[['o', 'ont_id', 'namespace']]\\\n",
    "    .groupby(['ont_id', 'namespace']).agg('count')\\\n",
    "    .reset_index()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "outputs": [
    {
     "data": {
      "text/plain": "                    o\nont_id               \n               655934\naeo                 3\nagricola         1112\narxiv               1\nbams             2234\n...               ...\nwormbase           13\nwwf.ecoregion      36\nymdb              102\nzfa                 1\nzfin              883\n\n[158 rows x 1 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>o</th>\n    </tr>\n    <tr>\n      <th>ont_id</th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th></th>\n      <td>655934</td>\n    </tr>\n    <tr>\n      <th>aeo</th>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>agricola</th>\n      <td>1112</td>\n    </tr>\n    <tr>\n      <th>arxiv</th>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>bams</th>\n      <td>2234</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>wormbase</th>\n      <td>13</td>\n    </tr>\n    <tr>\n      <th>wwf.ecoregion</th>\n      <td>36</td>\n    </tr>\n    <tr>\n      <th>ymdb</th>\n      <td>102</td>\n    </tr>\n    <tr>\n      <th>zfa</th>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>zfin</th>\n      <td>883</td>\n    </tr>\n  </tbody>\n</table>\n<p>158 rows × 1 columns</p>\n</div>"
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ids_unknown[['o', 'ont_id']].groupby(['ont_id']).agg('count')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "outputs": [
    {
     "data": {
      "text/plain": "{'comment': 'see comment here: https://github.com/obophenotype/ncbitaxon/issues/47',\n 'contributor': {'email': 'cthoyt@gmail.com',\n  'github': None,\n  'name': 'Charles Tapley Hoyt',\n  'orcid': '0000-0003-4423-4370'},\n 'description': 'Genetic code, mitochontrial genetic code, and other linked information to NCBI taxonomy entries.',\n 'example': '11',\n 'homepage': 'https://www.ncbi.nlm.nih.gov/Taxonomy/taxonomyhome.html',\n 'name': 'Genetic Code',\n 'pattern': '^\\\\d+$',\n 'preferred_prefix': 'gc',\n 'references': ['https://github.com/obophenotype/ncbitaxon/issues/47'],\n 'synonyms': ['gc_id'],\n 'uri_format': 'https://www.ncbi.nlm.nih.gov/Taxonomy/taxonomyhome.html/index.cgi?chapter=cgencodes#SG$1'}"
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bioregistry['gc']"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "outputs": [
    {
     "data": {
      "text/plain": "              o\nns_sep         \n:       2986743\n_         26572",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>o</th>\n    </tr>\n    <tr>\n      <th>ns_sep</th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>:</th>\n      <td>2986743</td>\n    </tr>\n    <tr>\n      <th>_</th>\n      <td>26572</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ids_unknown[df_ids_unknown.ont_id != ''][['o', 'ns_sep']].groupby('ns_sep').agg('count')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "Save found errors for further classification"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "outputs": [],
   "source": [
    "df_ids_unknown[df_ids_unknown.ont_id == ''][['s', 'p', 'o']].copy()\\\n",
    "    .reset_index()\\\n",
    "    .to_json('results/xrefs_unknown_id.json')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "outputs": [],
   "source": [
    "df_uris[['s', 'p', 'o']].copy()\\\n",
    "    .reset_index()\\\n",
    "    .to_json('results/xrefs_uri.json')\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}