data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of interactions: 420534\n",
      "Total number of proteins: 17185\n"
     ]
    }
   ],
   "source": [
    "import gzip\n",
    "interactions = {}\n",
    "data = []\n",
    "# Load data\n",
    "org_id = '9606' # Change to 9606 for Human\n",
    "\n",
    "with gzip.open(f'data/{org_id}.protein.links.v11.0.txt.gz', 'rt') as f:\n",
    "    next(f) # Skip header\n",
    "    for line in f:\n",
    "        p1, p2, score = line.strip().split()\n",
    "        if float(score) < 700: # Filter high confidence interactions\n",
    "            continue\n",
    "        if p1 not in interactions:\n",
    "            interactions[p1] = set()\n",
    "        if p2 not in interactions:\n",
    "            interactions[p2] = set()\n",
    "        if p2 not in interactions[p1]:\n",
    "            interactions[p1].add(p2)\n",
    "            interactions[p2].add(p1)\n",
    "            data.append((p1, p2))\n",
    "\n",
    "print('Total number of interactions:', len(data))\n",
    "print('Total number of proteins:', len(interactions.keys()))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split training, validation and testing data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of training interactions: 269143\n",
      "Number of validation interactions: 67285\n",
      "Number of testing interactions: 84106\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import math\n",
    "\n",
    "np.random.seed(seed=0) # Fix random seed for reproducibility\n",
    "np.random.shuffle(data)\n",
    "train_n = int(math.ceil(len(data) * 0.8))\n",
    "valid_n = int(math.ceil(train_n * 0.8))\n",
    "train_data = data[:valid_n]\n",
    "valid_data = data[valid_n:train_n]\n",
    "test_data = data[train_n:]\n",
    "print('Number of training interactions:', len(train_data))\n",
    "print('Number of validation interactions:', len(valid_data))\n",
    "print('Number of testing interactions:', len(test_data))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save(filename, data):\n",
    "    with open(filename, 'w') as f:\n",
    "        for p1, p2 in data:\n",
    "            f.write(f'{p1}\\t{p2}\\n')\n",
    "            f.write(f'{p2}\\t{p1}\\n')\n",
    "\n",
    "save(f'data/train/{org_id}.protein.links.v11.0.txt', train_data)\n",
    "save(f'data/valid/{org_id}.protein.links.v11.0.txt', valid_data)\n",
    "save(f'data/test/{org_id}.protein.links.v11.0.txt', test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate negative interactions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of negative interactions: 119051\n",
      "Number of negative training interactions: 76193\n",
      "Number of negative validation interactions: 19048\n",
      "Number of negative testing interactions: 23810\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "proteins =set ()\n",
    "negatives = []\n",
    "for (p1,p2) in data:\n",
    "        proteins.add(p1)\n",
    "        proteins.add(p2)\n",
    "while len(negatives)<len(data):\n",
    "        s = random.sample(proteins, 2)\n",
    "        prot1= s[0]\n",
    "        prot2= s[1]\n",
    "        if (prot1,prot2) in negatives or (prot2,prot1) in negatives :\n",
    "                 continue\n",
    "        if prot1 not in interactions[prot2]:\n",
    "                 negatives.append((prot1, prot2))\n",
    "print('Total number of negative interactions:', len(negatives))\n",
    "# Split negative data\n",
    "neg_train_data = negatives[:valid_n]\n",
    "neg_valid_data = negatives[valid_n:train_n]\n",
    "neg_test_data = negatives[train_n:]\n",
    "print('Number of negative training interactions:', len(neg_train_data))\n",
    "print('Number of negative validation interactions:', len(neg_valid_data))\n",
    "print('Number of negative testing interactions:', len(neg_test_data))\n",
    "# Save negative data \n",
    "save(f'data/train/{org_id}.negative_interactions.txt', neg_train_data)\n",
    "save(f'data/valid/{org_id}.negative_interactions.txt', neg_valid_data)\n",
    "save(f'data/test/{org_id}.negative_interactions.txt', neg_test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocess GO annotations\n",
    "### Load id mapping between annotation database and StringDB IDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded mappings 10616\n"
     ]
    }
   ],
   "source": [
    "mapping = {}\n",
    "source = {'4932': 'SGD_ID', '9606': 'Ensembl_UniProt_AC'} # mapping source\n",
    "\n",
    "with gzip.open(f'data/{org_id}.protein.aliases.v11.0.txt.gz', 'rt') as f:\n",
    "    next(f) # Skip header\n",
    "    for line in f:\n",
    "        string_id, p_id, sources = line.strip().split('\\t')\n",
    "        if source[org_id] not in sources.split():\n",
    "            continue\n",
    "        if p_id not in mapping:\n",
    "            mapping[p_id] = set()\n",
    "        mapping[p_id].add(string_id)\n",
    "print('Loaded mappings', len(mapping))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of annotations: 40577\n"
     ]
    }
   ],
   "source": [
    "gaf_files = {'4932': 'sgd.gaf.gz', '9606': 'goa_human.gaf.gz'}\n",
    "annotations = set()\n",
    "with gzip.open(f'data/{gaf_files[org_id]}', 'rt') as f:\n",
    "    for line in f:\n",
    "        if line.startswith('!'): # Skip header\n",
    "            continue\n",
    "        it = line.strip().split('\\t')\n",
    "        p_id = it[1]\n",
    "        go_id = it[4]\n",
    "        if it[6] == 'IEA' or it[6] == 'ND': # Ignore predicted or no data annotations\n",
    "            continue\n",
    "        if p_id not in mapping: # Not in StringDB\n",
    "            continue\n",
    "        s_ids = mapping[p_id]\n",
    "        for s_id in s_ids:\n",
    "            annotations.add((s_id, go_id))\n",
    "print('Number of annotations:', len(annotations))\n",
    "\n",
    "# Save annotations\n",
    "with open(f'data/train/{org_id}.annotation.txt', 'w') as f:\n",
    "    for p_id, go_id in annotations:\n",
    "        f.write(f'{p_id}\\t{go_id}\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate Plain Training Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "287137 data/train/4932.plain.nt\r\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "tdf = open(f'data/train/{org_id}.plain.nt', 'w')\n",
    "# Load GO\n",
    "with open('data/go.obo') as f:\n",
    "    tid = ''\n",
    "    for line in f:\n",
    "        line = line.strip()\n",
    "        if line.startswith('id:'):\n",
    "            tid = line[4:]\n",
    "        if not tid.startswith('GO:'):\n",
    "            continue\n",
    "        if line.startswith('is_a:'):\n",
    "            tid2 = line[6:].split(' ! ')[0]\n",
    "            tdf.write(f'<http://{tid}> <http://is_a> <http://{tid2}> .\\n')\n",
    "        if line.startswith('relationship:'):\n",
    "            it = line[14:].split(' ! ')[0].split()\n",
    "            tdf.write(f'<http://{tid}> <http://{it[0]}> <http://{it[1]}> .\\n')\n",
    "\n",
    "# Load interactions\n",
    "with open(f'data/train/{org_id}.protein.links.v11.0.txt') as f:\n",
    "    for line in f:\n",
    "        it = line.strip().split()\n",
    "        tdf.write(f'<http://{it[0]}> <http://interacts> <http://{it[1]}> .\\n')\n",
    "\n",
    "# Load annotations\n",
    "with open(f'data/train/{org_id}.annotation.txt') as f:\n",
    "    for line in f:\n",
    "        it = line.strip().split()\n",
    "        tdf.write(f'<http://{it[0]}> <http://hasFunction> <http://{it[1]}> .\\n')\n",
    "\n",
    "tdf.close()\n",
    "if not os.path.exists('data/transe'):\n",
    "    os.makedirs('data/transe')\n",
    "! wc -l 'data/train/{org_id}.plain.nt'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate Classes Training Data for ELEmbeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "error: Missing required options: g, o\r\n",
      "usage: groovy\r\n",
      " -g,--input <arg>    Gene Ontology file\r\n",
      " -h,--help           this information\r\n",
      " -i,--input <arg>    input STRING file\r\n",
      " -o,--output <arg>   output file containing generated ontology\r\n"
     ]
    }
   ],
   "source": [
    "if not os.path.exists('data/elembeddings'):\n",
    "    os.makedirs('data/elembeddings')\n",
    "! groovy el-embeddings/GenerateTrainingDataClasses -i 'data/train/{org_id}.protein.links.v11.0.txt' -a 'data/train/{org_id}.annotation.txt' -o 'data/train/{org_id}.classes.owl'\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Normalize training data classes into four normal forms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SLF4J: Failed to load class \"org.slf4j.impl.StaticLoggerBinder\".\n",
      "SLF4J: Defaulting to no-operation (NOP) logger implementation\n",
      "SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.\n",
      "Ignoring SubObjectPropertyOf*(8 7):The translation map is incomplete. Item id was not found: '8'.\n",
      "Ignoring SubObjectPropertyOf*(10 12):The translation map is incomplete. Item id was not found: '10'.\n",
      "Ignoring SubObjectPropertyOf*(11 12):The translation map is incomplete. Item id was not found: '11'.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "! groovy -cp el-embeddings/jar/jcel.jar el-embeddings/Normalizer.groovy -i 'data/train/{org_id}.classes.owl' -o 'data/train/{org_id}.classes-normalized.owl'\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate RDF Representation of ELEmbeddings training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rapper: Parsing URI file:///home/kulmanm/KAUST/CBRC/machine-learning-with-ontologies/data/train/9606.classes.owl with parser rdfxml\n",
      "rapper: Serializing with serializer ntriples\n",
      "rapper: Parsing returned 4368800 triples\n"
     ]
    }
   ],
   "source": [
    "\n",
    "! rapper 'data/train/{org_id}.classes.owl' -o ntriples > 'data/train/{org_id}.classes-rdf.nt'\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Generate Onto/OPA2Vec-compatible associations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "# generate OPA2VEC compatible associations\n",
    "with open(f'data/train/{org_id}.OPA_associations.txt', 'w') as f:\n",
    "    for p_id, go_id in annotations:\n",
    "        go_num = re.split ('[A-Z]|:+',go_id)\n",
    "        f.write (str(p_id)+\" \\t\"+\"<http://purl.obolibrary.org/obo/GO_\"+str(go_num[3])+\">\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}