Add_noise_to_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from SERGIO.sergio import sergio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create sim object\n",
    "sim = sergio(number_genes=5000, number_bins = 3, number_sc = 1500, noise_params = 1, \n",
    "             decays=0.8, sampling_state=15, noise_type='dpd')\n",
    "# read clean data\n",
    "expr_clean = pd.read_csv(\"./SERGIO_analyais/5000.gene.3ct.v3.clean.csv\", header = None).to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# expr_clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split to cell types\n",
    "expr = np.stack(np.hsplit(expr_clean, 3)) # 3 is the number of cell types (number_bins)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "expr_O = sim.outlier_effect(expr, outlier_prob = 0.01, mean = 0.8, scale = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Add Library Size Effect\n",
    "\"\"\"\n",
    "libFactor, expr_O_L = sim.lib_size_effect(expr_O, mean = 8, scale = 0.4)\n",
    "\n",
    "\"\"\"\n",
    "Add Dropouts\n",
    "\"\"\"\n",
    "binary_ind1 = sim.dropout_indicator(expr_O_L, shape = 6, percentile = 10)\n",
    "expr_O_L_D1 = np.multiply(binary_ind1, expr_O_L)\n",
    "binary_ind2 = sim.dropout_indicator(expr_O_L, shape = 6, percentile = 70)\n",
    "expr_O_L_D2 = np.multiply(binary_ind2, expr_O_L)\n",
    "binary_ind3 = sim.dropout_indicator(expr_O_L, shape = 6, percentile = 92)\n",
    "expr_O_L_D3 = np.multiply(binary_ind3, expr_O_L)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "count_matrix1 = sim.convert_to_UMIcounts(expr_O_L_D1)\n",
    "count_matrix_2d1 = np.concatenate(count_matrix1, axis = 1)\n",
    "\n",
    "count_matrix2 = sim.convert_to_UMIcounts(expr_O_L_D2)\n",
    "count_matrix_2d2 = np.concatenate(count_matrix2, axis = 1)\n",
    "\n",
    "count_matrix3 = sim.convert_to_UMIcounts(expr_O_L_D3)\n",
    "count_matrix_2d3 = np.concatenate(count_matrix3, axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.34540417777777777\n",
      "0.2157166222222222\n",
      "0.09069564444444445\n"
     ]
    }
   ],
   "source": [
    "print(np.count_nonzero(count_matrix1)/(3 * 5000 * 1500))\n",
    "print(np.count_nonzero(count_matrix2)/(3 * 5000 * 1500))\n",
    "print(np.count_nonzero(count_matrix3)/(3 * 5000 * 1500))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savetxt('./SERGIO_analyais/5000.gene.3ct.v3.dp1.csv', count_matrix_2d1, delimiter = ',', fmt='% 4d')\n",
    "np.savetxt('./SERGIO_analyais/5000.gene.3ct.v3.dp2.csv', count_matrix_2d2, delimiter = ',', fmt='% 4d')\n",
    "np.savetxt('./SERGIO_analyais/5000.gene.3ct.v3.dp3.csv', count_matrix_2d3, delimiter = ',', fmt='% 4d')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[2, 1, 2, ..., 0, 1, 1],\n",
       "       [1, 4, 2, ..., 0, 0, 1],\n",
       "       [1, 1, 0, ..., 0, 0, 0],\n",
       "       ...,\n",
       "       [0, 1, 1, ..., 1, 0, 1],\n",
       "       [0, 1, 2, ..., 0, 1, 0],\n",
       "       [3, 1, 1, ..., 1, 0, 0]])"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# count_matrix_2d2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create sim object for 5000 gene 4 ct\n",
    "sim = sergio(number_genes=5000, number_bins = 4, number_sc = 1000, noise_params = 1, \n",
    "             decays=0.8, sampling_state=15, noise_type='dpd')\n",
    "# read clean data\n",
    "expr_clean = pd.read_csv(\"./SERGIO_analyais/5000.gene.4ct.v3.dyn.clean.csv\", header = None).to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5000, 4000)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "expr_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split to cell types\n",
    "expr = np.stack(np.hsplit(expr_clean, 4)) # is the number of cell types (number_bins)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "expr_O = sim.outlier_effect(expr, outlier_prob = 0.01, mean = 0.8, scale = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Add Library Size Effect\n",
    "\"\"\"\n",
    "libFactor, expr_O_L = sim.lib_size_effect(expr_O, mean = 8, scale = 0.4)\n",
    "\n",
    "\"\"\"\n",
    "Add Dropouts\n",
    "\"\"\"\n",
    "binary_ind1 = sim.dropout_indicator(expr_O_L, shape = 6, percentile = 10)\n",
    "expr_O_L_D1 = np.multiply(binary_ind1, expr_O_L)\n",
    "binary_ind2 = sim.dropout_indicator(expr_O_L, shape = 6, percentile = 70)\n",
    "expr_O_L_D2 = np.multiply(binary_ind2, expr_O_L)\n",
    "binary_ind3 = sim.dropout_indicator(expr_O_L, shape = 6, percentile = 92)\n",
    "expr_O_L_D3 = np.multiply(binary_ind3, expr_O_L)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "count_matrix1 = sim.convert_to_UMIcounts(expr_O_L_D1)\n",
    "count_matrix_2d1 = np.concatenate(count_matrix1, axis = 1)\n",
    "\n",
    "count_matrix2 = sim.convert_to_UMIcounts(expr_O_L_D2)\n",
    "count_matrix_2d2 = np.concatenate(count_matrix2, axis = 1)\n",
    "\n",
    "count_matrix3 = sim.convert_to_UMIcounts(expr_O_L_D3)\n",
    "count_matrix_2d3 = np.concatenate(count_matrix3, axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.375133\n",
      "0.2089336\n",
      "0.1043842\n"
     ]
    }
   ],
   "source": [
    "print(np.count_nonzero(count_matrix1)/(4 * 5000 * 1000))\n",
    "print(np.count_nonzero(count_matrix2)/(4 * 5000 * 1000))\n",
    "print(np.count_nonzero(count_matrix3)/(4 * 5000 * 1000))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savetxt('./SERGIO_analyais/5000.gene.4ct.v3.dyn.dp1.csv', count_matrix_2d1, delimiter = ',', fmt='% 4d')\n",
    "np.savetxt('./SERGIO_analyais/5000.gene.4ct.v3.dyn.dp2.csv', count_matrix_2d2, delimiter = ',', fmt='% 4d')\n",
    "np.savetxt('./SERGIO_analyais/5000.gene.4ct.v3.dyn.dp3.csv', count_matrix_2d3, delimiter = ',', fmt='% 4d')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5000, 4000)"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# count_matrix_2d1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create sim object for 5000 gene 6 ct\n",
    "sim = sergio(number_genes=5000, number_bins = 6, number_sc = 800, noise_params = 1, \n",
    "             decays=0.8, sampling_state=15, noise_type='dpd')\n",
    "# read clean data\n",
    "expr_clean = pd.read_csv(\"./SERGIO_analyais/5000.gene.6ct.v3.dyn.clean.csv\", header = None).to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5000, 4800)"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "expr_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split to cell types\n",
    "expr = np.stack(np.hsplit(expr_clean, 6)) # is the number of cell types (number_bins)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "expr_O = sim.outlier_effect(expr, outlier_prob = 0.01, mean = 0.8, scale = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Add Library Size Effect\n",
    "\"\"\"\n",
    "libFactor, expr_O_L = sim.lib_size_effect(expr_O, mean = 8, scale = 0.4)\n",
    "\n",
    "\"\"\"\n",
    "Add Dropouts\n",
    "\"\"\"\n",
    "binary_ind1 = sim.dropout_indicator(expr_O_L, shape = 6, percentile = 10)\n",
    "expr_O_L_D1 = np.multiply(binary_ind1, expr_O_L)\n",
    "binary_ind2 = sim.dropout_indicator(expr_O_L, shape = 6, percentile = 70)\n",
    "expr_O_L_D2 = np.multiply(binary_ind2, expr_O_L)\n",
    "binary_ind3 = sim.dropout_indicator(expr_O_L, shape = 6, percentile = 92)\n",
    "expr_O_L_D3 = np.multiply(binary_ind3, expr_O_L)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "count_matrix1 = sim.convert_to_UMIcounts(expr_O_L_D1)\n",
    "count_matrix_2d1 = np.concatenate(count_matrix1, axis = 1)\n",
    "\n",
    "count_matrix2 = sim.convert_to_UMIcounts(expr_O_L_D2)\n",
    "count_matrix_2d2 = np.concatenate(count_matrix2, axis = 1)\n",
    "\n",
    "count_matrix3 = sim.convert_to_UMIcounts(expr_O_L_D3)\n",
    "count_matrix_2d3 = np.concatenate(count_matrix3, axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.37450979166666665\n",
      "0.20912708333333332\n",
      "0.10330295833333333\n"
     ]
    }
   ],
   "source": [
    "print(np.count_nonzero(count_matrix1)/(6 * 5000 * 800))\n",
    "print(np.count_nonzero(count_matrix2)/(6 * 5000 * 800))\n",
    "print(np.count_nonzero(count_matrix3)/(6 * 5000 * 800))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savetxt('./SERGIO_analyais/5000.gene.6ct.v3.dyn.dp1.csv', count_matrix_2d1, delimiter = ',', fmt='% 4d')\n",
    "np.savetxt('./SERGIO_analyais/5000.gene.6ct.v3.dyn.dp2.csv', count_matrix_2d2, delimiter = ',', fmt='% 4d')\n",
    "np.savetxt('./SERGIO_analyais/5000.gene.6ct.v3.dyn.dp3.csv', count_matrix_2d3, delimiter = ',', fmt='% 4d')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}