plot_figures_synthdata.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import cm\n",
    "from matplotlib.colors import to_rgba\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "import matplotlib\n",
    "\n",
    "matplotlib.rc('font', size=14)\n",
    "matplotlib.rc('axes', titlesize=14)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib notebook\n",
    "\n",
    "path = \"results/\"\n",
    "#metrics = [\"Accuracy\", \"Macro F1\", \"Micro F1\", \"Log loss\", \"Brier score\"]\n",
    "metrics = [\"Accuracy\", \"F1\", \"F1\", \"Log loss\", \"Brier score\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fig A\n",
    "## Setting\n",
    "To control:\n",
    "- generative model: simple or complex\n",
    "- sampling or deterministic\n",
    "  - No. of dataset already computed\n",
    "- Real model result's as maximum or not"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "css = np.array([2,3,4,5])\n",
    "wsp = np.array([0.5,1,2,5])\n",
    "n_models = 30\n",
    "n_datasets_gen = 30\n",
    "n_datasets = 30\n",
    "\n",
    "#kdb=1; gm=\"sg\" #simple gen model\n",
    "kdb=4; gm=\"cg\" #complex gen model\n",
    "n_rep = 1; directory = \"figA_determ/\"\n",
    "#n_rep = 5; directory = \"figA_sampling/\"\n",
    "#prev_name=\"Diff. \"; real_tsdata = -1 # any from 0 to 4 ; -1 means no max.\n",
    "prev_name=\"Rel. diff. \"; real_tsdata = 0 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collect results with only fully labeled data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mResultsFull = np.loadtxt(path+'res_exp_figA_'+gm+'_fully_labeled.resout', delimiter=\",\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collect results with real model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mResultsReal = np.loadtxt(path+'res_exp_figA_'+gm+'_real_model.resout', delimiter=\",\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collect data for `honest` scenario"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Z1_complete = []\n",
    "s = 0\n",
    "\n",
    "for m in np.arange(n_models):\n",
    "    for d in np.arange(n_datasets):\n",
    "        id_mrf = m*n_datasets_gen+d\n",
    "        id_mrr = (m*n_datasets_gen+d)*5+real_tsdata\n",
    "        for c in css:\n",
    "            orig_data = np.loadtxt(path+directory+'res_exp_figA_m_' + str(m) + \n",
    "                                                                '_d_' + str(d) + \n",
    "                                                                '_css_' + str(c) + \n",
    "                                                                '_k_' + str(kdb) + \n",
    "                                                                '_s_' + str(s) + \n",
    "                                                                 '.csv', delimiter=\",\")\n",
    "            for v in np.arange(len(wsp)):\n",
    "                vals = orig_data[np.arange(v, orig_data.shape[0], len(wsp)),:]\n",
    "                vals = np.mean(vals, axis=0) - mResultsFull[id_mrf] # mean over repetitions and remove real values\n",
    "                if real_tsdata >= 0: vals /= (mResultsReal[id_mrr] - mResultsFull[id_mrf])\n",
    "                Z1_complete.append( vals ) \n",
    "Z1_complete = np.array(Z1_complete)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collect data for `misleading` scenario"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Z2_complete = []\n",
    "s = 1\n",
    "\n",
    "for m in np.arange(n_models):\n",
    "    for d in np.arange(n_datasets):\n",
    "        id_mrf = m*n_datasets_gen+d\n",
    "        id_mrr = (m*n_datasets_gen+d)*5+real_tsdata\n",
    "        for c in css:\n",
    "            orig_data = np.loadtxt(path+directory+'res_exp_figA_m_' + str(m) + \n",
    "                                                                '_d_' + str(d) + \n",
    "                                                                '_css_' + str(c) + \n",
    "                                                                '_k_' + str(kdb) + \n",
    "                                                                '_s_' + str(s) + \n",
    "                                                                 '.csv', delimiter=\",\")\n",
    "            for v in np.arange(len(wsp)):\n",
    "                vals = orig_data[np.arange(v, orig_data.shape[0], len(wsp)),:]\n",
    "                vals = np.mean(vals, axis=0) - mResultsFull[id_mrf] # mean over repetitions and remove real values\n",
    "                if real_tsdata >= 0: vals /= (mResultsReal[id_mrr] - mResultsFull[id_mrf])\n",
    "                Z2_complete.append( vals ) \n",
    "Z2_complete = np.array(Z2_complete)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib widget\n",
    "## 0:Acc, 1:Macro F1, 2:Micro F1, 3:log loss, 4:Brier score\n",
    "metric_ind = 1\n",
    "transparencia = 0.6\n",
    "\n",
    "\n",
    "Z1 = Z1_complete[:,metric_ind]\n",
    "Z1.shape = (n_models*n_datasets,len(css)*len(wsp))\n",
    "Z1 = Z1.mean(axis=0)\n",
    "Z1.shape = (len(css),len(wsp))\n",
    "Z1 = np.transpose(Z1)\n",
    "Z1p = (Z1 - np.min(Z1))/(np.max(Z1) - np.min(Z1))\n",
    "\n",
    "Z2 = Z2_complete[:,metric_ind]\n",
    "Z2.shape = (n_models*n_datasets,len(css)*len(wsp))\n",
    "Z2 = Z2.mean(axis=0)\n",
    "Z2.shape = (len(css),len(wsp))\n",
    "Z2 = np.transpose(Z2)\n",
    "Z2p = (Z2 - np.min(Z2))/(np.max(Z2) - np.min(Z2))\n",
    "\n",
    "\n",
    "x = css\n",
    "y_labels = wsp\n",
    "y = np.arange(len(y_labels))\n",
    "X, Y = np.meshgrid(x, y)\n",
    "\n",
    "\n",
    "C1 = np.empty_like(Z1, dtype=object)\n",
    "C2 = np.empty_like(Z2, dtype=object)\n",
    "cmap1 = plt.get_cmap(\"spring\")\n",
    "cmap2 = plt.get_cmap(\"winter\")\n",
    "\n",
    "for i in range(Z1.shape[0]):\n",
    "    for j in range(Z1.shape[1]):\n",
    "        C1[i,j] = to_rgba(cmap1(Z1p[i,j]), transparencia)\n",
    "        C2[i,j] = to_rgba(cmap2(Z2p[i,j]), transparencia)\n",
    "\n",
    "\n",
    "# Create a transparent bridge region\n",
    "X_bridge = np.vstack([X[-1,:],X[-1,:]])\n",
    "Y_bridge = np.vstack([Y[-1,:],Y[-1,:]])\n",
    "Z_bridge = np.vstack([Z1[-1,:],Z2[-1,:]])\n",
    "color_bridge = np.empty_like(Z_bridge, dtype=object)\n",
    "\n",
    "color_bridge.fill((1,1,1,0)) # RGBA colour, onlt the last component matters.\n",
    "\n",
    "# Join the two surfaces flipping one of them (using also the bridge)\n",
    "X_full = np.vstack([X, X_bridge, np.flipud(X)])\n",
    "Y_full = np.vstack([Y, Y_bridge, np.flipud(Y)])\n",
    "Z_full = np.vstack([Z1, Z_bridge, np.flipud(Z2)])\n",
    "color_full = np.vstack([C1, color_bridge, np.flipud(C2)])\n",
    "\n",
    "\n",
    "### print figure\n",
    "fig = plt.figure(figsize=(10,5))\n",
    "ax = fig.gca(projection='3d')\n",
    "\n",
    "surf_full = ax.plot_surface(X_full, Y_full, Z_full, rstride=1, cstride=1,\n",
    "                            facecolors=color_full, linewidth=0,\n",
    "                            antialiased=False, shade=False)\n",
    "\n",
    "ax.set_xlabel(\"Size of the candidate sets\")\n",
    "ax.set_ylabel(\"Prop. weakly labeled ex.\")\n",
    "ax.set_zlabel(prev_name+metrics[metric_ind])\n",
    "\n",
    "ax.set_xticks(x)\n",
    "ax.set_xticklabels(x)\n",
    "ax.set_yticks(y)\n",
    "ax.set_yticklabels(y_labels)\n",
    "\n",
    "ax.text(3, 3, Z1[0,-1], \"Honest\", zdir=\"x\",\n",
    "        color=cmap1(Z1p[0,-1]))\n",
    "\n",
    "ax.text(0, 3, Z2[-1,-1], \"Misleading\", zdir=\"x\",\n",
    "        color=cmap2(Z2p[0,-1]))\n",
    "\n",
    "\n",
    "# manipulate view\n",
    "ax.view_init(20, -60)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FigB\n",
    "\n",
    "## Setting\n",
    "To control:\n",
    "- generative model: simple or complex\n",
    "- sampling or deterministic\n",
    "  - No. of dataset already computed\n",
    "- Real model result's as maximum or not"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ind_fss = np.arange(10)#np.array([0,1,3,6,9])\n",
    "fss = np.round((1+np.arange(10))*(100.0/3)).astype(int)\n",
    "fss = fss[ind_fss]\n",
    "n_fss = 10\n",
    "wsp = np.array([0.5,1,2,5])\n",
    "n_models = 30\n",
    "n_datasets_gen = 30\n",
    "n_datasets = 30\n",
    "\n",
    "#kdb=1; gm=\"sg\" #simple gen model\n",
    "kdb=4; gm=\"cg\" #complex gen model\n",
    "n_rep = 1; directory = \"figB_determ/\"\n",
    "#n_rep = 5; directory = \"figB_sampling/\"\n",
    "#prev_name=\"Diff. \"; real_tsdata = -1 # any from 0 to 4 ; -1 means no max.\n",
    "prev_name=\"Rel. diff. \"; real_tsdata = 0 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collect results with only fully labeled data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mResultsFull = np.loadtxt(path+'res_exp_figB_'+gm+'_fully_labeled.resout', delimiter=\",\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collect results with real model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mResultsReal = np.loadtxt(path+'res_exp_figB_'+gm+'_real_model.resout', delimiter=\",\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collect data for `honest` scenario"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "Z1_complete = []\n",
    "s = 0\n",
    "\n",
    "for m in np.arange(n_models):\n",
    "    for d in np.arange(n_datasets):\n",
    "        for f in ind_fss:\n",
    "            orig_data = np.loadtxt(path+directory+'res_exp_figB_m_' + str(m) + \n",
    "                                                                '_d_' + str(d) + \n",
    "                                                                '_fss_' + str(f) + \n",
    "                                                                '_k_' + str(kdb) + \n",
    "                                                                '_s_' + str(s) + \n",
    "                                                                 '.csv', delimiter=\",\")\n",
    "            id_mrf = (m*n_datasets_gen+d)*n_fss+f\n",
    "            id_mrr = ((m*n_datasets_gen+d)*n_fss+f)*5+real_tsdata\n",
    "            for v in np.arange(len(wsp)):\n",
    "                vals = orig_data[np.arange(v, orig_data.shape[0], len(wsp)),:]\n",
    "                vals = np.mean(vals, axis=0) - mResultsFull[id_mrf] # mean over repetitions and remove real values\n",
    "                if real_tsdata >= 0: vals /= (mResultsReal[id_mrr] - mResultsFull[id_mrf])\n",
    "                Z1_complete.append( vals ) \n",
    "Z1_complete = np.array(Z1_complete)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collect data for `misleading` scenario"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Z2_complete = []\n",
    "s = 1\n",
    "\n",
    "for m in np.arange(n_models):\n",
    "    for d in np.arange(n_datasets):\n",
    "        for f in ind_fss:\n",
    "            orig_data = np.loadtxt(path+directory+'res_exp_figB_m_' + str(m) + \n",
    "                                                                '_d_' + str(d) + \n",
    "                                                                '_fss_' + str(f) + \n",
    "                                                                '_k_' + str(kdb) + \n",
    "                                                                '_s_' + str(s) + \n",
    "                                                                 '.csv', delimiter=\",\")\n",
    "            id_mrf = (m*n_datasets_gen+d)*n_fss+f\n",
    "            id_mrr = ((m*n_datasets_gen+d)*n_fss+f)*5+real_tsdata\n",
    "            for v in np.arange(len(wsp)):\n",
    "                vals = orig_data[np.arange(v, orig_data.shape[0], len(wsp)),:]\n",
    "                vals = np.mean(vals, axis=0) - mResultsFull[id_mrf] # mean over repetitions and remove real values\n",
    "                if real_tsdata >= 0: vals /= (mResultsReal[id_mrr] - mResultsFull[id_mrf])\n",
    "                Z2_complete.append( vals ) \n",
    "            #mResults.append( final_data )\n",
    "Z2_complete = np.array(Z2_complete)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib widget\n",
    "## 0:Acc, 1:Macro F1, 2:Micro F1, 3:log loss, 4:Brier score\n",
    "metric_ind = 4\n",
    "transparencia = 0.6\n",
    "\n",
    "\n",
    "Z1 = Z1_complete[:,metric_ind]\n",
    "Z1.shape = (n_models*n_datasets,len(fss)*len(wsp))\n",
    "Z1 = Z1.mean(axis=0)\n",
    "Z1.shape = (len(fss),len(wsp))\n",
    "Z1 = np.transpose(Z1)\n",
    "Z1p = (Z1 - np.min(Z1))/(np.max(Z1) - np.min(Z1))\n",
    "\n",
    "Z2 = Z2_complete[:,metric_ind]\n",
    "Z2.shape = (n_models*n_datasets,len(fss)*len(wsp))\n",
    "Z2 = Z2.mean(axis=0)\n",
    "Z2.shape = (len(fss),len(wsp))\n",
    "Z2 = np.transpose(Z2)\n",
    "Z2p = (Z2 - np.min(Z2))/(np.max(Z2) - np.min(Z2))\n",
    "\n",
    "\n",
    "x = np.arange(len(fss))\n",
    "x_labels = fss\n",
    "y = np.arange(len(wsp))\n",
    "y_labels = wsp\n",
    "\n",
    "X, Y = np.meshgrid(x, y)\n",
    "\n",
    "\n",
    "C1 = np.empty_like(Z1, dtype=object)\n",
    "C2 = np.empty_like(Z2, dtype=object)\n",
    "cmap1 = plt.get_cmap(\"spring\")\n",
    "cmap2 = plt.get_cmap(\"winter\")\n",
    "\n",
    "for i in range(Z1.shape[0]):\n",
    "    for j in range(Z1.shape[1]):\n",
    "        C1[i,j] = to_rgba(cmap1(Z1p[i,j]), transparencia)\n",
    "        C2[i,j] = to_rgba(cmap2(Z2p[i,j]), transparencia)\n",
    "\n",
    "\n",
    "# Create a transparent bridge region\n",
    "X_bridge = np.vstack([X[-1,:],X[-1,:]])\n",
    "Y_bridge = np.vstack([Y[-1,:],Y[-1,:]])\n",
    "Z_bridge = np.vstack([Z1[-1,:],Z2[-1,:]])\n",
    "color_bridge = np.empty_like(Z_bridge, dtype=object)\n",
    "\n",
    "color_bridge.fill((1,1,1,0)) # RGBA colour, onlt the last component matters.\n",
    "\n",
    "# Join the two surfaces flipping one of them (using also the bridge)\n",
    "X_full = np.vstack([X, X_bridge, np.flipud(X)])\n",
    "Y_full = np.vstack([Y, Y_bridge, np.flipud(Y)])\n",
    "Z_full = np.vstack([Z1, Z_bridge, np.flipud(Z2)])\n",
    "color_full = np.vstack([C1, color_bridge, np.flipud(C2)])\n",
    "\n",
    "\n",
    "### print figure\n",
    "fig = plt.figure(figsize=(10,5))\n",
    "ax = fig.gca(projection='3d')\n",
    "\n",
    "surf_full = ax.plot_surface(X_full, Y_full, Z_full, rstride=1, cstride=1,\n",
    "                            facecolors=color_full, linewidth=0,\n",
    "                            antialiased=False, shade=False)\n",
    "\n",
    "ax.set_xlabel(\"Size of the fully labeled set\")\n",
    "ax.set_ylabel(\"Prop. weakly labeled ex.\")\n",
    "ax.set_zlabel(prev_name+metrics[metric_ind])\n",
    "\n",
    "#fig.colorbar(surf_full, shrink=0.5, aspect=25, label=\"DAV\")\n",
    "ax.set_xticks(x)\n",
    "plt.xticks(rotation = 45)\n",
    "ax.set_xticklabels(x_labels)\n",
    "ax.set_yticks(y)\n",
    "ax.set_yticklabels(y_labels)\n",
    "\n",
    "ax.text(3, 3, Z1[3,3], \"Honest\", zdir=\"x\",\n",
    "        color=cmap1(Z1p[0,-1]))\n",
    "\n",
    "ax.text(3, 3, Z2[3,3], \"Misleading\", zdir=\"x\",\n",
    "        color=cmap2(Z2p[0,-1]))\n",
    "\n",
    "# manipulate view\n",
    "ax.view_init(10, -80)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FigC-1\n",
    "\n",
    "## Setting\n",
    "To control:\n",
    "- generative model: simple or complex\n",
    "- sampling or deterministic\n",
    "  - No. of dataset already computed (different from FigA and FigC)\n",
    "- Real model result's as maximum or not"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "css = np.array([2,3,4,5])\n",
    "amb = np.array([0.25,0.5,0.75,1.0])\n",
    "wsp_ind = 2; wsp = np.array([0.5,1,2,5])\n",
    "n_models = 30\n",
    "n_datasets_gen = 30\n",
    "n_datasets_Z1 = 30 \n",
    "n_datasets_Z2 = 30\n",
    "\n",
    "#kdb=1; gm=\"sg\" #simple gen model\n",
    "kdb=4; gm=\"cg\" #complex gen model\n",
    "n_rep = 1; directory_Z1 = \"figC_determ/\"; directory_Z2 = \"figA_determ/\"\n",
    "#n_rep = 5; directory_Z1 = \"figC_sampling/\"; directory_Z2 = \"figA_sampling/\"\n",
    "#prev_name=\"Diff. \"; real_tsdata = -1 # any from 0 to 4 ; -1 means no max.\n",
    "prev_name=\"Rel. diff. \"; real_tsdata = 0 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collect results with only fully labeled data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mResultsFull = np.loadtxt(path+'res_exp_figA_'+gm+'_fully_labeled.resout', delimiter=\",\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collect results with real model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mResultsReal = np.loadtxt(path+'res_exp_figA_'+gm+'_real_model.resout', delimiter=\",\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collect data for `consistent-labeling` scenario"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Z1_complete = []\n",
    "\n",
    "for m in np.arange(n_models):\n",
    "    for d in np.arange(n_datasets_Z1):\n",
    "        id_mrf = m*n_datasets_gen+d\n",
    "        id_mrr = (m*n_datasets_gen+d)*5+real_tsdata\n",
    "        for c in css:\n",
    "            for a in np.arange(len(amb))+1:\n",
    "                orig_data = np.loadtxt(path+directory_Z1+'res_exp_figC_m_' + str(m) + \n",
    "                                                                    '_d_' + str(d) + \n",
    "                                                                    '_css_' + str(c) + \n",
    "                                                                    '_k_' + str(kdb) + \n",
    "                                                                    '_s_' + str(a) + \n",
    "                                                                     '.csv', delimiter=\",\")\n",
    "                vals = orig_data[np.arange(wsp_ind, orig_data.shape[0], len(wsp)),:]\n",
    "                vals = np.mean(vals, axis=0) - mResultsFull[id_mrf] # mean over repetitions and remove real values\n",
    "                if real_tsdata >= 0: vals /= (mResultsReal[id_mrr] - mResultsFull[id_mrf])\n",
    "                Z1_complete.append( vals ) \n",
    "Z1_complete = np.array(Z1_complete)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collect data for `inconsistent-labeling` scenario (baseline)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Z2_complete = []\n",
    "s = 0 # honest scenario\n",
    "\n",
    "for m in np.arange(n_models):\n",
    "    for d in np.arange(n_datasets_Z2):\n",
    "        id_mrf = m*n_datasets_gen+d\n",
    "        id_mrr = (m*n_datasets_gen+d)*5+real_tsdata\n",
    "        for c in css:\n",
    "            orig_data = np.loadtxt(path+directory_Z2+'res_exp_figC_m_' + str(m) + \n",
    "                                                                '_d_' + str(d) + \n",
    "                                                                '_css_' + str(c) + \n",
    "                                                                '_k_' + str(kdb) + \n",
    "                                                                '_s_' + str(s) + \n",
    "                                                                 '.csv', delimiter=\",\")\n",
    "            vals = orig_data[np.arange(wsp_ind, orig_data.shape[0], len(wsp)),:]\n",
    "            vals = np.mean(vals, axis=0) - mResultsFull[id_mrf] # mean over repetitions and remove real values\n",
    "            if real_tsdata >= 0: vals /= (mResultsReal[id_mrr] - mResultsFull[id_mrf])\n",
    "            Z2_complete.append( vals ) \n",
    "Z2_complete = np.array(Z2_complete)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib widget\n",
    "## 0:Acc, 1:Macro F1, 2:Micro F1, 3:log loss, 4:Brier score\n",
    "metric_ind = 1\n",
    "transparencia = 0.6\n",
    "\n",
    "\n",
    "Z1 = Z1_complete[:,metric_ind]\n",
    "Z1.shape = (n_models*n_datasets_Z1,len(css)*len(amb))\n",
    "Z1 = Z1.mean(axis=0)\n",
    "Z1.shape = (len(css),len(amb))\n",
    "Z1 = np.transpose(Z1)\n",
    "Z1p = (Z1 - np.min(Z1))/(np.max(Z1) - np.min(Z1))\n",
    "\n",
    "Z2 = Z2_complete[:,metric_ind]\n",
    "Z2.shape = (n_models*n_datasets_Z2,len(css))\n",
    "Z2 = Z2.mean(axis=0)\n",
    "Z2 = np.repeat(Z2, len(amb)) # dimensions of Z1 and Z2 are different!!! dim(Z1) = dim(Z2)*4\n",
    "Z2.shape = (len(css),len(amb))\n",
    "Z2 = np.transpose(Z2)\n",
    "Z2p = (Z2 - np.min(Z2))/(np.max(Z2) - np.min(Z2))\n",
    "\n",
    "\n",
    "x = np.arange(len(css))\n",
    "x_labels = css\n",
    "y = np.arange(len(amb))\n",
    "y_labels = amb\n",
    "\n",
    "X, Y = np.meshgrid(x, y)\n",
    "\n",
    "\n",
    "C1 = np.empty_like(Z1, dtype=object)\n",
    "C2 = np.empty_like(Z2, dtype=object)\n",
    "cmap1 = plt.get_cmap(\"spring\")\n",
    "cmap2 = plt.get_cmap(\"winter\")\n",
    "\n",
    "for i in range(Z1.shape[0]):\n",
    "    for j in range(Z1.shape[1]):\n",
    "        C1[i,j] = to_rgba(cmap1(Z1p[i,j]), transparencia)\n",
    "        C2[i,j] = to_rgba(cmap2(Z2p[i,j]), transparencia)\n",
    "\n",
    "\n",
    "# Create a transparent bridge region\n",
    "X_bridge = np.vstack([X[-1,:],X[-1,:]])\n",
    "Y_bridge = np.vstack([Y[-1,:],Y[-1,:]])\n",
    "Z_bridge = np.vstack([Z1[-1,:],Z2[-1,:]])\n",
    "color_bridge = np.empty_like(Z_bridge, dtype=object)\n",
    "\n",
    "color_bridge.fill((1,1,1,0)) # RGBA colour, onlt the last component matters.\n",
    "\n",
    "# Join the two surfaces flipping one of them (using also the bridge)\n",
    "X_full = np.vstack([X, X_bridge, np.flipud(X)])\n",
    "Y_full = np.vstack([Y, Y_bridge, np.flipud(Y)])\n",
    "Z_full = np.vstack([Z1, Z_bridge, np.flipud(Z2)])\n",
    "color_full = np.vstack([C1, color_bridge, np.flipud(C2)])\n",
    "\n",
    "\n",
    "### print figure\n",
    "fig = plt.figure(figsize=(10,5))\n",
    "ax = fig.gca(projection='3d')\n",
    "\n",
    "surf_full = ax.plot_surface(X_full, Y_full, Z_full, rstride=1, cstride=1,\n",
    "                            facecolors=color_full, linewidth=0,\n",
    "                            antialiased=False, shade=False)\n",
    "\n",
    "ax.set_xlabel(\"Size of candidate sets\")\n",
    "ax.set_ylabel(\"Prob. of co-occurrence\")\n",
    "ax.set_zlabel(prev_name+metrics[metric_ind])\n",
    "\n",
    "ax.set_xticks(x)\n",
    "ax.set_xticklabels(x_labels)\n",
    "ax.set_yticks(y)\n",
    "ax.set_yticklabels(y_labels)\n",
    "\n",
    "ax.text(3, 3, Z1[-1,0], \"With co-occurrence\", zdir=\"x\",\n",
    "        color=cmap1(Z1p[0,-1]))\n",
    "\n",
    "ax.text(3, 3, Z2[-1,0], \"Without\", zdir=\"x\",\n",
    "        color=cmap2(Z2p[0,-1]))\n",
    "\n",
    "\n",
    "# manipulate view\n",
    "ax.view_init(20, 30)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FigC-2\n",
    "\n",
    "## Setting\n",
    "To control:\n",
    "- generative model: simple or complex\n",
    "- sampling or deterministic\n",
    "  - No. of dataset already computed (different from FigA and FigC)\n",
    "- Real model result's as maximum or not"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "css_ind = 0; css = np.array([2,3,4,5])\n",
    "amb = np.array([0.25,0.5,0.75,1.0])\n",
    "wsp = np.array([0.5,1,2,5])\n",
    "n_models = 30\n",
    "n_datasets_gen = 30\n",
    "n_datasets_Z1 = 30\n",
    "n_datasets_Z2 = 30\n",
    "\n",
    "kdb=1; gm=\"sg\" #simple gen model\n",
    "#kdb=4; gm=\"cg\" #complex gen model\n",
    "n_rep = 1; directory_Z1 = \"figC_determ/\"; directory_Z2 = \"figA_determ/\"\n",
    "#n_rep = 5; directory_Z1 = \"figC_sampling/\"; directory_Z2 = \"figA_sampling/\"\n",
    "#prev_name=\"Diff. \"; real_tsdata = -1 # any from 0 to 4 ; -1 means no max.\n",
    "prev_name=\"Rel. diff. \"; real_tsdata = 0 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collect results with only fully labeled data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mResultsFull = np.loadtxt(path+'res_exp_figA_'+gm+'_fully_labeled.resout', delimiter=\",\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collect results with real model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mResultsReal = np.loadtxt(path+'res_exp_figA_'+gm+'_real_model.resout', delimiter=\",\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collect data for `consistent-labeling` scenario"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Z1_complete = []\n",
    "inta = 0\n",
    "for m in np.arange(n_models):\n",
    "    for d in np.arange(n_datasets_Z1):\n",
    "        id_mrf = m*n_datasets_gen+d\n",
    "        id_mrr = (m*n_datasets_gen+d)*5+real_tsdata\n",
    "        for a in np.arange(len(amb))+1:\n",
    "            orig_data = np.loadtxt(path+directory_Z1+'res_exp_figC_m_' + str(m) + \n",
    "                                                                '_d_' + str(d) + \n",
    "                                                                '_css_' + str(css[css_ind]) + \n",
    "                                                                '_k_' + str(kdb) + \n",
    "                                                                '_s_' + str(a) + \n",
    "                                                                 '.csv', delimiter=\",\")\n",
    "            for v in np.arange(len(wsp)):\n",
    "                vals = orig_data[np.arange(v, orig_data.shape[0], len(wsp)),:]\n",
    "                vals = np.mean(vals, axis=0) - mResultsFull[id_mrf] # mean over repetitions and remove real values\n",
    "                if real_tsdata >= 0: vals /= (mResultsReal[id_mrr] - mResultsFull[id_mrf])\n",
    "                Z1_complete.append( vals ) \n",
    "                inta+=1\n",
    "Z1_complete = np.array(Z1_complete)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collect data for `inconsistent-labeling` scenario (baseline)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Z2_complete = []\n",
    "s = 0 # honest scenario\n",
    "\n",
    "for m in np.arange(n_models):\n",
    "    for d in np.arange(n_datasets_Z2):\n",
    "        id_mrf = m*n_datasets_gen+d\n",
    "        id_mrr = (m*n_datasets_gen+d)*5+real_tsdata\n",
    "        orig_data = np.loadtxt(path+directory_Z2+'res_exp_figA_m_' + str(m) + \n",
    "                                                            '_d_' + str(d) + \n",
    "                                                            '_css_' + str(css[css_ind]) + \n",
    "                                                            '_k_' + str(kdb) + \n",
    "                                                            '_s_' + str(s) + \n",
    "                                                             '.csv', delimiter=\",\")\n",
    "        for v in np.arange(len(wsp)):\n",
    "            vals = orig_data[np.arange(v, orig_data.shape[0], len(wsp)),:]\n",
    "            vals = np.mean(vals, axis=0) - mResultsFull[id_mrf] # mean over repetitions and remove real values\n",
    "            if real_tsdata >= 0: vals /= (mResultsReal[id_mrr] - mResultsFull[id_mrf])\n",
    "            Z2_complete.append( vals ) \n",
    "Z2_complete = np.array(Z2_complete)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib widget\n",
    "## 0:Acc, 1:Macro F1, 2:Micro F1, 3:log loss, 4:Brier score\n",
    "metric_ind = 4\n",
    "transparencia = 0.6\n",
    "\n",
    "\n",
    "Z1 = Z1_complete[:,metric_ind]\n",
    "Z1.shape = (n_models*n_datasets_Z1,len(amb)*len(wsp))\n",
    "Z1 = Z1.mean(axis=0)\n",
    "Z1.shape = (len(amb),len(wsp))\n",
    "Z1 = np.transpose(Z1)\n",
    "Z1p = (Z1 - np.min(Z1))/(np.max(Z1) - np.min(Z1))\n",
    "\n",
    "Z2 = Z2_complete[:,metric_ind]\n",
    "Z2.shape = (n_models*n_datasets_Z2,len(wsp))\n",
    "Z2 = Z2.mean(axis=0)\n",
    "Z2 = np.tile(Z2, len(amb)) # dimensions of Z1 and Z2 are different!!! dim(Z1) = dim(Z2)*4\n",
    "Z2.shape = (len(amb),len(wsp))\n",
    "Z2 = np.transpose(Z2)\n",
    "Z2p = (Z2 - np.min(Z2))/(np.max(Z2) - np.min(Z2))\n",
    "\n",
    "\n",
    "x = np.arange(len(amb))\n",
    "x_labels = amb\n",
    "y = np.arange(len(wsp))\n",
    "y_labels = wsp\n",
    "\n",
    "X, Y = np.meshgrid(x, y)\n",
    "\n",
    "\n",
    "C1 = np.empty_like(Z1, dtype=object)\n",
    "C2 = np.empty_like(Z2, dtype=object)\n",
    "cmap1 = plt.get_cmap(\"spring\")\n",
    "cmap2 = plt.get_cmap(\"winter\")\n",
    "\n",
    "for i in range(Z1.shape[0]):\n",
    "    for j in range(Z1.shape[1]):\n",
    "        C1[i,j] = to_rgba(cmap1(Z1p[i,j]), transparencia)\n",
    "        C2[i,j] = to_rgba(cmap2(Z2p[i,j]), transparencia)\n",
    "\n",
    "\n",
    "# Create a transparent bridge region\n",
    "X_bridge = np.vstack([X[-1,:],X[-1,:]])\n",
    "Y_bridge = np.vstack([Y[-1,:],Y[-1,:]])\n",
    "Z_bridge = np.vstack([Z1[-1,:],Z2[-1,:]])\n",
    "color_bridge = np.empty_like(Z_bridge, dtype=object)\n",
    "\n",
    "color_bridge.fill((1,1,1,0)) # RGBA colour, onlt the last component matters.\n",
    "\n",
    "# Join the two surfaces flipping one of them (using also the bridge)\n",
    "X_full = np.vstack([X, X_bridge, np.flipud(X)])\n",
    "Y_full = np.vstack([Y, Y_bridge, np.flipud(Y)])\n",
    "Z_full = np.vstack([Z1, Z_bridge, np.flipud(Z2)])\n",
    "color_full = np.vstack([C1, color_bridge, np.flipud(C2)])\n",
    "\n",
    "\n",
    "### print figure\n",
    "fig = plt.figure(figsize=(10,5))\n",
    "ax = fig.gca(projection='3d')\n",
    "\n",
    "surf_full = ax.plot_surface(X_full, Y_full, Z_full, rstride=1, cstride=1,\n",
    "                            facecolors=color_full, linewidth=0,\n",
    "                            antialiased=False, shade=False)\n",
    "\n",
    "ax.set_xlabel(\"Prob. of co-occurrence\")\n",
    "ax.set_ylabel(\"Prop. weakly labeled ex.\")\n",
    "ax.set_zlabel(prev_name+metrics[metric_ind])\n",
    "\n",
    "ax.set_xticks(x)\n",
    "ax.set_xticklabels(x_labels)\n",
    "ax.set_yticks(y)\n",
    "ax.set_yticklabels(y_labels)\n",
    "\n",
    "ax.text(3, 3, Z1[-1,0], \"With co-occurrence\", zdir=\"x\",\n",
    "        color=cmap1(Z1p[0,-1]))\n",
    "\n",
    "ax.text(3, 3, Z2[-1,0], \"Without\", zdir=\"x\",\n",
    "        color=cmap2(Z2p[0,-1]))\n",
    "\n",
    "\n",
    "# manipulate view\n",
    "ax.view_init(20, -60)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}