fidelity.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fidelity"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os.path\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "\n",
    "def bin_data(dt1, dt2, c = 10):\n",
    "    dt1 = dt1.copy()\n",
    "    dt2 = dt2.copy()\n",
    "    # quantile binning of numerics\n",
    "    num_cols = dt1.dtypes[dt1.dtypes!='object'].index\n",
    "    for col in num_cols:\n",
    "        # determine breaks based on `dt1`\n",
    "        breaks = dt1[col].quantile(np.linspace(0, 1, c+1)).unique()\n",
    "        dt1[col] = pd.cut(dt1[col], bins=breaks, include_lowest=True).astype(str)\n",
    "        dt2_vals = pd.to_numeric(dt2[col], 'coerce')\n",
    "        dt2_bins = pd.cut(dt2_vals, bins=breaks, include_lowest=True).astype(str)\n",
    "        dt2_bins[dt2_vals < min(breaks)] = '_other_'\n",
    "        dt2_bins[dt2_vals > max(breaks)] = '_other_'\n",
    "        dt2[col] = dt2_bins\n",
    "    # top-C binning of categoricals\n",
    "    cat_cols = dt1.dtypes[dt1.dtypes=='object'].index\n",
    "    for col in cat_cols:\n",
    "        # determine top values based on `dt1`\n",
    "        top_vals = dt1[col].value_counts().head(c).index.tolist()\n",
    "        dt1[col].replace(np.setdiff1d(dt1[col].unique().tolist(), top_vals), '_other_', inplace=True)\n",
    "        dt2[col].replace(np.setdiff1d(dt2[col].unique().tolist(), top_vals), '_other_', inplace=True)\n",
    "    return [dt1, dt2]\n",
    "\n",
    "def hellinger(p1, p2):\n",
    "  return np.sqrt(1 - np.sum(np.sqrt(p1*p2)))\n",
    "\n",
    "def kullback_leibler(p1, p2):\n",
    "  idx = p1>0\n",
    "  return np.sum(p1[idx] * np.log(p1[idx]/p2[idx]))\n",
    "\n",
    "def jensen_shannon(p1, p2):\n",
    "  m = 0.5 * (p1 + p2)\n",
    "  return 0.5 * kullback_leibler(p1, m) + 0.5 * kullback_leibler(p2, m)\n",
    "\n",
    "def fidelity(dt1, dt2, c = 100, k = 1):\n",
    "    [dt1_bin, dt2_bin] = bin_data(dt1, dt2, c = c)\n",
    "    # build grid of all cross-combinations\n",
    "    cols = trn.columns\n",
    "    interactions = pd.DataFrame(np.array(np.meshgrid(cols, cols, cols)).reshape(3, len(cols)**3).T)\n",
    "    interactions.columns = ['dim1', 'dim2', 'dim3']\n",
    "    if k == 1:\n",
    "        interactions = interactions.loc[(interactions['dim1']==interactions['dim2']) & (interactions['dim2']==interactions['dim3'])]\n",
    "    elif k == 2:\n",
    "        interactions = interactions.loc[(interactions['dim1']<interactions['dim2']) & (interactions['dim2']==interactions['dim3'])]\n",
    "    elif k == 3:\n",
    "        interactions = interactions.loc[(interactions['dim1']<interactions['dim2']) & (interactions['dim2']<interactions['dim3'])]\n",
    "    else:\n",
    "        raise('k>3 not supported')\n",
    "\n",
    "    results = []\n",
    "    for idx in range(interactions.shape[0]):\n",
    "        row = interactions.iloc[idx]\n",
    "        val1 = dt1_bin[row.dim1] + dt1_bin[row.dim2] + dt1_bin[row.dim3]\n",
    "        val2 = dt2_bin[row.dim1] + dt2_bin[row.dim2] + dt2_bin[row.dim3]\n",
    "        freq1 = val1.value_counts(normalize=True).to_frame(name='p1')\n",
    "        freq2 = val2.value_counts(normalize=True).to_frame(name='p2')\n",
    "        freq = freq1.join(freq2, how='outer').fillna(0.0)\n",
    "        p1 = freq['p1']\n",
    "        p2 = freq['p2']\n",
    "        out = pd.DataFrame({\n",
    "          'k': k,\n",
    "          'dim1': [row.dim1], 'dim2': [row.dim2], 'dim3': [row.dim3],\n",
    "          'tvd': [np.sum(np.abs(p1 - p2)) / 2], \n",
    "          'mae': [np.mean(np.abs(p1 - p2))], \n",
    "          'max': [np.max(np.abs(p1 - p2))],\n",
    "          'l1d': [np.sum(np.abs(p1 - p2))],\n",
    "          'l2d': [np.sqrt(np.sum((p1 - p2)**2))],\n",
    "          'hellinger': [hellinger(p1, p2)],\n",
    "          'jensen_shannon': [jensen_shannon(p1, p2)]})\n",
    "        results.append(out)\n",
    "\n",
    "    return pd.concat(results)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test Drive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "k                 1.000000\n",
       "tvd               0.037641\n",
       "mae               0.003392\n",
       "max               0.013368\n",
       "l1d               0.075281\n",
       "l2d               0.019235\n",
       "hellinger         0.034161\n",
       "jensen_shannon    0.001531\n",
       "dtype: float64"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trn = pd.read_csv('data/credit-default_trn.csv.gz')\n",
    "syn = pd.read_csv('data/credit-default_mostly.csv.gz')\n",
    "#syn = pd.read_csv('data/credit-default_synthpop.csv.gz')\n",
    "\n",
    "fidelity(trn, syn, k=1, c=100).agg('mean')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Benchmark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data/adult_mostly.csv.gz\n",
      "data/adult_copulagan.csv.gz\n",
      "data/adult_ctgan.csv.gz\n",
      "data/adult_tvae.csv.gz\n",
      "data/adult_gaussian_copula.csv.gz\n",
      "data/adult_gretel.csv.gz\n",
      "data/adult_synthpop.csv.gz\n",
      "data/adult_mostly_e1.csv.gz\n",
      "data/adult_mostly_e2.csv.gz\n",
      "data/adult_mostly_e4.csv.gz\n",
      "data/adult_mostly_e8.csv.gz\n",
      "data/adult_mostly_e16.csv.gz\n",
      "data/adult_flip10.csv.gz\n",
      "data/adult_flip20.csv.gz\n",
      "data/adult_flip30.csv.gz\n",
      "data/adult_flip40.csv.gz\n",
      "data/adult_flip50.csv.gz\n",
      "data/adult_flip60.csv.gz\n",
      "data/adult_flip70.csv.gz\n",
      "data/adult_flip80.csv.gz\n",
      "data/adult_flip90.csv.gz\n",
      "data/adult_val.csv.gz\n",
      "data/credit-default_mostly.csv.gz\n",
      "data/credit-default_copulagan.csv.gz\n",
      "data/credit-default_ctgan.csv.gz\n",
      "data/credit-default_tvae.csv.gz\n",
      "data/credit-default_gaussian_copula.csv.gz\n",
      "data/credit-default_gretel.csv.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/mplatzer/miniconda3/envs/jupyter/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3071: DtypeWarning: Columns (18,21) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data/credit-default_synthpop.csv.gz\n",
      "data/credit-default_mostly_e1.csv.gz\n",
      "data/credit-default_mostly_e2.csv.gz\n",
      "data/credit-default_mostly_e4.csv.gz\n",
      "data/credit-default_mostly_e8.csv.gz\n",
      "data/credit-default_mostly_e16.csv.gz\n",
      "data/credit-default_flip10.csv.gz\n",
      "data/credit-default_flip20.csv.gz\n",
      "data/credit-default_flip30.csv.gz\n",
      "data/credit-default_flip40.csv.gz\n",
      "data/credit-default_flip50.csv.gz\n",
      "data/credit-default_flip60.csv.gz\n",
      "data/credit-default_flip70.csv.gz\n",
      "data/credit-default_flip80.csv.gz\n",
      "data/credit-default_flip90.csv.gz\n",
      "data/credit-default_val.csv.gz\n",
      "data/bank-marketing_mostly.csv.gz\n",
      "data/bank-marketing_copulagan.csv.gz\n",
      "data/bank-marketing_ctgan.csv.gz\n",
      "data/bank-marketing_tvae.csv.gz\n",
      "data/bank-marketing_gaussian_copula.csv.gz\n",
      "data/bank-marketing_gretel.csv.gz\n",
      "data/bank-marketing_synthpop.csv.gz\n",
      "data/bank-marketing_mostly_e1.csv.gz\n",
      "data/bank-marketing_mostly_e2.csv.gz\n",
      "data/bank-marketing_mostly_e4.csv.gz\n",
      "data/bank-marketing_mostly_e8.csv.gz\n",
      "data/bank-marketing_mostly_e16.csv.gz\n",
      "data/bank-marketing_flip10.csv.gz\n",
      "data/bank-marketing_flip20.csv.gz\n",
      "data/bank-marketing_flip30.csv.gz\n",
      "data/bank-marketing_flip40.csv.gz\n",
      "data/bank-marketing_flip50.csv.gz\n",
      "data/bank-marketing_flip60.csv.gz\n",
      "data/bank-marketing_flip70.csv.gz\n",
      "data/bank-marketing_flip80.csv.gz\n",
      "data/bank-marketing_flip90.csv.gz\n",
      "data/bank-marketing_val.csv.gz\n",
      "data/online-shoppers_mostly.csv.gz\n",
      "data/online-shoppers_copulagan.csv.gz\n",
      "data/online-shoppers_ctgan.csv.gz\n",
      "data/online-shoppers_tvae.csv.gz\n",
      "data/online-shoppers_gaussian_copula.csv.gz\n",
      "data/online-shoppers_gretel.csv.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/mplatzer/miniconda3/envs/jupyter/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3071: DtypeWarning: Columns (11,16) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data/online-shoppers_synthpop.csv.gz\n",
      "data/online-shoppers_mostly_e1.csv.gz\n",
      "data/online-shoppers_mostly_e2.csv.gz\n",
      "data/online-shoppers_mostly_e4.csv.gz\n",
      "data/online-shoppers_mostly_e8.csv.gz\n",
      "data/online-shoppers_mostly_e16.csv.gz\n",
      "data/online-shoppers_flip10.csv.gz\n",
      "data/online-shoppers_flip20.csv.gz\n",
      "data/online-shoppers_flip30.csv.gz\n",
      "data/online-shoppers_flip40.csv.gz\n",
      "data/online-shoppers_flip50.csv.gz\n",
      "data/online-shoppers_flip60.csv.gz\n",
      "data/online-shoppers_flip70.csv.gz\n",
      "data/online-shoppers_flip80.csv.gz\n",
      "data/online-shoppers_flip90.csv.gz\n",
      "data/online-shoppers_val.csv.gz\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>k</th>\n",
       "      <th>dim1</th>\n",
       "      <th>dim2</th>\n",
       "      <th>dim3</th>\n",
       "      <th>tvd</th>\n",
       "      <th>mae</th>\n",
       "      <th>max</th>\n",
       "      <th>l1d</th>\n",
       "      <th>l2d</th>\n",
       "      <th>hellinger</th>\n",
       "      <th>jensen_shannon</th>\n",
       "      <th>dataset</th>\n",
       "      <th>synthesizer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>age</td>\n",
       "      <td>age</td>\n",
       "      <td>age</td>\n",
       "      <td>0.028415</td>\n",
       "      <td>0.001114</td>\n",
       "      <td>0.005074</td>\n",
       "      <td>0.056831</td>\n",
       "      <td>0.010763</td>\n",
       "      <td>0.029035</td>\n",
       "      <td>8.420200e-04</td>\n",
       "      <td>adult</td>\n",
       "      <td>mostly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>workclass</td>\n",
       "      <td>workclass</td>\n",
       "      <td>workclass</td>\n",
       "      <td>0.005931</td>\n",
       "      <td>0.001186</td>\n",
       "      <td>0.003501</td>\n",
       "      <td>0.011862</td>\n",
       "      <td>0.005244</td>\n",
       "      <td>0.023622</td>\n",
       "      <td>4.055906e-04</td>\n",
       "      <td>adult</td>\n",
       "      <td>mostly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>fnlwgt</td>\n",
       "      <td>fnlwgt</td>\n",
       "      <td>fnlwgt</td>\n",
       "      <td>0.034485</td>\n",
       "      <td>0.000690</td>\n",
       "      <td>0.002471</td>\n",
       "      <td>0.068970</td>\n",
       "      <td>0.008635</td>\n",
       "      <td>0.030822</td>\n",
       "      <td>9.494985e-04</td>\n",
       "      <td>adult</td>\n",
       "      <td>mostly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>education</td>\n",
       "      <td>education</td>\n",
       "      <td>education</td>\n",
       "      <td>0.012151</td>\n",
       "      <td>0.001519</td>\n",
       "      <td>0.004540</td>\n",
       "      <td>0.024302</td>\n",
       "      <td>0.008049</td>\n",
       "      <td>0.013165</td>\n",
       "      <td>1.732776e-04</td>\n",
       "      <td>adult</td>\n",
       "      <td>mostly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>education-num</td>\n",
       "      <td>education-num</td>\n",
       "      <td>education-num</td>\n",
       "      <td>0.011926</td>\n",
       "      <td>0.001704</td>\n",
       "      <td>0.004540</td>\n",
       "      <td>0.023853</td>\n",
       "      <td>0.008105</td>\n",
       "      <td>0.013329</td>\n",
       "      <td>1.775985e-04</td>\n",
       "      <td>adult</td>\n",
       "      <td>mostly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>Browser</td>\n",
       "      <td>Revenue</td>\n",
       "      <td>Weekend</td>\n",
       "      <td>0.000811</td>\n",
       "      <td>0.000811</td>\n",
       "      <td>0.000811</td>\n",
       "      <td>0.001622</td>\n",
       "      <td>0.001147</td>\n",
       "      <td>0.000793</td>\n",
       "      <td>6.288789e-07</td>\n",
       "      <td>online-shoppers</td>\n",
       "      <td>val</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>Region</td>\n",
       "      <td>Revenue</td>\n",
       "      <td>SpecialDay</td>\n",
       "      <td>0.009084</td>\n",
       "      <td>0.004542</td>\n",
       "      <td>0.009084</td>\n",
       "      <td>0.018167</td>\n",
       "      <td>0.012304</td>\n",
       "      <td>0.008111</td>\n",
       "      <td>6.579021e-05</td>\n",
       "      <td>online-shoppers</td>\n",
       "      <td>val</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>Region</td>\n",
       "      <td>Revenue</td>\n",
       "      <td>TrafficType</td>\n",
       "      <td>0.022709</td>\n",
       "      <td>0.002839</td>\n",
       "      <td>0.008110</td>\n",
       "      <td>0.045418</td>\n",
       "      <td>0.014866</td>\n",
       "      <td>0.021295</td>\n",
       "      <td>4.533528e-04</td>\n",
       "      <td>online-shoppers</td>\n",
       "      <td>val</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>Region</td>\n",
       "      <td>Revenue</td>\n",
       "      <td>VisitorType</td>\n",
       "      <td>0.012165</td>\n",
       "      <td>0.002028</td>\n",
       "      <td>0.007299</td>\n",
       "      <td>0.024331</td>\n",
       "      <td>0.009978</td>\n",
       "      <td>0.014966</td>\n",
       "      <td>2.219092e-04</td>\n",
       "      <td>online-shoppers</td>\n",
       "      <td>val</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>Region</td>\n",
       "      <td>Revenue</td>\n",
       "      <td>Weekend</td>\n",
       "      <td>0.009084</td>\n",
       "      <td>0.004542</td>\n",
       "      <td>0.009084</td>\n",
       "      <td>0.018167</td>\n",
       "      <td>0.012304</td>\n",
       "      <td>0.008111</td>\n",
       "      <td>6.579021e-05</td>\n",
       "      <td>online-shoppers</td>\n",
       "      <td>val</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>103818 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    k           dim1           dim2           dim3       tvd       mae  \\\n",
       "0   1            age            age            age  0.028415  0.001114   \n",
       "0   1      workclass      workclass      workclass  0.005931  0.001186   \n",
       "0   1         fnlwgt         fnlwgt         fnlwgt  0.034485  0.000690   \n",
       "0   1      education      education      education  0.012151  0.001519   \n",
       "0   1  education-num  education-num  education-num  0.011926  0.001704   \n",
       ".. ..            ...            ...            ...       ...       ...   \n",
       "0   3        Browser        Revenue        Weekend  0.000811  0.000811   \n",
       "0   3         Region        Revenue     SpecialDay  0.009084  0.004542   \n",
       "0   3         Region        Revenue    TrafficType  0.022709  0.002839   \n",
       "0   3         Region        Revenue    VisitorType  0.012165  0.002028   \n",
       "0   3         Region        Revenue        Weekend  0.009084  0.004542   \n",
       "\n",
       "         max       l1d       l2d  hellinger  jensen_shannon          dataset  \\\n",
       "0   0.005074  0.056831  0.010763   0.029035    8.420200e-04            adult   \n",
       "0   0.003501  0.011862  0.005244   0.023622    4.055906e-04            adult   \n",
       "0   0.002471  0.068970  0.008635   0.030822    9.494985e-04            adult   \n",
       "0   0.004540  0.024302  0.008049   0.013165    1.732776e-04            adult   \n",
       "0   0.004540  0.023853  0.008105   0.013329    1.775985e-04            adult   \n",
       "..       ...       ...       ...        ...             ...              ...   \n",
       "0   0.000811  0.001622  0.001147   0.000793    6.288789e-07  online-shoppers   \n",
       "0   0.009084  0.018167  0.012304   0.008111    6.579021e-05  online-shoppers   \n",
       "0   0.008110  0.045418  0.014866   0.021295    4.533528e-04  online-shoppers   \n",
       "0   0.007299  0.024331  0.009978   0.014966    2.219092e-04  online-shoppers   \n",
       "0   0.009084  0.018167  0.012304   0.008111    6.579021e-05  online-shoppers   \n",
       "\n",
       "   synthesizer  \n",
       "0       mostly  \n",
       "0       mostly  \n",
       "0       mostly  \n",
       "0       mostly  \n",
       "0       mostly  \n",
       "..         ...  \n",
       "0          val  \n",
       "0          val  \n",
       "0          val  \n",
       "0          val  \n",
       "0          val  \n",
       "\n",
       "[103818 rows x 13 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# benchmark all\n",
    "datasets = ['adult', 'credit-default', 'bank-marketing', 'online-shoppers']\n",
    "fns = ['mostly', 'copulagan', 'ctgan', 'tvae', 'gaussian_copula', 'gretel', 'synthpop',\n",
    "       'mostly_e1', 'mostly_e2', 'mostly_e4', 'mostly_e8', 'mostly_e16',\n",
    "       'flip10', 'flip20', 'flip30', 'flip40', 'flip50', \n",
    "       'flip60', 'flip70', 'flip80', 'flip90',\n",
    "       'val']\n",
    "\n",
    "results = []\n",
    "for dataset in datasets:\n",
    "    trn = pd.read_csv('data/' + dataset + '_trn.csv.gz')\n",
    "    for fn in fns:\n",
    "        syn_fn = 'data/' + dataset  + '_' + fn + '.csv.gz'\n",
    "        print(syn_fn)\n",
    "        if (os.path.exists(syn_fn)):\n",
    "            syn = pd.read_csv(syn_fn)\n",
    "            fid1 = fidelity(trn, syn, k=1, c=100)\n",
    "            fid2 = fidelity(trn, syn, k=2, c=10)\n",
    "            fid3 = fidelity(trn, syn, k=3, c=5)\n",
    "            out = pd.concat([fid1, fid2, fid3])\n",
    "            out['dataset'] = dataset\n",
    "            out['synthesizer'] = fn\n",
    "            results.append(out)\n",
    "\n",
    "x = pd.concat(results)\n",
    "x.to_csv('fidelity.csv', index=False)\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>tvd</th>\n",
       "      <th>mae</th>\n",
       "      <th>max</th>\n",
       "      <th>l1d</th>\n",
       "      <th>l2d</th>\n",
       "      <th>hellinger</th>\n",
       "      <th>jensen_shannon</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th>synthesizer</th>\n",
       "      <th>k</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"20\" valign=\"top\">adult</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">copulagan</th>\n",
       "      <th>1</th>\n",
       "      <td>0.131132</td>\n",
       "      <td>0.024476</td>\n",
       "      <td>0.088083</td>\n",
       "      <td>0.262265</td>\n",
       "      <td>0.127586</td>\n",
       "      <td>0.140999</td>\n",
       "      <td>0.035665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.207133</td>\n",
       "      <td>0.027109</td>\n",
       "      <td>0.093471</td>\n",
       "      <td>0.414266</td>\n",
       "      <td>0.147127</td>\n",
       "      <td>0.215095</td>\n",
       "      <td>0.063381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.264201</td>\n",
       "      <td>0.012318</td>\n",
       "      <td>0.082287</td>\n",
       "      <td>0.528402</td>\n",
       "      <td>0.142305</td>\n",
       "      <td>0.275851</td>\n",
       "      <td>0.092314</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">ctgan</th>\n",
       "      <th>1</th>\n",
       "      <td>0.158024</td>\n",
       "      <td>0.032822</td>\n",
       "      <td>0.109902</td>\n",
       "      <td>0.316048</td>\n",
       "      <td>0.150377</td>\n",
       "      <td>0.156599</td>\n",
       "      <td>0.040073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.209406</td>\n",
       "      <td>0.026458</td>\n",
       "      <td>0.097115</td>\n",
       "      <td>0.418813</td>\n",
       "      <td>0.148934</td>\n",
       "      <td>0.215927</td>\n",
       "      <td>0.058027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.263186</td>\n",
       "      <td>0.011949</td>\n",
       "      <td>0.081526</td>\n",
       "      <td>0.526372</td>\n",
       "      <td>0.140765</td>\n",
       "      <td>0.272194</td>\n",
       "      <td>0.083775</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">flip10</th>\n",
       "      <th>1</th>\n",
       "      <td>0.005298</td>\n",
       "      <td>0.000953</td>\n",
       "      <td>0.002204</td>\n",
       "      <td>0.010597</td>\n",
       "      <td>0.003359</td>\n",
       "      <td>0.006028</td>\n",
       "      <td>0.000051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.016747</td>\n",
       "      <td>0.001341</td>\n",
       "      <td>0.005056</td>\n",
       "      <td>0.033495</td>\n",
       "      <td>0.008773</td>\n",
       "      <td>0.023427</td>\n",
       "      <td>0.001432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.029523</td>\n",
       "      <td>0.001281</td>\n",
       "      <td>0.006973</td>\n",
       "      <td>0.059046</td>\n",
       "      <td>0.013089</td>\n",
       "      <td>0.043117</td>\n",
       "      <td>0.003421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">flip20</th>\n",
       "      <th>1</th>\n",
       "      <td>0.005175</td>\n",
       "      <td>0.000588</td>\n",
       "      <td>0.001766</td>\n",
       "      <td>0.010350</td>\n",
       "      <td>0.002896</td>\n",
       "      <td>0.005788</td>\n",
       "      <td>0.000054</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.027762</td>\n",
       "      <td>0.001953</td>\n",
       "      <td>0.008107</td>\n",
       "      <td>0.055524</td>\n",
       "      <td>0.014502</td>\n",
       "      <td>0.035824</td>\n",
       "      <td>0.003245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.052230</td>\n",
       "      <td>0.002211</td>\n",
       "      <td>0.012600</td>\n",
       "      <td>0.104460</td>\n",
       "      <td>0.023574</td>\n",
       "      <td>0.066937</td>\n",
       "      <td>0.007676</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">flip30</th>\n",
       "      <th>1</th>\n",
       "      <td>0.005600</td>\n",
       "      <td>0.000926</td>\n",
       "      <td>0.002182</td>\n",
       "      <td>0.011201</td>\n",
       "      <td>0.003503</td>\n",
       "      <td>0.006264</td>\n",
       "      <td>0.000056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.038621</td>\n",
       "      <td>0.002812</td>\n",
       "      <td>0.011601</td>\n",
       "      <td>0.077242</td>\n",
       "      <td>0.020525</td>\n",
       "      <td>0.046876</td>\n",
       "      <td>0.005260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.073627</td>\n",
       "      <td>0.003159</td>\n",
       "      <td>0.018328</td>\n",
       "      <td>0.147253</td>\n",
       "      <td>0.033772</td>\n",
       "      <td>0.087227</td>\n",
       "      <td>0.012371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">flip40</th>\n",
       "      <th>1</th>\n",
       "      <td>0.005181</td>\n",
       "      <td>0.000835</td>\n",
       "      <td>0.002068</td>\n",
       "      <td>0.010362</td>\n",
       "      <td>0.003277</td>\n",
       "      <td>0.006083</td>\n",
       "      <td>0.000050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.046845</td>\n",
       "      <td>0.003287</td>\n",
       "      <td>0.013690</td>\n",
       "      <td>0.093690</td>\n",
       "      <td>0.024678</td>\n",
       "      <td>0.055291</td>\n",
       "      <td>0.007215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.090810</td>\n",
       "      <td>0.003837</td>\n",
       "      <td>0.022340</td>\n",
       "      <td>0.181619</td>\n",
       "      <td>0.041519</td>\n",
       "      <td>0.103311</td>\n",
       "      <td>0.016959</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">flip50</th>\n",
       "      <th>1</th>\n",
       "      <td>0.005031</td>\n",
       "      <td>0.000659</td>\n",
       "      <td>0.002111</td>\n",
       "      <td>0.010063</td>\n",
       "      <td>0.003137</td>\n",
       "      <td>0.005516</td>\n",
       "      <td>0.000047</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.054200</td>\n",
       "      <td>0.003788</td>\n",
       "      <td>0.015872</td>\n",
       "      <td>0.108401</td>\n",
       "      <td>0.028610</td>\n",
       "      <td>0.062217</td>\n",
       "      <td>0.009129</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            tvd       mae       max       l1d       l2d  \\\n",
       "dataset synthesizer k                                                     \n",
       "adult   copulagan   1  0.131132  0.024476  0.088083  0.262265  0.127586   \n",
       "                    2  0.207133  0.027109  0.093471  0.414266  0.147127   \n",
       "                    3  0.264201  0.012318  0.082287  0.528402  0.142305   \n",
       "        ctgan       1  0.158024  0.032822  0.109902  0.316048  0.150377   \n",
       "                    2  0.209406  0.026458  0.097115  0.418813  0.148934   \n",
       "                    3  0.263186  0.011949  0.081526  0.526372  0.140765   \n",
       "        flip10      1  0.005298  0.000953  0.002204  0.010597  0.003359   \n",
       "                    2  0.016747  0.001341  0.005056  0.033495  0.008773   \n",
       "                    3  0.029523  0.001281  0.006973  0.059046  0.013089   \n",
       "        flip20      1  0.005175  0.000588  0.001766  0.010350  0.002896   \n",
       "                    2  0.027762  0.001953  0.008107  0.055524  0.014502   \n",
       "                    3  0.052230  0.002211  0.012600  0.104460  0.023574   \n",
       "        flip30      1  0.005600  0.000926  0.002182  0.011201  0.003503   \n",
       "                    2  0.038621  0.002812  0.011601  0.077242  0.020525   \n",
       "                    3  0.073627  0.003159  0.018328  0.147253  0.033772   \n",
       "        flip40      1  0.005181  0.000835  0.002068  0.010362  0.003277   \n",
       "                    2  0.046845  0.003287  0.013690  0.093690  0.024678   \n",
       "                    3  0.090810  0.003837  0.022340  0.181619  0.041519   \n",
       "        flip50      1  0.005031  0.000659  0.002111  0.010063  0.003137   \n",
       "                    2  0.054200  0.003788  0.015872  0.108401  0.028610   \n",
       "\n",
       "                       hellinger  jensen_shannon  \n",
       "dataset synthesizer k                             \n",
       "adult   copulagan   1   0.140999        0.035665  \n",
       "                    2   0.215095        0.063381  \n",
       "                    3   0.275851        0.092314  \n",
       "        ctgan       1   0.156599        0.040073  \n",
       "                    2   0.215927        0.058027  \n",
       "                    3   0.272194        0.083775  \n",
       "        flip10      1   0.006028        0.000051  \n",
       "                    2   0.023427        0.001432  \n",
       "                    3   0.043117        0.003421  \n",
       "        flip20      1   0.005788        0.000054  \n",
       "                    2   0.035824        0.003245  \n",
       "                    3   0.066937        0.007676  \n",
       "        flip30      1   0.006264        0.000056  \n",
       "                    2   0.046876        0.005260  \n",
       "                    3   0.087227        0.012371  \n",
       "        flip40      1   0.006083        0.000050  \n",
       "                    2   0.055291        0.007215  \n",
       "                    3   0.103311        0.016959  \n",
       "        flip50      1   0.005516        0.000047  \n",
       "                    2   0.062217        0.009129  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.groupby(['dataset', 'synthesizer', 'k']).agg('mean').head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}