run_example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "from base_methods.base_sklearn import generate_probabilities\n",
    "from ebm.dataset import get_train_dev_data, get_test_data\n",
    "from params import MultiBaseParams, SingleBaseParams\n",
    "from utils.feature_names import *\n",
    "from utils.metrics import confidence_intervals\n",
    "from utils.stacking import get_feature_name_sets_with_highest_probable_accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load train and test data (takes some time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = get_train_dev_data()\n",
    "test = get_test_data()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run grid search of parameters:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from examples.run_grid_search import run_grid_search\n",
    "params = SingleBaseParams(feature_name, classifier, dirname=sklearn_master_name, params={}, pca=False)\n",
    "parameters = {\n",
    "        'clf__C': [0.1, 1, 10, 100, 1000],\n",
    "        'clf__gamma': [1, 0.1, 0.01, 0.001, 0.0001],\n",
    "        'clf__kernel': ['rbf', 'linear']}\n",
    "run_grid_search(train=train, test=test, params=params, parameters=parameters)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In order to calculate confidence intervals for a particular set of parameters, run:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score = 0.7049\n",
      "score = 0.7049\n",
      "score = 0.7049\n",
      "score = 0.7049\n",
      "score = 0.6995\n",
      "score = 0.7049\n",
      "score = 0.6995\n",
      "score = 0.7049\n",
      "score = 0.7049\n",
      "score = 0.7049\n",
      "score = 0.7049\n",
      "score = 0.7104\n",
      "score = 0.7049\n",
      "score = 0.7049\n",
      "score = 0.7049\n",
      "publication_type & RandomForestClassifier & 70.455 & 70.3-70.6\n"
     ]
    }
   ],
   "source": [
    "accuracies = []\n",
    "feature_name = publication_type_key\n",
    "classifier = RandomForestClassifier_name\n",
    "for i in range(15):\n",
    "    params = SingleBaseParams(feature_name, classifier, dirname=sklearn_master_name, params={}, pca=False)\n",
    "    a = generate_probabilities(train, test, params, test_only=True)\n",
    "    accuracies.append(a * 100)\n",
    "\n",
    "l, r = confidence_intervals(accuracies)\n",
    "m = float(np.mean([l, r]))\n",
    "results = \"%s & %s & %.3f & %.1f-%.1f\" % (feature_name, classifier, m, l, r)\n",
    "print(results)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In order to generate intermediate features, run:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0/86 (0.00%)\n",
      "score = 0.7000\n",
      "1/86 (1.16%)\n",
      "score = 0.8000\n",
      "2/86 (2.33%)\n",
      "score = 0.7000\n",
      "3/86 (3.49%)\n",
      "score = 0.8000\n",
      "4/86 (4.65%)\n",
      "score = 0.8000\n",
      "5/86 (5.81%)\n",
      "score = 0.4000\n",
      "6/86 (6.98%)\n",
      "score = 0.8000\n",
      "7/86 (8.14%)\n",
      "score = 0.5000\n",
      "8/86 (9.30%)\n",
      "score = 1.0000\n",
      "9/86 (10.47%)\n",
      "score = 0.3000\n",
      "10/86 (11.63%)\n",
      "score = 0.9000\n",
      "11/86 (12.79%)\n",
      "score = 0.4000\n",
      "12/86 (13.95%)\n",
      "score = 0.8000\n",
      "13/86 (15.12%)\n",
      "score = 0.6000\n",
      "14/86 (16.28%)\n",
      "score = 0.5000\n",
      "15/86 (17.44%)\n",
      "score = 0.5000\n",
      "16/86 (18.60%)\n",
      "score = 0.9000\n",
      "17/86 (19.77%)\n",
      "score = 0.8000\n",
      "18/86 (20.93%)\n",
      "score = 1.0000\n",
      "19/86 (22.09%)\n",
      "score = 0.5000\n",
      "20/86 (23.26%)\n",
      "score = 0.6000\n",
      "21/86 (24.42%)\n",
      "score = 0.5000\n",
      "22/86 (25.58%)\n",
      "score = 0.5000\n",
      "23/86 (26.74%)\n",
      "score = 0.6000\n",
      "24/86 (27.91%)\n",
      "score = 0.6000\n",
      "25/86 (29.07%)\n",
      "score = 0.7000\n",
      "26/86 (30.23%)\n",
      "score = 0.8000\n",
      "27/86 (31.40%)\n",
      "score = 0.5000\n",
      "28/86 (32.56%)\n",
      "score = 0.4000\n",
      "29/86 (33.72%)\n",
      "score = 0.8000\n",
      "30/86 (34.88%)\n",
      "score = 0.5000\n",
      "31/86 (36.05%)\n",
      "score = 0.9000\n",
      "32/86 (37.21%)\n",
      "score = 0.5000\n",
      "33/86 (38.37%)\n",
      "score = 0.6000\n",
      "34/86 (39.53%)\n",
      "score = 0.5000\n",
      "35/86 (40.70%)\n",
      "score = 0.7000\n",
      "36/86 (41.86%)\n",
      "score = 0.6000\n",
      "37/86 (43.02%)\n",
      "score = 0.6000\n",
      "38/86 (44.19%)\n",
      "score = 0.5000\n",
      "39/86 (45.35%)\n",
      "score = 0.9000\n",
      "40/86 (46.51%)\n",
      "score = 0.6000\n",
      "41/86 (47.67%)\n",
      "score = 0.6000\n",
      "42/86 (48.84%)\n",
      "score = 0.5000\n",
      "43/86 (50.00%)\n",
      "score = 0.8000\n",
      "44/86 (51.16%)\n",
      "score = 0.5000\n",
      "45/86 (52.33%)\n",
      "score = 0.8000\n",
      "46/86 (53.49%)\n",
      "score = 0.5000\n",
      "47/86 (54.65%)\n",
      "score = 0.9000\n",
      "48/86 (55.81%)\n",
      "score = 0.2000\n",
      "49/86 (56.98%)\n",
      "score = 0.8000\n",
      "50/86 (58.14%)\n",
      "score = 0.7000\n",
      "51/86 (59.30%)\n",
      "score = 0.6000\n",
      "52/86 (60.47%)\n",
      "score = 0.5000\n",
      "53/86 (61.63%)\n",
      "score = 0.5000\n",
      "54/86 (62.79%)\n",
      "score = 0.3000\n",
      "55/86 (63.95%)\n",
      "score = 0.6000\n",
      "56/86 (65.12%)\n",
      "score = 0.5000\n",
      "57/86 (66.28%)\n",
      "score = 0.6000\n",
      "58/86 (67.44%)\n",
      "score = 0.8000\n",
      "59/86 (68.60%)\n",
      "score = 0.5000\n",
      "60/86 (69.77%)\n",
      "score = 0.8000\n",
      "61/86 (70.93%)\n",
      "score = 0.3000\n",
      "62/86 (72.09%)\n",
      "score = 0.6000\n",
      "63/86 (73.26%)\n",
      "score = 0.9000\n",
      "64/86 (74.42%)\n",
      "score = 0.3000\n",
      "65/86 (75.58%)\n",
      "score = 0.6000\n",
      "66/86 (76.74%)\n",
      "score = 0.3000\n",
      "67/86 (77.91%)\n",
      "score = 0.6000\n",
      "68/86 (79.07%)\n",
      "score = 0.7000\n",
      "69/86 (80.23%)\n",
      "score = 0.9000\n",
      "70/86 (81.40%)\n",
      "score = 0.7000\n",
      "71/86 (82.56%)\n",
      "score = 0.7000\n",
      "72/86 (83.72%)\n",
      "score = 0.7000\n",
      "73/86 (84.88%)\n",
      "score = 0.9000\n",
      "74/86 (86.05%)\n",
      "score = 0.8000\n",
      "75/86 (87.21%)\n",
      "score = 0.6000\n",
      "76/86 (88.37%)\n",
      "score = 0.3000\n",
      "77/86 (89.53%)\n",
      "score = 0.5000\n",
      "78/86 (90.70%)\n",
      "score = 0.8000\n",
      "79/86 (91.86%)\n",
      "score = 0.7000\n",
      "80/86 (93.02%)\n",
      "score = 0.7000\n",
      "81/86 (94.19%)\n",
      "score = 0.6000\n",
      "82/86 (95.35%)\n",
      "score = 1.0000\n",
      "83/86 (96.51%)\n",
      "score = 0.7000\n",
      "84/86 (97.67%)\n",
      "score = 0.5000\n",
      "85/86 (98.84%)\n",
      "score = 0.6000\n",
      "score = 0.7104\n",
      "publication_type RandomForestClassifier new:\n",
      "accuracy (A): 0.6483622350674374\n",
      "balanced_accuracy (BA): 0.6366131103904407\n",
      "balanced_accuracy_adjusted (BA): 0.454919665585661\n",
      "precision (P): 0.6631598449488478\n",
      "recall (R): 0.6483622350674374\n",
      "jaccard (IoU): 0.4808633027281686\n",
      "f1_score (F): 0.6493233785362001\n",
      "aed_score (AED): 0.18063583815028902\n",
      "confusion_matrix (C):\n",
      "[[210 104   2]\n",
      " [126 326  28]\n",
      " [  8  97 137]]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "params = SingleBaseParams(feature_name=publication_type_key, classifier=RandomForestClassifier_name, dirname=\"new\", params={}, pca=False)\n",
    "generate_probabilities(train=train, test=test, params=params, test_only=False)\n",
    "params.serialize()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In order to generate intermediate features from multiple features just use:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = MultiBaseParams(features=[publication_type_key, title_key], classifier=AdaBoostClassifier_name, dirname=\"new\", params={}, pca=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ensemble algorithms:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/aleksandra/PycharmProjects/mgr/venv/lib/python3.6/site-packages/sklearn/metrics/_classification.py:1268: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "max_acc 0.31812865497076026 3.4000000000000012\n",
      "min_aed 0.44853801169590646 3.4000000000000012\n",
      "accuracy (A): 0.31812865497076026\n",
      "balanced_accuracy (BA): 0.3109187487704112\n",
      "balanced_accuracy_adjusted (BA): -0.03362187684438316\n",
      "precision (P): 0.3021012541288641\n",
      "recall (R): 0.31812865497076026\n",
      "jaccard (IoU): 0.15753896541692705\n",
      "f1_score (F): 0.2601768581855642\n",
      "aed_score (AED): 0.44853801169590646\n",
      "confusion_matrix (C):\n",
      "[[184  76   0]\n",
      " [303  88   0]\n",
      " [184  20   0]]\n"
     ]
    }
   ],
   "source": [
    "from ensemble_methods.ensemble_auctioning import run\n",
    "params1 = SingleBaseParams(feature_name=publication_type_key, classifier=RandomForestClassifier_name, dirname=\"new\", params={}, pca=False)\n",
    "params2 = SingleBaseParams(feature_name=title_key, classifier=RandomForestClassifier_name, dirname=\"sklearn\", params={}, pca=False)\n",
    "\n",
    "a = run([params1, params2], verbose=False, with_save=False, name=\"new\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<params.SingleBaseParams object at 0x11fcca320> 0.6350877192982456\n",
      "<params.SingleBaseParams object at 0x11fccae80> 0.5169590643274854\n",
      "<params.SingleBaseParams object at 0x11fcca320> 0.7103825136612022\n",
      "<params.SingleBaseParams object at 0x11fccae80> 0.5081967213114754\n",
      "accuracy (A): 0.6557377049180327\n",
      "balanced_accuracy (BA): 0.59190462110332\n",
      "balanced_accuracy_adjusted (BA): 0.38785693165498\n",
      "precision (P): 0.7124817734173736\n",
      "recall (R): 0.6557377049180327\n",
      "jaccard (IoU): 0.47017326949001814\n",
      "f1_score (F): 0.631683239272597\n",
      "aed_score (AED): 0.1721311475409836\n",
      "confusion_matrix (C):\n",
      "[[19 37  0]\n",
      " [ 5 81  3]\n",
      " [ 0 18 20]]\n"
     ]
    }
   ],
   "source": [
    "from ensemble_methods.ensemble_consensus import run\n",
    "params1 = SingleBaseParams(feature_name=publication_type_key, classifier=RandomForestClassifier_name, dirname=\"new\", params={}, pca=False)\n",
    "params2 = SingleBaseParams(feature_name=title_key, classifier=RandomForestClassifier_name, dirname=\"sklearn\", params={}, pca=False)\n",
    "\n",
    "a, line = run([params1, params2], name=\"new\", verbose=False, with_save=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy (A): 0.7049180327868853\n",
      "balanced_accuracy (BA): 0.6677895581650756\n",
      "balanced_accuracy_adjusted (BA): 0.5016843372476133\n",
      "precision (P): 0.7241683007652482\n",
      "recall (R): 0.7049180327868853\n",
      "jaccard (IoU): 0.5396418297676399\n",
      "f1_score (F): 0.6984485312732961\n",
      "aed_score (AED): 0.15027322404371585\n",
      "confusion_matrix (C):\n",
      "[[29 26  1]\n",
      " [10 76  3]\n",
      " [ 0 14 24]]\n"
     ]
    }
   ],
   "source": [
    "from ensemble_methods.ensemble_game_theory import run\n",
    "params1 = SingleBaseParams(feature_name=publication_type_key, classifier=RandomForestClassifier_name, dirname=\"new\", params={}, pca=False)\n",
    "params2 = SingleBaseParams(feature_name=title_key, classifier=RandomForestClassifier_name, dirname=\"sklearn\", params={}, pca=False)\n",
    "\n",
    "a, line = run([params1, params2], verbose=False, with_save=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run ensemble classifiers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best_score 0.7170998632010944\n",
      "0.7170998632010944 0.03298147568364048\n",
      "best_params {}\n"
     ]
    }
   ],
   "source": [
    "from ensemble_methods.ensemble_sklearn import run\n",
    "params1 = SingleBaseParams(feature_name=publication_type_key, classifier=RandomForestClassifier_name, dirname=\"new\", params={}, pca=False)\n",
    "params2 = SingleBaseParams(feature_name=title_key, classifier=RandomForestClassifier_name, dirname=\"sklearn\", params={}, pca=False)\n",
    "\n",
    "mea = run([params1, params2], RandomForestClassifier_name, to_binary=False, use_only_ab=False, balanced=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run ensemble neural networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "183/183 [==============================] - 0s 194us/step\n",
      "183/183 [==============================] - 0s 156us/step\n",
      "183/183 [==============================] - 0s 160us/step\n",
      "183/183 [==============================] - 0s 155us/step\n",
      "183/183 [==============================] - 0s 168us/step\n",
      "183/183 [==============================] - 0s 156us/step\n",
      "183/183 [==============================] - 0s 161us/step\n",
      "183/183 [==============================] - 0s 155us/step\n",
      "183/183 [==============================] - 0s 186us/step\n",
      "183/183 [==============================] - 0s 156us/step\n",
      "publication type, title M(0) & adadelta & 68.25 & 66.04-70.47 \\\\\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from ensemble_methods.ensemble_keras import run\n",
    "params1 = SingleBaseParams(feature_name=publication_type_key, classifier=RandomForestClassifier_name, dirname=\"new\", params={}, pca=False)\n",
    "params2 = SingleBaseParams(feature_name=title_key, classifier=RandomForestClassifier_name, dirname=\"sklearn\", params={}, pca=False)\n",
    "\n",
    "mea = run([params1, params2], 'adadelta', use_only_ab=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}