classifier+results.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b72b9a1a-ce90-4838-b2dd-be0bbe31ca78",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os, json\n",
    "import xgboost as xgb\n",
    "import lightgbm as lgb\n",
    "import numpy as np\n",
    "from optuna.samplers import TPESampler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec5513a3-8492-4332-949e-925771117e71",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import optuna\n",
    "\n",
    "import sklearn.datasets\n",
    "import sklearn.ensemble\n",
    "import sklearn.model_selection\n",
    "import sklearn.svm\n",
    "import sklearn.neural_network\n",
    "import sklearn.tree\n",
    "import sklearn.metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bdd9c394-035e-4e09-b0b8-0bd53870b9b3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aa15d044-1feb-4970-995f-0069a6a9833e",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "with open(\"connectives_dict.json\") as f:\n",
    "    connective_dict = json.load(f)\n",
    "connective_dict[\"20\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ecb701b-9c5b-4923-910d-9bfee32a801d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# def filter_columns(df, connective_selection):\n",
    "#     meta_cols = list()\n",
    "#     cols_conn = list()\n",
    "#     for col in df.columns:\n",
    "#         if col == \"index\":\n",
    "#             pass\n",
    "#         elif not \"bert\" in col:\n",
    "#             meta_cols.append(col)\n",
    "#         else:\n",
    "#             model_name, connective = col.split(\"_\")\n",
    "#             if connective in connective_selection:\n",
    "#                 cols_conn.append(col)\n",
    "#     return df[meta_cols+cols_conn]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1568b11e-c683-48ca-ac52-128dec51a1e5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for filename in [f for f in os.listdir(\"features/all_conns_roberta/all_conns/\") if f.endswith(\".csv\")]:\n",
    "    df = pd.read_csv(\"features/all_conns_roberta/all_conns/\"+filename)\n",
    "    for perc in [\"all\", \"80\", \"40\", \"20\"]:\n",
    "        for name, split_perc, split_conn in [(perc+\"_attack_support/\", perc, \"attack+support\"),  (perc+\"_attack/\", perc, \"attack\"),  (perc+\"_support/\", perc, \"support\"),  (perc+\"_both/\", perc, \"both\")]:\n",
    "\n",
    "            if split_conn == \"attack+support\":\n",
    "                df_new = filter_columns(df, connective_dict[split_perc][\"attack\"]+connective_dict[split_perc][\"support\"])\n",
    "            else:\n",
    "                if split_conn in connective_dict[split_perc]:\n",
    "                    df_new = filter_columns(df, connective_dict[split_perc][split_conn])\n",
    "                    # print(connective_dict[split_perc][split_conn])\n",
    "            print(filename, name, split_perc, split_conn, df_new.shape)\n",
    "            if not os.path.exists(\"features/all_conns_roberta/\"+name):\n",
    "                os.makedirs(\"features/all_conns_roberta/\"+name)\n",
    "            df_new.to_csv(\"features/all_conns_roberta/\"+name+filename,index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca5be908-9ae4-490f-abdf-90ca38735b2c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# for filename in [f for f in os.listdir(\"features/all_conns_bert/all_conns/\") if f.endswith(\".csv\")]:\n",
    "#     df = pd.read_csv(\"features/all_conns_bert/all_conns/\"+filename)\n",
    "#     for perc in [\"all\", \"80\", \"40\", \"20\"]:\n",
    "#         for name, split_perc, split_conn in [(perc+\"_attack_support/\", perc, \"attack+support\"),  (perc+\"_attack/\", perc, \"attack\"),  (perc+\"_support/\", perc, \"support\"),  (perc+\"_both/\", perc, \"both\")]:\n",
    "\n",
    "#             if split_conn == \"attack+support\":\n",
    "#                 df_new = filter_columns(df, connective_dict[split_perc][\"attack\"]+connective_dict[split_perc][\"support\"])\n",
    "#             else:\n",
    "#                 if split_conn in connective_dict[split_perc]:\n",
    "#                     df_new = filter_columns(df, connective_dict[split_perc][split_conn])\n",
    "#                     # print(connective_dict[split_perc][split_conn])\n",
    "#             print(filename, name, split_perc, split_conn, df_new.shape)\n",
    "#             # if not os.path.exists(\"features/all_conns_bert/\"+name):\n",
    "#             #     os.makedirs(\"features/all_conns_bert/\"+name)\n",
    "#             # df_new.to_csv(\"features/all_conns_bert/\"+name+filename,index=False)\n"
   ]
  },
  {
   "cell_type": "raw",
   "id": "b57a3173-0eb1-44ae-8108-5b93dc300eb6",
   "metadata": {
    "tags": []
   },
   "source": [
    "split_perc = \"all\"\n",
    "split_conn = \"attack_support\"\n",
    "filename = \"ibmcs_all.csv\"\n",
    "df = pd.read_csv(\"features/all_conns_bert/\"+split_perc+\"_\"+split_conn+\"/\"+filename)"
   ]
  },
  {
   "cell_type": "raw",
   "id": "f60a9bb5-5091-44ff-840e-8c148114aa6b",
   "metadata": {
    "tags": []
   },
   "source": [
    "feat_cols = [col for col in df.columns if \"bert\" in col]\n",
    "feat_cols"
   ]
  },
  {
   "cell_type": "raw",
   "id": "8f15de49-2652-4965-9468-27d9f1f1ab08",
   "metadata": {
    "tags": []
   },
   "source": [
    "df_train = df[df[\"set\"]== \"train\"]\n",
    "df_dev = df[df[\"set\"]== \"dev\"]\n",
    "df_test = df[df[\"set\"]== \"test\"]"
   ]
  },
  {
   "cell_type": "raw",
   "id": "827a7669-dcf7-4147-99e3-24af8e852669",
   "metadata": {
    "tags": []
   },
   "source": [
    "train_df.shape, dev_df.shape, test_df.shape"
   ]
  },
  {
   "cell_type": "raw",
   "id": "7077e8ae-61ed-4889-9c1b-34b055b9c5d8",
   "metadata": {},
   "source": [
    "    for classifier_name, parameter in [\n",
    "                                       # (\"SVC\", {\"kernel\": \"rbf\"}), \n",
    "                                       # (\"SVC\", {\"kernel\": \"linear\"}), (\"LogisticRegression\", {}),\n",
    "                                      (\"MLPClassifier\", {\"activation\": \"tanh\"}), \n",
    "                                       #(\"MLPClassifier\", {\"activation\": \"relu\"}),\n",
    "                                       (\"DecisionTreeClassifier\", {}),\n",
    "                                       (\"DummyClassifier\", {\"strategy\": \"uniform\"}), \n",
    "                                       (\"DummyClassifier\", {\"strategy\": \"most_frequent\"}),\n",
    "                                       # (\"DummyClassifier\", {\"strategy\": \"constant\", \"constant\": majority_class_label})\n",
    "                                      ]:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19b1c695-2a5f-4c72-be73-c78591c748fd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def detailed_objective_test(trial):\n",
    "    classifier_name = trial.suggest_categorical(\"classifier\", [\"SVC\", \"MLPClassifier\", \"DecisionTreeClassifier\", \"RandomForestClassifier\", \"XGBoost\", \"LightGBM\"])\n",
    "    # classifier_name = \"LightGBM\"\n",
    "    if classifier_name == \"SVC\":\n",
    "        svc_c = trial.suggest_float(\"svc_c\", 1e-3, 1e3, log=True)\n",
    "        svc_kernel = trial.suggest_categorical(\"svc_kernel\", [\"rbf\", \"linear\", \"poly\"])\n",
    "        classifier_obj = sklearn.svm.SVC(C=svc_c, gamma=\"auto\", kernel=svc_kernel)\n",
    "    elif classifier_name == \"RandomForestClassifier\":\n",
    "        rf_max_depth = trial.suggest_int(\"rf_max_depth\", 2, 32, log=True)\n",
    "        rf_estimators = trial.suggest_int(\"rf_estimator\", 10, 100)\n",
    "        classifier_obj = sklearn.ensemble.RandomForestClassifier(\n",
    "            max_depth=rf_max_depth, n_estimators=rf_estimators\n",
    "        )\n",
    "    elif classifier_name == \"MLPClassifier\":\n",
    "        mlp_activation = trial.suggest_categorical(\"mlp_activation\", [\"tanh\", \"relu\", \"logistic\", \"identity\"])\n",
    "        mlp_solver = trial.suggest_categorical(\"mlp_solver\", [\"adam\", \"lbfgs\", \"sgd\"])\n",
    "        # first_layer_neurons = trial.suggest_int('first_layer_neurons', 10, 100, step=10)\n",
    "        # second_layer_neurons = trial.suggest_int('second_layer_neurons', 10, 100, step=10)\n",
    "        n_layers = trial.suggest_int('n_layers', 1, 4)\n",
    "        layers = []\n",
    "        for i in range(n_layers):\n",
    "            layers.append(trial.suggest_int('n_units_l_{}'.format(i), 10, 100, step=10))\n",
    "        classifier_obj = sklearn.neural_network.MLPClassifier(activation=mlp_activation, solver=mlp_solver, max_iter=500, hidden_layer_sizes=tuple(layers))\n",
    "    elif classifier_name == \"DecisionTreeClassifier\":\n",
    "        dct_criterion = trial.suggest_categorical(\"dct_criterion\", [\"gini\", \"entropy\", \"log_loss\"])\n",
    "        dct_max_depth = trial.suggest_int(\"dct_max_depth\", 2, 32, log=True)\n",
    "        classifier_obj = sklearn.tree.DecisionTreeClassifier(criterion=dct_criterion, max_depth=dct_max_depth)\n",
    "    elif classifier_name == \"XGBoost\":\n",
    "        dtrain = xgb.DMatrix(df_train[feat_cols], label=df_train[\"stance\"])\n",
    "        dvalid = xgb.DMatrix(df_test[feat_cols], label=df_test[\"stance\"])\n",
    "\n",
    "        param = {\n",
    "            \"verbosity\": 0,\n",
    "            \"objective\": \"binary:logistic\",\n",
    "            # use exact for small dataset.\n",
    "            \"tree_method\": \"exact\",\n",
    "            # defines booster, gblinear for linear functions.\n",
    "            \"booster\": trial.suggest_categorical(\"booster\", [\"gbtree\", \"gblinear\", \"dart\"]),\n",
    "            # L2 regularization weight.\n",
    "            \"lambda\": trial.suggest_float(\"lambda\", 1e-8, 1.0, log=True),\n",
    "            # L1 regularization weight.\n",
    "            \"alpha\": trial.suggest_float(\"alpha\", 1e-8, 1.0, log=True),\n",
    "            # sampling ratio for training data.\n",
    "            \"subsample\": trial.suggest_float(\"subsample\", 0.2, 1.0),\n",
    "            # sampling according to each tree.\n",
    "            \"colsample_bytree\": trial.suggest_float(\"colsample_bytree\", 0.2, 1.0),\n",
    "        }\n",
    "\n",
    "        if param[\"booster\"] in [\"gbtree\", \"dart\"]:\n",
    "            # maximum depth of the tree, signifies complexity of the tree.\n",
    "            param[\"max_depth\"] = trial.suggest_int(\"max_depth\", 3, 9, step=2)\n",
    "            # minimum child weight, larger the term more conservative the tree.\n",
    "            param[\"min_child_weight\"] = trial.suggest_int(\"min_child_weight\", 2, 10)\n",
    "            param[\"eta\"] = trial.suggest_float(\"eta\", 1e-8, 1.0, log=True)\n",
    "            # defines how selective algorithm is.\n",
    "            param[\"gamma\"] = trial.suggest_float(\"gamma\", 1e-8, 1.0, log=True)\n",
    "            param[\"grow_policy\"] = trial.suggest_categorical(\"grow_policy\", [\"depthwise\", \"lossguide\"])\n",
    "\n",
    "        if param[\"booster\"] == \"dart\":\n",
    "            param[\"sample_type\"] = trial.suggest_categorical(\"sample_type\", [\"uniform\", \"weighted\"])\n",
    "            param[\"normalize_type\"] = trial.suggest_categorical(\"normalize_type\", [\"tree\", \"forest\"])\n",
    "            param[\"rate_drop\"] = trial.suggest_float(\"rate_drop\", 1e-8, 1.0, log=True)\n",
    "            param[\"skip_drop\"] = trial.suggest_float(\"skip_drop\", 1e-8, 1.0, log=True)\n",
    "    elif classifier_name == \"LightGBM\":\n",
    "        dtrain = lgb.Dataset(df_train[feat_cols], label=df_train[\"stance\"])\n",
    "\n",
    "        param = {\n",
    "            \"objective\": \"binary\",\n",
    "            \"metric\": \"binary_logloss\",\n",
    "            \"verbosity\": -1,\n",
    "            \"boosting_type\": \"gbdt\",\n",
    "            \"lambda_l1\": trial.suggest_float(\"lambda_l1\", 1e-8, 1.0, log=True),\n",
    "            \"lambda_l2\": trial.suggest_float(\"lambda_l2\", 1e-8, 1.0, log=True),\n",
    "            \"num_leaves\": trial.suggest_int(\"num_leaves\", 2, 256),\n",
    "            \"feature_fraction\": trial.suggest_float(\"feature_fraction\", 0.4, 1.0, step=0.1),\n",
    "            \"bagging_fraction\": trial.suggest_float(\"bagging_fraction\", 0.4, 1.0, step=0.1),\n",
    "            \"bagging_freq\": trial.suggest_int(\"bagging_freq\", 1, 3),\n",
    "            \"min_child_samples\": trial.suggest_int(\"min_child_samples\", 10, 100, step=10),\n",
    "        }\n",
    "\n",
    "    if classifier_name == \"XGBoost\":\n",
    "        bst = xgb.train(param, dtrain)\n",
    "        preds_prec = bst.predict(dvalid)\n",
    "        preds = np.rint(preds_prec)\n",
    "    elif classifier_name == \"LightGBM\":\n",
    "        gbm = lgb.train(param, dtrain)\n",
    "        preds_prec = gbm.predict(df_test[feat_cols])\n",
    "        preds = np.rint(preds_prec)\n",
    "    else:\n",
    "        classifier_obj.fit(df_train[feat_cols], df_train[\"stance\"])\n",
    "        preds = classifier_obj.predict(df_test[feat_cols])\n",
    "\n",
    "    acc = sklearn.metrics.accuracy_score(preds, df_test[\"stance\"])\n",
    "    precision, recall, f1, support = sklearn.metrics.precision_recall_fscore_support(y_true=df_test[\"stance\"], y_pred=preds, average=\"macro\")\n",
    "    precision_per_label, recall_per_label, f1_per_label, support = sklearn.metrics.precision_recall_fscore_support(y_true=df_test[\"stance\"], y_pred=preds, average=None, labels=[0,1])\n",
    "\n",
    "    return acc, f1, recall, precision, f1_per_label, recall_per_label, precision_per_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16ca4e9b-a79b-464c-a9e4-19fd76dfa31e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def objective_optuna(trial):\n",
    "\n",
    "    classifier_name = trial.suggest_categorical(\"classifier\", [\"SVC\", \"MLPClassifier\", \"DecisionTreeClassifier\", \"RandomForestClassifier\", \"XGBoost\", \"LightGBM\"])\n",
    "    # classifier_name = \"LightGBM\"\n",
    "    if classifier_name == \"SVC\":\n",
    "        svc_c = trial.suggest_float(\"svc_c\", 1e-3, 1e3, log=True)\n",
    "        svc_kernel = trial.suggest_categorical(\"svc_kernel\", [\"rbf\", \"linear\", \"poly\"])\n",
    "        classifier_obj = sklearn.svm.SVC(C=svc_c, gamma=\"auto\", kernel=svc_kernel)\n",
    "    elif classifier_name == \"RandomForestClassifier\":\n",
    "        rf_max_depth = trial.suggest_int(\"rf_max_depth\", 2, 32, log=True)\n",
    "        rf_estimators = trial.suggest_int(\"rf_estimator\", 10, 100)\n",
    "        classifier_obj = sklearn.ensemble.RandomForestClassifier(\n",
    "            max_depth=rf_max_depth, n_estimators=rf_estimators\n",
    "        )\n",
    "    elif classifier_name == \"MLPClassifier\":\n",
    "        mlp_activation = trial.suggest_categorical(\"mlp_activation\", [\"tanh\", \"relu\", \"logistic\", \"identity\"])\n",
    "        mlp_solver = trial.suggest_categorical(\"mlp_solver\", [\"adam\", \"lbfgs\", \"sgd\"])\n",
    "        # first_layer_neurons = trial.suggest_int('first_layer_neurons', 10, 100, step=10)\n",
    "        # second_layer_neurons = trial.suggest_int('second_layer_neurons', 10, 100, step=10)\n",
    "        n_layers = trial.suggest_int('n_layers', 1, 4)\n",
    "        layers = []\n",
    "        for i in range(n_layers):\n",
    "            layers.append(trial.suggest_int('n_units_l_{}'.format(i), 10, 100, step=10))\n",
    "        classifier_obj = sklearn.neural_network.MLPClassifier(activation=mlp_activation, solver=mlp_solver, max_iter=500, hidden_layer_sizes=tuple(layers))\n",
    "    elif classifier_name == \"DecisionTreeClassifier\":\n",
    "        dct_criterion = trial.suggest_categorical(\"dct_criterion\", [\"gini\", \"entropy\", \"log_loss\"])\n",
    "        dct_max_depth = trial.suggest_int(\"dct_max_depth\", 2, 32, log=True)\n",
    "        classifier_obj = sklearn.tree.DecisionTreeClassifier(criterion=dct_criterion, max_depth=dct_max_depth)\n",
    "    elif classifier_name == \"XGBoost\":\n",
    "        dtrain = xgb.DMatrix(df_train[feat_cols], label=df_train[\"stance\"])\n",
    "        dvalid = xgb.DMatrix(df_dev[feat_cols], label=df_dev[\"stance\"])\n",
    "\n",
    "        param = {\n",
    "            \"verbosity\": 0,\n",
    "            \"objective\": \"binary:logistic\",\n",
    "            # use exact for small dataset.\n",
    "            \"tree_method\": \"exact\",\n",
    "            # defines booster, gblinear for linear functions.\n",
    "            \"booster\": trial.suggest_categorical(\"booster\", [\"gbtree\", \"gblinear\", \"dart\"]),\n",
    "            # L2 regularization weight.\n",
    "            \"lambda\": trial.suggest_float(\"lambda\", 1e-8, 1.0, log=True),\n",
    "            # L1 regularization weight.\n",
    "            \"alpha\": trial.suggest_float(\"alpha\", 1e-8, 1.0, log=True),\n",
    "            # sampling ratio for training data.\n",
    "            \"subsample\": trial.suggest_float(\"subsample\", 0.2, 1.0),\n",
    "            # sampling according to each tree.\n",
    "            \"colsample_bytree\": trial.suggest_float(\"colsample_bytree\", 0.2, 1.0),\n",
    "        }\n",
    "\n",
    "        if param[\"booster\"] in [\"gbtree\", \"dart\"]:\n",
    "            # maximum depth of the tree, signifies complexity of the tree.\n",
    "            param[\"max_depth\"] = trial.suggest_int(\"max_depth\", 3, 9, step=2)\n",
    "            # minimum child weight, larger the term more conservative the tree.\n",
    "            param[\"min_child_weight\"] = trial.suggest_int(\"min_child_weight\", 2, 10)\n",
    "            param[\"eta\"] = trial.suggest_float(\"eta\", 1e-8, 1.0, log=True)\n",
    "            # defines how selective algorithm is.\n",
    "            param[\"gamma\"] = trial.suggest_float(\"gamma\", 1e-8, 1.0, log=True)\n",
    "            param[\"grow_policy\"] = trial.suggest_categorical(\"grow_policy\", [\"depthwise\", \"lossguide\"])\n",
    "\n",
    "        if param[\"booster\"] == \"dart\":\n",
    "            param[\"sample_type\"] = trial.suggest_categorical(\"sample_type\", [\"uniform\", \"weighted\"])\n",
    "            param[\"normalize_type\"] = trial.suggest_categorical(\"normalize_type\", [\"tree\", \"forest\"])\n",
    "            param[\"rate_drop\"] = trial.suggest_float(\"rate_drop\", 1e-8, 1.0, log=True)\n",
    "            param[\"skip_drop\"] = trial.suggest_float(\"skip_drop\", 1e-8, 1.0, log=True)\n",
    "    elif classifier_name == \"LightGBM\":\n",
    "        dtrain = lgb.Dataset(df_train[feat_cols], label=df_train[\"stance\"])\n",
    "\n",
    "        param = {\n",
    "            \"objective\": \"binary\",\n",
    "            \"metric\": \"binary_logloss\",\n",
    "            \"verbosity\": -1,\n",
    "            \"boosting_type\": \"gbdt\",\n",
    "            \"lambda_l1\": trial.suggest_float(\"lambda_l1\", 1e-8, 1.0, log=True),\n",
    "            \"lambda_l2\": trial.suggest_float(\"lambda_l2\", 1e-8, 1.0, log=True),\n",
    "            \"num_leaves\": trial.suggest_int(\"num_leaves\", 2, 256),\n",
    "            \"feature_fraction\": trial.suggest_float(\"feature_fraction\", 0.4, 1.0, step=0.1),\n",
    "            \"bagging_fraction\": trial.suggest_float(\"bagging_fraction\", 0.4, 1.0, step=0.1),\n",
    "            \"bagging_freq\": trial.suggest_int(\"bagging_freq\", 1, 3),\n",
    "            \"min_child_samples\": trial.suggest_int(\"min_child_samples\", 10, 100, step=10),\n",
    "        }\n",
    "\n",
    "\n",
    "\n",
    "    # score = sklearn.model_selection.cross_val_score(classifier_obj, df_train[feat_cols], df_train[\"stance\"], n_jobs=-1, cv=3)\n",
    "    # accuracy = score.mean()\n",
    "    # # todo use something else then mean?\n",
    "    # return accuracy\n",
    "    if classifier_name == \"XGBoost\":\n",
    "        bst = xgb.train(param, dtrain)\n",
    "        preds_prec = bst.predict(dvalid)\n",
    "        preds = np.rint(preds_prec)\n",
    "    elif classifier_name == \"LightGBM\":\n",
    "        gbm = lgb.train(param, dtrain)\n",
    "        preds_prec = gbm.predict(df_dev[feat_cols])\n",
    "        preds = np.rint(preds_prec)\n",
    "    else:\n",
    "        classifier_obj.fit(df_train[feat_cols], df_train[\"stance\"])\n",
    "        preds = classifier_obj.predict(df_dev[feat_cols])\n",
    "    # probs = clf.predict_proba(X_valid)\n",
    "    \n",
    "    f1_macro = sklearn.metrics.f1_score(y_true=df_dev[\"stance\"], y_pred=preds, average=\"macro\")\n",
    "    return f1_macro\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14542f23-7700-4129-98df-dcf71926781c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def detailed_objective_dev(trial):\n",
    "    classifier_name = trial.suggest_categorical(\"classifier\", [\"SVC\", \"MLPClassifier\", \"DecisionTreeClassifier\", \"RandomForestClassifier\", \"XGBoost\", \"LightGBM\"])\n",
    "    # classifier_name = \"LightGBM\"\n",
    "    if classifier_name == \"SVC\":\n",
    "        svc_c = trial.suggest_float(\"svc_c\", 1e-3, 1e3, log=True)\n",
    "        svc_kernel = trial.suggest_categorical(\"svc_kernel\", [\"rbf\", \"linear\", \"poly\"])\n",
    "        classifier_obj = sklearn.svm.SVC(C=svc_c, gamma=\"auto\", kernel=svc_kernel)\n",
    "    elif classifier_name == \"RandomForestClassifier\":\n",
    "        rf_max_depth = trial.suggest_int(\"rf_max_depth\", 2, 32, log=True)\n",
    "        rf_estimators = trial.suggest_int(\"rf_estimator\", 10, 100)\n",
    "        classifier_obj = sklearn.ensemble.RandomForestClassifier(\n",
    "            max_depth=rf_max_depth, n_estimators=rf_estimators\n",
    "        )\n",
    "    elif classifier_name == \"MLPClassifier\":\n",
    "        mlp_activation = trial.suggest_categorical(\"mlp_activation\", [\"tanh\", \"relu\", \"logistic\", \"identity\"])\n",
    "        mlp_solver = trial.suggest_categorical(\"mlp_solver\", [\"adam\", \"lbfgs\", \"sgd\"])\n",
    "        # first_layer_neurons = trial.suggest_int('first_layer_neurons', 10, 100, step=10)\n",
    "        # second_layer_neurons = trial.suggest_int('second_layer_neurons', 10, 100, step=10)\n",
    "        n_layers = trial.suggest_int('n_layers', 1, 4)\n",
    "        layers = []\n",
    "        for i in range(n_layers):\n",
    "            layers.append(trial.suggest_int('n_units_l_{}'.format(i), 10, 100, step=10))\n",
    "        classifier_obj = sklearn.neural_network.MLPClassifier(activation=mlp_activation, solver=mlp_solver, max_iter=500, hidden_layer_sizes=tuple(layers))\n",
    "    elif classifier_name == \"DecisionTreeClassifier\":\n",
    "        dct_criterion = trial.suggest_categorical(\"dct_criterion\", [\"gini\", \"entropy\", \"log_loss\"])\n",
    "        dct_max_depth = trial.suggest_int(\"dct_max_depth\", 2, 32, log=True)\n",
    "        classifier_obj = sklearn.tree.DecisionTreeClassifier(criterion=dct_criterion, max_depth=dct_max_depth)\n",
    "    elif classifier_name == \"XGBoost\":\n",
    "        dtrain = xgb.DMatrix(df_train[feat_cols], label=df_train[\"stance\"])\n",
    "        dvalid = xgb.DMatrix(df_dev[feat_cols], label=df_dev[\"stance\"])\n",
    "\n",
    "        param = {\n",
    "            \"verbosity\": 0,\n",
    "            \"objective\": \"binary:logistic\",\n",
    "            # use exact for small dataset.\n",
    "            \"tree_method\": \"exact\",\n",
    "            # defines booster, gblinear for linear functions.\n",
    "            \"booster\": trial.suggest_categorical(\"booster\", [\"gbtree\", \"gblinear\", \"dart\"]),\n",
    "            # L2 regularization weight.\n",
    "            \"lambda\": trial.suggest_float(\"lambda\", 1e-8, 1.0, log=True),\n",
    "            # L1 regularization weight.\n",
    "            \"alpha\": trial.suggest_float(\"alpha\", 1e-8, 1.0, log=True),\n",
    "            # sampling ratio for training data.\n",
    "            \"subsample\": trial.suggest_float(\"subsample\", 0.2, 1.0),\n",
    "            # sampling according to each tree.\n",
    "            \"colsample_bytree\": trial.suggest_float(\"colsample_bytree\", 0.2, 1.0),\n",
    "        }\n",
    "\n",
    "        if param[\"booster\"] in [\"gbtree\", \"dart\"]:\n",
    "            # maximum depth of the tree, signifies complexity of the tree.\n",
    "            param[\"max_depth\"] = trial.suggest_int(\"max_depth\", 3, 9, step=2)\n",
    "            # minimum child weight, larger the term more conservative the tree.\n",
    "            param[\"min_child_weight\"] = trial.suggest_int(\"min_child_weight\", 2, 10)\n",
    "            param[\"eta\"] = trial.suggest_float(\"eta\", 1e-8, 1.0, log=True)\n",
    "            # defines how selective algorithm is.\n",
    "            param[\"gamma\"] = trial.suggest_float(\"gamma\", 1e-8, 1.0, log=True)\n",
    "            param[\"grow_policy\"] = trial.suggest_categorical(\"grow_policy\", [\"depthwise\", \"lossguide\"])\n",
    "\n",
    "        if param[\"booster\"] == \"dart\":\n",
    "            param[\"sample_type\"] = trial.suggest_categorical(\"sample_type\", [\"uniform\", \"weighted\"])\n",
    "            param[\"normalize_type\"] = trial.suggest_categorical(\"normalize_type\", [\"tree\", \"forest\"])\n",
    "            param[\"rate_drop\"] = trial.suggest_float(\"rate_drop\", 1e-8, 1.0, log=True)\n",
    "            param[\"skip_drop\"] = trial.suggest_float(\"skip_drop\", 1e-8, 1.0, log=True)\n",
    "    elif classifier_name == \"LightGBM\":\n",
    "        dtrain = lgb.Dataset(df_train[feat_cols], label=df_train[\"stance\"])\n",
    "\n",
    "        param = {\n",
    "            \"objective\": \"binary\",\n",
    "            \"metric\": \"binary_logloss\",\n",
    "            \"verbosity\": -1,\n",
    "            \"boosting_type\": \"gbdt\",\n",
    "            \"lambda_l1\": trial.suggest_float(\"lambda_l1\", 1e-8, 1.0, log=True),\n",
    "            \"lambda_l2\": trial.suggest_float(\"lambda_l2\", 1e-8, 1.0, log=True),\n",
    "            \"num_leaves\": trial.suggest_int(\"num_leaves\", 2, 256),\n",
    "            \"feature_fraction\": trial.suggest_float(\"feature_fraction\", 0.4, 1.0, step=0.1),\n",
    "            \"bagging_fraction\": trial.suggest_float(\"bagging_fraction\", 0.4, 1.0, step=0.1),\n",
    "            \"bagging_freq\": trial.suggest_int(\"bagging_freq\", 1, 3),\n",
    "            \"min_child_samples\": trial.suggest_int(\"min_child_samples\", 10, 100, step=10),\n",
    "        }\n",
    "\n",
    "\n",
    "\n",
    "    # score = sklearn.model_selection.cross_val_score(classifier_obj, df_train[feat_cols], df_train[\"stance\"], n_jobs=-1, cv=3)\n",
    "    # accuracy = score.mean()\n",
    "    # # todo use something else then mean?\n",
    "    # return accuracy\n",
    "    if classifier_name == \"XGBoost\":\n",
    "        bst = xgb.train(param, dtrain)\n",
    "        preds_prec = bst.predict(dvalid)\n",
    "        preds = np.rint(preds_prec)\n",
    "    elif classifier_name == \"LightGBM\":\n",
    "        gbm = lgb.train(param, dtrain)\n",
    "        preds_prec = gbm.predict(df_dev[feat_cols])\n",
    "        preds = np.rint(preds_prec)\n",
    "    else:\n",
    "        classifier_obj.fit(df_train[feat_cols], df_train[\"stance\"])\n",
    "        preds = classifier_obj.predict(df_dev[feat_cols])\n",
    "\n",
    "    acc = sklearn.metrics.accuracy_score(preds, df_dev[\"stance\"])\n",
    "    precision, recall, f1, support = sklearn.metrics.precision_recall_fscore_support(y_true=df_dev[\"stance\"], y_pred=preds, average=\"macro\")\n",
    "    precision_per_label, recall_per_label, f1_per_label, support = sklearn.metrics.precision_recall_fscore_support(y_true=df_dev[\"stance\"], y_pred=preds, average=None, labels=[0,1])\n",
    "\n",
    "    return acc, f1, recall, precision, f1_per_label, recall_per_label, precision_per_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99544939-d69e-481f-9995-8391efe777c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_eval_lightgbm(params, setting):\n",
    "    dtrain = lgb.Dataset(df_train[feat_cols], label=df_train[\"stance\"])\n",
    "    param = {\n",
    "            \"objective\": \"binary\",\n",
    "            \"metric\": \"binary_logloss\",\n",
    "            \"verbosity\": -1,\n",
    "            \"boosting_type\": \"gbdt\",\n",
    "            \"lambda_l1\": params[\"lambda_l1\"],\n",
    "            \"lambda_l2\": params[\"lambda_l2\"],\n",
    "            \"num_leaves\": params[\"num_leaves\"],\n",
    "            \"feature_fraction\": params[\"feature_fraction\"],\n",
    "            \"bagging_fraction\": params[\"bagging_fraction\"],\n",
    "            \"bagging_freq\": params[\"bagging_freq\"],\n",
    "            \"min_child_samples\": params[\"min_child_samples\"],\n",
    "        }\n",
    "    gbm = lgb.train(param, dtrain)\n",
    "    if setting == \"dev\":\n",
    "        preds_prec = gbm.predict(df_dev[feat_cols])\n",
    "        preds = np.rint(preds_prec)\n",
    "        acc = sklearn.metrics.accuracy_score(preds, df_dev[\"stance\"])\n",
    "        precision, recall, f1, support = sklearn.metrics.precision_recall_fscore_support(y_true=df_dev[\"stance\"], y_pred=preds, average=\"macro\")\n",
    "        precision_per_label, recall_per_label, f1_per_label, support = sklearn.metrics.precision_recall_fscore_support(y_true=df_dev[\"stance\"], y_pred=preds, average=None, labels=[0,1])\n",
    "    elif setting == \"test\":\n",
    "        preds_prec = gbm.predict(df_test[feat_cols])\n",
    "        preds = np.rint(preds_prec)\n",
    "        acc = sklearn.metrics.accuracy_score(preds, df_test[\"stance\"])\n",
    "        precision, recall, f1, support = sklearn.metrics.precision_recall_fscore_support(y_true=df_test[\"stance\"], y_pred=preds, average=\"macro\")\n",
    "        precision_per_label, recall_per_label, f1_per_label, support = sklearn.metrics.precision_recall_fscore_support(y_true=df_test[\"stance\"], y_pred=preds, average=None, labels=[0,1])\n",
    "        \n",
    "    \n",
    "    \n",
    "    return acc, f1, recall, precision, f1_per_label, recall_per_label, precision_per_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83434505-5eec-4748-b5f3-7c91aa929e6d",
   "metadata": {},
   "outputs": [],
   "source": [
    "best_parametr_mean = {\"classifier\": \"LightGBM\", \"lambda_l1\": 0.025, \"lambda_l2\": 0.05, \"num_leaves\": 200, \"feature_fraction\": 0.8, \"bagging_fraction\": 0.8, \"bagging_freq\": 2, \"min_child_samples\": 48}\n",
    "best_parameter_median = {\"classifier\": \"LightGBM\", \"lambda_l1\": 0.0001, \"lambda_l2\": 0.002, \"num_leaves\": 220, \"feature_fraction\": 0.9, \"bagging_fraction\": 0.8, \"bagging_freq\": 2, \"min_child_samples\": 50}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a2610a3-f25f-4671-b0b3-1a30f4329a99",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "best_results = list()\n",
    "best_params = best_parameter_median\n",
    "search_params = False\n",
    "feat_names = [\"bert-base\", \"bert-large\", \"distilbert\", \"all_lms\"]\n",
    "for split_perc in [\"all\", \"80\", \"40\"]:  # \"40\", \n",
    "    for model_conns in [\"all_conns_roberta\"]:  # , \"all_conns_bert\"]:\n",
    "        if model_conns == \"all_conns_roberta\":\n",
    "            feat_names.extend([\"roberta-base\", \"roberta-large\", \"xlm-roberta-base\", \"xlm-roberta-large\"])\n",
    "        for split_conn in [\"attack_support\", \"attack\", \"support\"]:  # , \"all\"]:\n",
    "            for filename in [\"ibmcs_all.csv\", \"argmin_all.csv\", \"perspectrum_all.csv\"]:  # , \"ibmargq_all.csv\"]:  \n",
    "                for feat_name in feat_names:\n",
    "                    # for i in range(0,5):\n",
    "                    df = pd.read_csv(\"features/\"+model_conns+\"/\"+split_perc+\"_\"+split_conn+\"/\"+filename)\n",
    "                    if feat_name == \"all_lms\":\n",
    "                        feat_cols = [col for col in df.columns if \"bert\" in col]\n",
    "                    else: \n",
    "                        feat_cols = [col for col in df.columns if col.startswith(feat_name)]\n",
    "                    # split_conn = \"attack_support\"\n",
    "                    # filename = \"ibmcs_all.csv\"\n",
    "                    print(split_perc, split_conn, filename, feat_name)\n",
    "                    overview_line = {\"split_perc\": split_perc, \"split_conn\": split_conn, \"filename\": filename, \"feat_name\": feat_name}\n",
    "                    df_train = df[df[\"set\"]== \"train\"]\n",
    "                    if \"dev\" in set(df[\"set\"]):\n",
    "                        df_dev = df[df[\"set\"]== \"dev\"]\n",
    "                    elif \"val\" in set(df[\"set\"]):\n",
    "                        df_dev = df[df[\"set\"]== \"val\"]\n",
    "                    df_test = df[df[\"set\"]== \"test\"]\n",
    "\n",
    "                    # for i in range(5):\n",
    "                    i = 0\n",
    "                    if search_params:\n",
    "                        study = optuna.create_study(direction=\"maximize\", sampler=TPESampler())\n",
    "                        study.optimize(objective_optuna, n_trials=250, show_progress_bar=False, timeout=600,  gc_after_trial=True, n_jobs=-1)\n",
    "                        # study.optimize(detailed_objective_dev, n_trials=2, show_progress_bar=True)\n",
    "                        # print(study.best_trial)\n",
    "                        out_df = study.trials_dataframe()\n",
    "                        out_df.loc[out_df['number'] == study.best_trial.number, 'best_trial'] = 1\n",
    "                        out_df.loc[out_df['number'] != study.best_trial.number, 'best_trial'] = 0\n",
    "                        overview_line.update(study.best_trial.params)\n",
    "\n",
    "                        acc_dev, f1_dev, recall_dev, precision_dev, f1_per_label_dev, recall_per_label_dev, precision_per_label_dev = detailed_objective_dev(study.best_trial)\n",
    "                        acc_test, f1_test, recall_test, precision_test, f1_per_label_test, recall_per_label_test, precision_per_label_test = detailed_objective_test(study.best_trial)\n",
    "                    else:\n",
    "                        acc_dev, f1_dev, recall_dev, precision_dev, f1_per_label_dev, recall_per_label_dev, precision_per_label_dev = train_eval_lightgbm(best_params, \"dev\")\n",
    "                        acc_test, f1_test, recall_test, precision_test, f1_per_label_test, recall_per_label_test, precision_per_label_test = train_eval_lightgbm(best_params, \"test\")\n",
    "                    print(acc_dev, f1_dev, recall_dev, precision_dev)\n",
    "                    if search_params:\n",
    "                        out_df.loc[out_df['number'] == study.best_trial.number, 'dev_acc'] = acc_dev\n",
    "                        out_df.loc[out_df['number'] == study.best_trial.number, 'dev_recall_macro'] = recall_dev\n",
    "                        out_df.loc[out_df['number'] == study.best_trial.number, 'dev_precision_macro'] = precision_dev\n",
    "                        out_df.loc[out_df['number'] == study.best_trial.number, 'dev_f1_macro'] = f1_dev\n",
    "                        out_df.loc[out_df['number'] == study.best_trial.number, 'dev_recall_0'] = recall_per_label_dev[0]\n",
    "                        out_df.loc[out_df['number'] == study.best_trial.number, 'dev_precision_0'] = precision_per_label_dev[0]\n",
    "                        out_df.loc[out_df['number'] == study.best_trial.number, 'dev_f1_0'] = f1_per_label_dev[0]\n",
    "                        out_df.loc[out_df['number'] == study.best_trial.number, 'dev_recall_1'] = recall_per_label_dev[1]\n",
    "                        out_df.loc[out_df['number'] == study.best_trial.number, 'dev_precision_1'] = precision_per_label_dev[1]\n",
    "                        out_df.loc[out_df['number'] == study.best_trial.number, 'dev_f1_1'] = f1_per_label_dev[1]\n",
    "                        out_df.loc[out_df['number'] == study.best_trial.number, 'run'] = i\n",
    "                    overview_line.update({\"run\": i, \"f1_dev\": f1_dev, \"recall_dev\": recall_dev, \"precision_dev\": precision_dev, \"f1_dev_0\": f1_per_label_dev[0], \"recall_dev_0\": recall_per_label_dev[0], \"prec_dev_0\": precision_per_label_dev[0], \"f1_dev_1\": f1_per_label_dev[1], \"recall_dev_1\": recall_per_label_dev[1], \"prec_dev_1\": precision_per_label_dev[1]})\n",
    "\n",
    "\n",
    "                    print(acc_test, f1_test, recall_test, precision_test)\n",
    "                    if search_params:\n",
    "                        out_df.loc[out_df['number'] == study.best_trial.number, 'test_acc'] = acc_test\n",
    "                        out_df.loc[out_df['number'] == study.best_trial.number, 'test_recall_macro'] = recall_test\n",
    "                        out_df.loc[out_df['number'] == study.best_trial.number, 'test_precision_macro'] = precision_test\n",
    "                        out_df.loc[out_df['number'] == study.best_trial.number, 'test_f1_macro'] = f1_test\n",
    "                        out_df.loc[out_df['number'] == study.best_trial.number, 'test_recall_0'] = recall_per_label_test[0]\n",
    "                        out_df.loc[out_df['number'] == study.best_trial.number, 'test_precision_0'] = precision_per_label_test[0]\n",
    "                        out_df.loc[out_df['number'] == study.best_trial.number, 'test_f1_0'] = f1_per_label_test[0]\n",
    "                        out_df.loc[out_df['number'] == study.best_trial.number, 'test_recall_1'] = recall_per_label_test[1]\n",
    "                        out_df.loc[out_df['number'] == study.best_trial.number, 'test_precision_1'] = precision_per_label_test[1]\n",
    "                        out_df.loc[out_df['number'] == study.best_trial.number, 'test_f1_1'] = f1_per_label_test[1]\n",
    "                        out_df.to_csv(\"results/run\"+str(run)+\"_\"+model_conns+\"_\"+filename+\"_\"+split_perc+\"_\"+split_conn+\"_\"+feat_name+\".csv\")\n",
    "                    overview_line.update({\"run\": i, \"f1_test\": f1_test, \"recall_test\": recall_test, \"precision_test\": precision_test, \"f1_test_0\": f1_per_label_test[0], \"recall_test_0\": recall_per_label_test[0], \"prec_test_0\": precision_per_label_test[0], \"f1_test_1\": f1_per_label_test[1], \"recall_test_1\": recall_per_label_test[1], \"prec_test_1\": precision_per_label_test[1]})\n",
    "                    best_results.append(overview_line)\n",
    "                    pd.DataFrame(best_results).to_csv(\"results/overview.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75230323-9903-4502-8275-63aa95ce5781",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "out_df[out_df[\"best_trial\"]==1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c06a90a4-1829-44aa-a875-4ccd6338387f",
   "metadata": {},
   "outputs": [],
   "source": [
    "study.trials_dataframe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f6602500-de7d-4dc2-aac9-400f2b59866d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "study.best_trial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca78e0c5-739d-4e58-b87f-4313462ce126",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "study.best_trial.params[\"classifier\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f45e6741-4ecf-4a79-80ec-9e7da258d58a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}