test attension.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %load models.py\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "from keras.models import Model, Input\n",
    "from keras.layers import Dense, LSTM, Dropout, Embedding, SpatialDropout1D, Bidirectional, concatenate, InputSpec\n",
    "from keras.layers import GlobalAveragePooling1D, GlobalMaxPooling1D\n",
    "from keras.optimizers import Adam, RMSprop\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.preprocessing import MultiLabelBinarizer, LabelEncoder\n",
    "import regex as re\n",
    "import pickle\n",
    "import numpy as np\n",
    "\n",
    "from keras.engine.topology import Layer\n",
    "from keras import initializers as initializers, regularizers, constraints\n",
    "from keras import backend as K\n",
    "\n",
    "class KerasTextClassifier(BaseEstimator, TransformerMixin):\n",
    "    '''Wrapper class for keras text classification models that takes raw text as input.'''\n",
    "    \n",
    "    def __init__(self, max_words=30000, input_length=50, emb_dim=50, n_classes=10):\n",
    "        self.max_words = max_words\n",
    "        self.input_length = input_length\n",
    "        self.emb_dim = emb_dim\n",
    "        self.n_classes = n_classes\n",
    "        self.return_attention = True\n",
    "        self.model = self._get_model()\n",
    "        self.encoder = LabelEncoder()\n",
    "        self.tokenizer = Tokenizer(num_words=self.max_words+1, filters='!\"#$%&()*+,-.:;=?@[\\\\]^_`{|}~\\t\\n', lower=True, split=' ', oov_token=\"UNK\")\n",
    "    \n",
    "    def _get_model(self):\n",
    "        d = 0.5\n",
    "        rd = 0.5\n",
    "        rnn_units = 128\n",
    "        input_text = Input((self.input_length,))\n",
    "        text_embedding = Embedding(input_dim=self.max_words + 2, output_dim=self.emb_dim,\n",
    "                                   input_length=self.input_length, mask_zero=True)(input_text)\n",
    "        text_embedding = SpatialDropout1D(0.5)(text_embedding)\n",
    "        bilstm = Bidirectional(LSTM(units=rnn_units, return_sequences=True, dropout=d,\n",
    "                                    recurrent_dropout=rd))(text_embedding)\n",
    "        x, attn = AttentionWeightedAverage(return_attention=True)(bilstm)\n",
    "        x = Dropout(0.5)(x)\n",
    "        out = Dense(units=self.n_classes, activation=\"softmax\")(x)\n",
    "        model = Model(input_text, out)\n",
    "        return model\n",
    "    \n",
    "    def _get_attention_map(self, texts):\n",
    "        att_model_output = self.model.layers[0:-2]\n",
    "        att_model = Model(att_model_output[0].input, att_model_output[-1].output)\n",
    "        att_model.compile(optimizer=RMSprop(),\n",
    "                          loss=\"sparse_categorical_crossentropy\",\n",
    "                          metrics=[\"accuracy\"])\n",
    "        return att_model.predict(self._get_sequences(texts))[1]\n",
    "    \n",
    "    def _get_sequences(self, texts):\n",
    "        seqs = self.tokenizer.texts_to_sequences(texts)\n",
    "        return pad_sequences(seqs, maxlen=self.input_length,\n",
    "                             value=0, padding='post', truncating='post')\n",
    "    \n",
    "    def _labels(self, labels):\n",
    "        return self.encoder.transform(labels)\n",
    "    \n",
    "    def fit(self, X, y, X_val=None, y_val=None, lr=0.001, resume=False,\n",
    "            epochs=10, batch_size=32):\n",
    "        '''\n",
    "        Fit the vocabulary and the model.\n",
    "        \n",
    "        :params:\n",
    "        X: list of texts\n",
    "        y: labels\n",
    "        X_val: list of texts for validation\n",
    "        y_val: labels for validation.\n",
    "        '''\n",
    "        self.model.compile(optimizer=RMSprop(clipnorm=10., lr=lr),\n",
    "                           loss=\"sparse_categorical_crossentropy\",\n",
    "                           metrics=[\"accuracy\"])\n",
    "        \n",
    "        if not resume:\n",
    "            self.tokenizer.fit_on_texts(X)\n",
    "            self.encoder.fit(y)\n",
    "            self.tokenizer.word_index = {e: i for e,i in self.tokenizer.word_index.items() if i <= self.max_words}\n",
    "            self.tokenizer.word_index[self.tokenizer.oov_token] = self.max_words + 1\n",
    "        else:\n",
    "            print(\"Resuming training...\")\n",
    "        seqs = self._get_sequences(X)\n",
    "        categorical_y = self._labels(y)\n",
    "        print(\"Fit text model with {} classes\".format(len(self.encoder.classes_)))\n",
    "        if X_val:\n",
    "            val_seqs = self._get_sequences(X_val)\n",
    "            categorical_y_val = self._labels(y_val)\n",
    "            self.model.fit(seqs, categorical_y, batch_size=batch_size,\n",
    "                           epochs=epochs, validation_data=(val_seqs, categorical_y_val))\n",
    "        else:\n",
    "            self.model.fit(seqs, categorical_y, batch_size=batch_size,\n",
    "                           epochs=epochs, validation_split=0.1)\n",
    "    \n",
    "    def predict_proba(self, X, y=None):\n",
    "        return self.model.predict(self._get_sequences(X))\n",
    "    \n",
    "    def predict(self, X, y=None):\n",
    "        return np.argmax(self.predict_proba(X), axis=1)\n",
    "    \n",
    "    def save(self, path=\"model\"):                               \n",
    "        self.model.save_weights('{}_weights.h5'.format(path))          \n",
    "        with open(\"{}_index.pkl\".format(path), \"wb\") as f:                      \n",
    "            pickle.dump([self.encoder, self.tokenizer, self.max_words,\n",
    "                         self.emb_dim, self.input_length, self.n_classes], f)         \n",
    "            \n",
    "    def load(self, path=\"model\"):                                                              \n",
    "        with open(\"{}_index.pkl\".format(path), \"rb\") as f:\n",
    "            self.encoder, self.tokenizer, self.max_words, self.emb_dim, self.input_length, self.n_classes = pickle.load(f)                                                                     \n",
    "        self.model = self._get_model()                                           \n",
    "        self.model.load_weights('{}_weights.h5'.format(path))\n",
    "        \n",
    "        \n",
    "class AttentionWeightedAverage(Layer):\n",
    "    \"\"\"\n",
    "    Computes a weighted average attention mechanism from:\n",
    "        Zhou, Peng, Wei Shi, Jun Tian, Zhenyu Qi, Bingchen Li, Hongwei Hao and Bo Xu.\n",
    "        “Attention-Based Bidirectional Long Short-Term Memory Networks for Relation Classification.”\n",
    "        ACL (2016). http://www.aclweb.org/anthology/P16-2034\n",
    "    How to use:\n",
    "    see: [BLOGPOST]\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, return_attention=False, **kwargs):\n",
    "        self.init = initializers.get('uniform')\n",
    "        self.supports_masking = True\n",
    "        self.return_attention = return_attention\n",
    "        super(AttentionWeightedAverage, self).__init__(** kwargs)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        self.input_spec = [InputSpec(ndim=3)]\n",
    "        assert len(input_shape) == 3\n",
    "\n",
    "        self.w = self.add_weight(shape=(input_shape[2], 1),\n",
    "                                 name='{}_w'.format(self.name),\n",
    "                                 initializer=self.init)\n",
    "        self.trainable_weights = [self.w]\n",
    "        super(AttentionWeightedAverage, self).build(input_shape)\n",
    "\n",
    "    def call(self, h, mask=None):\n",
    "        h_shape = K.shape(h)\n",
    "        d_w, T = h_shape[0], h_shape[1]\n",
    "        \n",
    "        logits = K.dot(h, self.w)  # w^T h\n",
    "        logits = K.reshape(logits, (d_w, T))\n",
    "        alpha = K.exp(logits - K.max(logits, axis=-1, keepdims=True))  # exp\n",
    "        \n",
    "        # masked timesteps have zero weight\n",
    "        if mask is not None:\n",
    "            mask = K.cast(mask, K.floatx())\n",
    "            alpha = alpha * mask\n",
    "        alpha = alpha / K.sum(alpha, axis=1, keepdims=True) # softmax\n",
    "        r = K.sum(h * K.expand_dims(alpha), axis=1)  # r = h*alpha^T\n",
    "        h_star = K.tanh(r)  # h^* = tanh(r)\n",
    "        if self.return_attention:\n",
    "            return [h_star, alpha]\n",
    "        return h_star\n",
    "\n",
    "    def get_output_shape_for(self, input_shape):\n",
    "        return self.compute_output_shape(input_shape)\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        output_len = input_shape[2]\n",
    "        if self.return_attention:\n",
    "            return [(input_shape[0], output_len), (input_shape[0], input_shape[1])]\n",
    "        return (input_shape[0], output_len)\n",
    "\n",
    "    def compute_mask(self, input, input_mask=None):\n",
    "        if isinstance(input_mask, list):\n",
    "            return [None] * len(input_mask)\n",
    "        else:\n",
    "            return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import numpy as np \n",
    "import scipy as sp \n",
    "import sklearn\n",
    "import random \n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "import time \n",
    "from sklearn import preprocessing, model_selection\n",
    "from keras.models import Sequential \n",
    "from keras.layers import Dense \n",
    "from keras.utils import np_utils\n",
    "from imblearn.over_sampling import RandomOverSampler\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from keras.utils.np_utils import to_categorical\n",
    "from sklearn.utils import shuffle\n",
    "from keras.layers import Embedding, LSTM, SpatialDropout1D\n",
    "data = pd.read_csv('camelversion.csv')\n",
    "#data = shuffle(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2784,)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#data = data.drop(['name', 'version', 'name'], axis =1)\n",
    "X = data.drop(['bug'], axis = 1)\n",
    "X = np.array(X)\n",
    "Y = data['bug']\n",
    "Y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        wmc       dit       noc       cbo       rfc      lcom        ca  \\\n",
      "0  0.024096  0.333333  0.000000  0.013393  0.024845  0.000441  0.004484   \n",
      "1  0.036145  0.500000  0.000000  0.046875  0.102484  0.001102  0.002242   \n",
      "2  0.012048  0.500000  0.000000  0.006696  0.021739  0.000073  0.000000   \n",
      "3  0.156627  0.166667  0.025641  0.022321  0.145963  0.000000  0.011211   \n",
      "4  0.024096  0.500000  0.000000  0.008929  0.059006  0.000441  0.002242   \n",
      "\n",
      "         ce       npm  lcom3  ...  dam       moa       mfa       cam    ic  \\\n",
      "0  0.065789  0.025478   1.00  ...  0.0  0.000000  0.896552  0.500000  0.00   \n",
      "1  0.276316  0.012739   1.00  ...  0.0  0.000000  0.800000  0.500000  0.50   \n",
      "2  0.039474  0.006369   1.00  ...  0.0  0.000000  0.833333  0.666667  0.25   \n",
      "3  0.065789  0.152866   0.04  ...  1.0  0.111111  0.000000  0.258242  0.00   \n",
      "4  0.052632  0.019108   1.00  ...  0.0  0.000000  0.888889  0.375000  0.25   \n",
      "\n",
      "        cbm       amc    max_cc    avg_cc  bug  \n",
      "0  0.000000  0.034664  0.030303  0.088235    0  \n",
      "1  0.095238  0.178571  0.030303  0.078435    0  \n",
      "2  0.047619  0.069328  0.030303  0.058824    0  \n",
      "3  0.000000  0.050663  0.060606  0.117647    0  \n",
      "4  0.047619  0.091387  0.030303  0.058824    0  \n",
      "\n",
      "[5 rows x 21 columns]\n"
     ]
    }
   ],
   "source": [
    "scaler = MinMaxScaler()\n",
    "MinMaxScaler(copy=True, feature_range=(0, 1))\n",
    "cols_to_norm = ['wmc', 'dit', 'noc', 'cbo', 'rfc', 'lcom', 'ca', 'ce', 'npm', 'lcom3', 'loc', 'dam', 'moa', 'mfa', 'cam', 'ic', 'cbm', 'amc', 'max_cc', 'avg_cc']\n",
    "data[cols_to_norm] = MinMaxScaler().fit_transform(data[cols_to_norm])\n",
    "print(data.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "from models import KerasTextClassifier\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %load attensions.py\n",
    "class attention(Layer):\n",
    "    def __init__(self,**kwargs):\n",
    "        super(attention,self).__init__(**kwargs)\n",
    "\n",
    "    def build(self,input_shape):\n",
    "        self.W=self.add_weight(name=\"att_weight\",shape=(input_shape[-1],1),initializer=\"normal\")\n",
    "        self.b=self.add_weight(name=\"att_bias\",shape=(input_shape[1],1),initializer=\"zeros\")        \n",
    "        super(attention, self).build(input_shape)\n",
    "\n",
    "    def call(self,x):\n",
    "        et=K.squeeze(K.tanh(K.dot(x,self.W)+self.b),axis=-1)\n",
    "        at=K.softmax(et)\n",
    "        at=K.expand_dims(at,axis=-1)\n",
    "        output=x*at\n",
    "        return K.sum(output,axis=1)\n",
    "\n",
    "    def compute_output_shape(self,input_shape):\n",
    "        return (input_shape[0],input_shape[-1])\n",
    "\n",
    "    def get_config(self):\n",
    "        return super(attention,self).get_config()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'embeddings' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-47-6fecf8f25398>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mvocab_size\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0membeddings\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[0membedding_size\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0membeddings\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;31m#ros = RandomOverSampler(random_state=0)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;31m#X_resampled, Y_resampled = ros.fit_resample(X, Y)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;31m#tr_sent, te_sent, tr_rel, te_rel = train_test_split(sentences, relations, test_size=0.1)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'embeddings' is not defined"
     ]
    }
   ],
   "source": [
    "vocab_size = embeddings.shape[0]\n",
    "embedding_size = embeddings.shape[1]\n",
    "#ros = RandomOverSampler(random_state=0)\n",
    "#X_resampled, Y_resampled = ros.fit_resample(X, Y)\n",
    "#tr_sent, te_sent, tr_rel, te_rel = train_test_split(sentences, relations, test_size=0.1)\n",
    "train_x, test_x, train_y, test_y = model_selection.train_test_split(X_resampled,Y_resampled,test_size = 0.2, random_state = 0)\n",
    "train_x = np.reshape(train_x, (train_x.shape[0], 1, train_x.shape[1]))\n",
    "test_x = np.reshape(test_x, (test_x.shape[0], 1, test_x.shape[1]))\n",
    "input_dim = len(data.columns) - 1\n",
    "#kclf.fit(X=tr_sent, y=tr_rel, X_val=te_sent, y_val=te_rel,\n",
    " #        batch_size=10, lr=0.001, epochs=20)\n",
    "model = Sequential()\n",
    "#model.add(Embedding(1024, 1, input_length=20))\n",
    "model.add(LSTM(100, input_shape = (1,20), dropout = 0.2, return_sequences=True))\n",
    "#model.add(LSTM(512, dropout = 0.3, return_sequences=True))\n",
    "#model.add(LSTM(256, dropout = 0.4, return_sequences=True))\n",
    "model.add(LSTM(80, dropout = 0.2, return_sequences=True))\n",
    "model.add(LSTM(60, dropout = 0.2, return_sequences=True))\n",
    "model.add(Embedding(\n",
    "        input_dim=21,\n",
    "        output_dim=4,\n",
    "        input_length=100,\n",
    "        trainable=False,\n",
    "        mask_zero=True,\n",
    "        weights=[1]\n",
    "    ))\n",
    "#model.add(attensions(16, input_shape=(1,20), dropout = 0.67))\n",
    "#model.add(Dense(1, activation='softmax'))\n",
    "#model.add(SpatialDropout1D(0.7))\n",
    "#model.add(LSTM(64, dropout=0.7))\n",
    "#model.add(LSTM(16, dropout=0.7, return_sequences=True))\n",
    "#model.add(LSTM(8, dropout=0.7))\n",
    "#model.add(Dense(10, input_dim = input_dim , activation = 'relu'))\n",
    "#model.add(Dense(10, activation = 'relu'))\n",
    "#model.add(Dense(8, activation = 'relu'))\n",
    "#model.add(Dense(8, activation = 'relu'))\n",
    "#model.add(Dense(6, activation = 'relu'))\n",
    "#model.add(Dense(32, activation = 'relu'))\n",
    "model.add(Dense(29, activation = 'softmax'))\n",
    "model.compile(loss = 'sparse_categorical_crossentropy' , optimizer = 'adam' , metrics = ['accuracy','mse', 'mae', 'mape', 'cosine'] )\n",
    "history = model.fit(train_x, train_y, validation_split = 0.3, epochs = 50, batch_size = 50)\n",
    "score = model.evaluate(test_x, test_y)\n",
    "print(history.history.keys())\n",
    "model.test_on_batch(test_x, test_y)\n",
    "model.metrics_names\n",
    "print(\"\\n%s: %.2f%%\" % (model.metrics_names[1], score[1]*100))\n",
    "print(\"\\n%s: %.2f%%\" % (model.metrics_names[2], score[2]*100))\n",
    "print(\"\\n%s: %.2f%%\" % (model.metrics_names[3], score[3]*100))\n",
    "print(\"\\n%s: %.2f%%\" % (model.metrics_names[4], score[4]))\n",
    "print(\"\\n%s: %.2f%%\" % (model.metrics_names[5], score[5]*100))\n",
    "#print(history.history.keys())\n",
    "plt.plot(history.history['acc'])\n",
    "plt.plot(history.history['val_acc'])\n",
    "plt.title('model accuracy')\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'val'], loc='upper left')\n",
    "plt.show()\n",
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "#plt.title\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs=Input((21,))\n",
    "x=Embedding(input_dim=+1,output_dim=32,input_length=features,\\\n",
    "            embeddings_regularizer=keras.regularizers.l2(.001))(inputs)\n",
    "att_in=LSTM(no_of_neurons,return_sequences=True,dropout=0.3,recurrent_dropout=0.2)(x)\n",
    "att_out=attention()(att_in)\n",
    "outputs=Dense(1,activation='softmax',trainable=True)(att_out)\n",
    "model=Model(inputs,outputs)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.add(Embedding(\n",
    "        input_dim=vocab_size,\n",
    "        output_dim=embedding_size,\n",
    "        input_length=max_length,\n",
    "        trainable=False,\n",
    "        mask_zero=True,\n",
    "        weights=[embeddings]\n",
    "    ))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}