3embeddings-sampled-lstm.ipynb

{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"scrolled":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tensorflow import set_random_seed\n\nnp.random.seed(1)\nset_random_seed(1)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"00f95dd996737f9214040cd8964096619a671389"},"cell_type":"code","source":"import nltk, re, string\nfrom nltk.corpus import stopwords\nfrom nltk.stem import SnowballStemmer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d863603e48fe4d68ac05a5de353b8bb58038de27"},"cell_type":"code","source":"def clean_text(text):\n    print(text)\n    \n    ## Remove puncuation\n    text = text.translate(string.punctuation)\n    \n    ## Convert words to lower case and split them\n    text = text.lower()\n    \n    ## Remove stop words\n    #text = text.split()\n    #stops = set(stopwords.words(\"english\"))\n    #text = [w for w in text if not w in stops and len(w) >= 3]\n    \n    #text = \" \".join(text)\n\n    # Clean the text\n    text = re.sub(r\"[^A-Za-z0-9^,!.\\/'+-=]\", \" \", text)\n    text = re.sub(r\"what's\", \"what is \", text)\n    text = re.sub(r\"\\'s\", \" \", text)\n    text = re.sub(r\"\\'ve\", \" have \", text)\n    text = re.sub(r\"n't\", \" not \", text)\n    text = re.sub(r\"i'm\", \"i am \", text)\n    text = re.sub(r\"\\'re\", \" are \", text)\n    text = re.sub(r\"\\'d\", \" would \", text)\n    text = re.sub(r\"\\'ll\", \" will \", text)\n    text = re.sub(r\",\", \" \", text)\n    text = re.sub(r\"\\.\", \" \", text)\n    text = re.sub(r\"!\", \" ! \", text)\n    text = re.sub(r\"\\/\", \" \", text)\n    text = re.sub(r\"\\^\", \" ^ \", text)\n    text = re.sub(r\"\\+\", \" + \", text)\n    text = re.sub(r\"\\-\", \" - \", text)\n    text = re.sub(r\"\\=\", \" = \", text)\n    text = re.sub(r\"'\", \" \", text)\n    text = re.sub(r\"(\\d+)(k)\", r\"\\g<1>000\", text)\n    text = re.sub(r\":\", \" : \", text)\n    text = re.sub(r\" e g \", \" eg \", text)\n    text = re.sub(r\" b g \", \" bg \", text)\n    text = re.sub(r\" u s \", \" american \", text)\n    text = re.sub(r\"\\0s\", \"0\", text)\n    text = re.sub(r\" 9 11 \", \"911\", text)\n    text = re.sub(r\"e - mail\", \"email\", text)\n    text = re.sub(r\"j k\", \"jk\", text)\n    text = re.sub(r\"\\s{2,}\", \" \", text)\n    \n    #text = text.split()\n    #stemmer = SnowballStemmer('english')\n    #stemmed_words = [stemmer.stem(word) for word in text]\n    #text = \" \".join(stemmed_words)\n\n    print(text)\n    print(\"\")\n    return text","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')\n#train_df[\"question_text\"] = train_df[\"question_text\"].map(lambda x: clean_text(x))\n\ntest_df = pd.read_csv('../input/quora-insincere-questions-classification/test.csv')\n#test_df[\"question_text\"] = test_df[\"question_text\"].map(lambda x: clean_text(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3061e388d57b70d44117d57ec305b81a96edc7e7"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe82b4a66241d31a4a64e94c94618fd3e4df817b"},"cell_type":"code","source":"train_0_df = train_df.loc[train_df[\"target\"]==0]  # 负例 (1225312, 3)\ntrain_1_df = train_df.loc[train_df[\"target\"]==1]  # 正例 (80810, 3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a1ee212ca560b861c0e817e46df33dd6ebd78934"},"cell_type":"code","source":"def down_sample(df): \n#     df1=df[df['target']==1]#正例 \n#     df2=df[df['target']==0]##负例 \n    df_new=df.sample(frac=0.8)##抽负例 \n    return df_new\n#     return pd.concat([df1,df3],ignore_index=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd83a5d40c47f9488ac26a8cbce52be3552943d0"},"cell_type":"code","source":"def up_sample(df): \n#     df1=df[df['target']==1]#正例 \n#     df2=df[df['target']==0]##负例 \n    df_new=pd.concat([df,df,df],ignore_index=True) \n#     return pd.concat([df2,df3],ignore_index=True)\n    return df_new","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"007154ea85189cac4c8ce3de048aec780cafc737"},"cell_type":"code","source":"train_0_down_df=down_sample(train_0_df) # frac=0.8 copy=3 (980250, 3)\ntrain_1_up_df=up_sample(train_1_df) # frac=0.8 copy=3 (242430, 3)\nprint(train_0_down_df.shape)\nprint(train_1_up_df.shape)\ntrain_df_new = pd.concat([train_0_down_df, train_1_up_df], ignore_index=True) # frac=0.8 copy=3 (1222680, 3)\nprint(train_df_new.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f60d53030e44d95ee023d473585d52202d371ec"},"cell_type":"code","source":"X_train = train_df_new[\"question_text\"].fillna(\"na\").values\nX_test = test_df[\"question_text\"].fillna(\"na\").values\ny = train_df_new[\"target\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3b5f8c01064f0936611dd3a36dc1478f1c005110"},"cell_type":"code","source":"num_0 = 0\nnum_1 = 0\nfor x in y:\n    if x==0:\n        num_0 += 1\n    else:\n        num_1 += 1\n        \nprint(\"num_0: \", num_0)  # 1225312\nprint(\"num_1: \", num_1)  # 80810","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1506ab48c373d4b281a8af968c30e1e956b9bd54"},"cell_type":"code","source":"print(\"y_size:\", y.shape)\ny_0 = 0\ny_1 = 1\nfor label in y:\n    if label==0:\n        y_0 += 1\n    if label==1:\n        y_1 += 1\nprint(\"y_0:\", y_0)\nprint(\"y_1:\", y_1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d565c03e9b7b2f133e4e8f4eb0ed687c355f1fc4"},"cell_type":"code","source":"from keras.models import Model\nfrom keras.layers import Input, Dense, Embedding, concatenate\nfrom keras.layers import CuDNNGRU, Bidirectional, GlobalAveragePooling1D, GlobalMaxPooling1D, Conv1D\nfrom keras.layers import Add, BatchNormalization, Activation, CuDNNLSTM, Dropout\nfrom keras.layers import *\nfrom keras.models import *\nfrom keras.preprocessing import text, sequence\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\nimport gc\nfrom sklearn import metrics\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca0edbb51d3a75ca4d667c5eca1dcdfb916a7fa7"},"cell_type":"code","source":"maxlen = 70\nmax_features = 50000\nembed_size = 300\n\ntokenizer = text.Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(X_train) + list(X_test))\n\nX_train = tokenizer.texts_to_sequences(X_train)\nX_test = tokenizer.texts_to_sequences(X_test)\n\nx_train = sequence.pad_sequences(X_train, maxlen=maxlen)\nx_test = sequence.pad_sequences(X_test, maxlen=maxlen)\nprint(\"finished!\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3f5c8eee53cf45ccfd1ce56f4011f1df00186a5e"},"cell_type":"code","source":"def attention_3d_block(inputs):\n    # inputs.shape = (batch_size, time_steps, input_dim)\n    TIME_STEPS = inputs.shape[1].value\n    SINGLE_ATTENTION_VECTOR = False\n    \n    input_dim = int(inputs.shape[2])\n    a = Permute((2, 1))(inputs)\n    a = Reshape((input_dim, TIME_STEPS))(a) # this line is not useful. It's just to know which dimension is what.\n    a = Dense(TIME_STEPS, activation='softmax')(a)\n    if SINGLE_ATTENTION_VECTOR:\n        a = Lambda(lambda x: K.mean(x, axis=1))(a)\n        a = RepeatVector(input_dim)(a)\n    a_probs = Permute((2, 1))(a)\n    output_attention_mul = Multiply()([inputs, a_probs])\n    return output_attention_mul","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"43ba65d695360fe604e235ead98e2bc38ecaeb1b"},"cell_type":"code","source":"from keras import backend as K\nfrom keras.engine.topology import Layer, InputSpec\nfrom keras import initializers\n\nclass AttLayer(Layer):\n    def __init__(self, attention_dim):\n        self.init = initializers.get('normal')\n        self.supports_masking = True\n        self.attention_dim = attention_dim\n        super(AttLayer, self).__init__()\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n        self.W = K.variable(self.init((input_shape[-1], self.attention_dim)))\n        self.b = K.variable(self.init((self.attention_dim, )))\n        self.u = K.variable(self.init((self.attention_dim, 1)))\n        self.trainable_weights = [self.W, self.b, self.u]\n        super(AttLayer, self).build(input_shape)\n\n    def compute_mask(self, inputs, mask=None):\n        return mask\n\n    def call(self, x, mask=None):\n        # size of x :[batch_size, sel_len, attention_dim]\n        # size of u :[batch_size, attention_dim]\n        # uit = tanh(xW+b)\n        uit = K.tanh(K.bias_add(K.dot(x, self.W), self.b))\n        ait = K.dot(uit, self.u)\n        ait = K.squeeze(ait, -1)\n\n        ait = K.exp(ait)\n\n        if mask is not None:\n            # Cast the mask to floatX to avoid float64 upcasting in theano\n            ait *= K.cast(mask, K.floatx())\n        ait /= K.cast(K.sum(ait, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n        ait = K.expand_dims(ait)\n        weighted_input = x * ait\n        output = K.sum(weighted_input, axis=1)\n\n        return output\n\n    def compute_output_shape(self, input_shape):\n        return (input_shape[0], input_shape[-1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"def48377e4113ab78d5b5bafc323f36d3b9232ba"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/quora-insincere-questions-classification/embeddings/glove.840B.300d/glove.840B.300d.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n\nall_embs = np.stack(embeddings_index.values()) # 二维数组 2096016*300\nprint(all_embs.shape[0])\nemb_mean,emb_std = all_embs.mean(), all_embs.std()  # scalar 均值，标准差\nprint(\"mean=\",emb_mean)\nembed_size = all_embs.shape[1]  # 300\n\nword_index = tokenizer.word_index  # 单词对应的整数编号形成的列表\nnb_words = min(max_features, len(word_index))  # 只取两者中较小者的单词数量\n# 结合下面的代码。对于embeddings中没有的单词，使用随机初始化的词向量\nembedding_matrix_1 = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: \n        continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: \n        embedding_matrix_1[i] = embedding_vector\n\ndel embeddings_index; gc.collect() # 内存空间清理\nprint(\"finished!\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"29c446e36dc3e0cfeb86b35828664f739d3b3eae"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/quora-insincere-questions-classification/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values())  # 999994*300\nprint(all_embs.shape[0])\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix_2 = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix_2[i] = embedding_vector\ndel embeddings_index; gc.collect()\nprint(\"finished!\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1cf537631d802e0d8434d75e8a3f5f39d1f73477"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/quora-insincere-questions-classification/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values()) # 1703755*300\nprint(all_embs.shape[0])\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix_3 = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix_3[i] = embedding_vector\n        \ndel embeddings_index; gc.collect()   \nprint(\"finished!\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5693a3faaabc51c49434b7057ec68fe3f6729756"},"cell_type":"markdown","source":"# Concatenating the embeddings"},{"metadata":{"trusted":true,"_uuid":"7d891d3452e22c4fa9f7e2781527618041a38f83"},"cell_type":"code","source":"embedding_matrix = np.concatenate((embedding_matrix_1, embedding_matrix_2, embedding_matrix_3), axis=1)  \ndel embedding_matrix_1, embedding_matrix_2, embedding_matrix_3\ngc.collect()\nprint(np.shape(embedding_matrix))\nprint(\"finished!\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2f14547280cb1ee50b34dc2d884d878520177073"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_tra, X_val, y_tra, y_val = train_test_split(x_train, y, test_size = 0.1, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"48b25baac8abdcf0ec2423f97b2b282f93c476de"},"cell_type":"markdown","source":"# MODEL 2: LSTM"},{"metadata":{"trusted":true,"_uuid":"37e7f8f9e9c1f21011e91b84f56b545f0934835e"},"cell_type":"code","source":"def model2():\n    inp = Input(shape=(maxlen, ))\n    embed = Embedding(max_features, embed_size * 3, weights=[embedding_matrix], trainable=False)(inp)\n    x = embed\n    \n    x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)\n    x = attention_3d_block(x)\n    x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)\n    x = AttLayer(64)(x)\n    x = Dropout(0.3)(x)\n    x = Dense(128, activation='relu')(x)\n    outp = Dense(1, activation=\"sigmoid\")(x)\n    model = Model(inputs=inp, outputs=outp)\n    model.compile(loss='binary_crossentropy',\n                  optimizer='adam',\n                  metrics=['accuracy'])    \n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"2d5cb289d080483143edc805fe68de4ad8705b83"},"cell_type":"code","source":"MODEL2 = model2()\nMODEL2.summary()\n\nbatch_size = 2048\nepochs = 3\n\nearly_stopping = EarlyStopping(patience=3, verbose=1, monitor='val_loss', mode='min')\nmodel_checkpoint = ModelCheckpoint('./model2.model', save_best_only=True, verbose=1, monitor='val_loss', mode='min')\nreduce_lr = ReduceLROnPlateau(factor=0.5, patience=3, min_lr=0.0001, verbose=1)\n\nhist = MODEL2.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val), verbose=True)\nMODEL2.save('./model2.h5')\nprint(\"finished!!\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a4ec75afa353a89fa6b785e6173e8e5584af6f4b"},"cell_type":"code","source":"pred_val_y_2 = MODEL2.predict([X_val], batch_size=1024, verbose=1)\nthresholds = []\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    acc = metrics.accuracy_score(y_val, (pred_val_y_2 > thresh).astype(int))\n    f1_score = metrics.f1_score(y_val, (pred_val_y_2 > thresh).astype(int))\n    pre = metrics.precision_score(y_val, (pred_val_y_2 > thresh).astype(int))\n    recall = metrics.recall_score(y_val, (pred_val_y_2 > thresh).astype(int))\n    thresholds.append([thresh, f1_score])\n    print(\"acc score at threshold {0} is {1}\".format(thresh, acc))\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, f1_score))\n    print(\"precision at threshold {0} is {1}\".format(thresh, pre))\n    print(\"recall at threshold {0} is {1}\".format(thresh, recall))\n    \nthresholds.sort(key=lambda x: x[1], reverse=True)\nbest_thresh_2 = str(thresholds[0][0])\nbest_thresh_2_f1 = str(thresholds[0][1])\nprint(\"Best threshold:{0} ， f1:{1}\".format(best_thresh_2, best_thresh_2_f1))\n# print(\"Best threshold:{0} ， f1:{1}\".format(best_thresh_2, best_thresh_2_f1))\n\ny_pred_2 = MODEL2.predict(x_test, batch_size=1024, verbose=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"43475a88d9a21680f16d5a81bd63ce1f5bb55e9d"},"cell_type":"markdown","source":"# Predict"},{"metadata":{"trusted":true,"_uuid":"aa0305b09325f5b423cca73670278540579ee1b0"},"cell_type":"code","source":"# lstm = load_model('./model2.h5', custom_objects={'AttLayer': AttLayer})\n\npred_val_y_2 = MODEL2.predict([X_val], batch_size=1024, verbose=1)\nthresholds = []\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)  # 四舍五入，保留两位小数\n    f1_score = metrics.f1_score(y_val, (pred_val_y_2 > thresh).astype(int))\n    pre = metrics.precision_score(y_val, (pred_val_y_2 > thresh).astype(int))\n    recall = metrics.recall_score(y_val, (pred_val_y_2 > thresh).astype(int))\n    thresholds.append([thresh, f1_score])\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, f1_score))\n    print(\"precision at threshold {0} is {1}\".format(thresh, pre))\n    print(\"recall at threshold {0} is {1}\".format(thresh, recall))\n\nthresholds.sort(key=lambda x: x[1], reverse=True)\nbest_thresh_2 = thresholds[0][0]\nprint(\"Best threshold: \", best_thresh_2)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d9c997b453417c3284558399df57a6ac98eedf83"},"cell_type":"markdown","source":"# MODEL 3: Conv1D"},{"metadata":{"trusted":true,"_uuid":"06ef150ff0f7fcc7c47d023f97f31b4580724068"},"cell_type":"code","source":"def model3():\n    filters = 128\n    \n    inp = Input(shape=(maxlen, ))\n    embed = Embedding(max_features, embed_size * 3, weights=[embedding_matrix], trainable=False)(inp)\n    x = embed\n    \n    x = Conv1D(filters, 1, activation='relu')(x)\n    x = Dropout(0.1)(x)\n    \n    x = Conv1D(filters, 2, activation='relu')(x)\n    x = Dropout(0.1)(x)\n    \n    x = Conv1D(filters, 3, activation='relu')(x)\n    x = Dropout(0.1)(x)\n    \n    x = Conv1D(filters, 5, activation='relu')(x)\n    x = Dropout(0.1)(x)\n    \n    #x = Flatten()(x)\n    x = GlobalAveragePooling1D()(x)\n    \n    x = Dropout(0.3)(x)\n    x = Dense(128, activation='relu')(x)\n    outp = Dense(1, activation=\"sigmoid\")(x)\n    model = Model(inputs=inp, outputs=outp)\n    model.compile(loss='binary_crossentropy',\n                  optimizer='adam',\n                  metrics=['accuracy'])    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"4c3c4ce420cbb1161e88e9d64050a0b9f836042a"},"cell_type":"code","source":"MODEL3 = model3()\nMODEL3.summary()\n\nbatch_size = 2048\nepochs = 1\n\nearly_stopping = EarlyStopping(patience=3, verbose=1, monitor='val_loss', mode='min')\nmodel_checkpoint = ModelCheckpoint('./model3.model', save_best_only=True, verbose=1, monitor='val_loss', mode='min')\nreduce_lr = ReduceLROnPlateau(factor=0.5, patience=3, min_lr=0.0001, verbose=1)\n\nhist = MODEL3.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val), verbose=True)\nMODEL3.save('./model3.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c6ca1e0f7c17c3424f04b07212b0662ae49d7309","scrolled":true},"cell_type":"code","source":"pred_val_y_3 = MODEL3.predict([X_val], batch_size=1024, verbose=1)\nthresholds = []\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    res = metrics.f1_score(y_val, (pred_val_y_3 > thresh).astype(int))\n    thresholds.append([thresh, res])\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, res))\n    \nthresholds.sort(key=lambda x: x[1], reverse=True)\nbest_thresh_3 = thresholds[0][0]\nprint(\"Best threshold: \", best_thresh_3)\n\ny_pred_3 = MODEL3.predict(x_test, batch_size=1024, verbose=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"305c64b3e7c6002ce7a9ac63613f8a1b7aa2901b"},"cell_type":"markdown","source":"# MODEL 4: GRU"},{"metadata":{"trusted":true,"_uuid":"aa0e64fcd54a87d6ce152bf45e5caa902aaff035"},"cell_type":"code","source":"def model4():\n    inp = Input(shape=(maxlen, ))\n    embed = Embedding(max_features, embed_size * 3, weights=[embedding_matrix], trainable=False)(inp)\n    x = embed\n    \n    x = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)\n    x = attention_3d_block(x)\n    x = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)\n    x = AttLayer(64)(x)\n    \n    x = Dropout(0.3)(x)\n    x = Dense(128, activation='relu')(x)\n    outp = Dense(1, activation=\"sigmoid\")(x)\n    outp = Dense(1, activation=\"sigmoid\")(x)\n    model = Model(inputs=inp, outputs=outp)\n    model.compile(loss='binary_crossentropy',\n                  optimizer='adam',\n                  metrics=['accuracy'])    \n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"3ff5e82c52e11d83fe4d70e5d0fb0a8316125fc4"},"cell_type":"code","source":"MODEL4 = model4()\nMODEL4.summary()\n\nbatch_size = 1536\nepochs = 3\n\nearly_stopping = EarlyStopping(patience=3, verbose=1, monitor='val_loss', mode='min')\nmodel_checkpoint = ModelCheckpoint('./model4.model', save_best_only=True, verbose=1, monitor='val_loss', mode='min')\nreduce_lr = ReduceLROnPlateau(factor=0.5, patience=3, min_lr=0.0001, verbose=1)\n\n\nhist = MODEL4.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val), verbose=True)\nMODEL4.save('./model4.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"def9f127ac6118c7d14f094512069e1c3d26cff0","scrolled":true},"cell_type":"code","source":"pred_val_y_4 = MODEL4.predict([X_val], batch_size=1024, verbose=1)\nthresholds = []\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    res = metrics.f1_score(y_val, (pred_val_y_4 > thresh).astype(int))\n    thresholds.append([thresh, res])\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, res))\n    \nthresholds.sort(key=lambda x: x[1], reverse=True)\nbest_thresh_4 = thresholds[0][0]\nprint(\"Best threshold: \", best_thresh_4)\n\ny_pred_4 = MODEL4.predict(x_test, batch_size=1024, verbose=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0044cbecdcad37be125e65d8c4b5b8dcc0c42a89"},"cell_type":"markdown","source":"# Model 5: GRU Add"},{"metadata":{"trusted":true,"_uuid":"b1d920874d748f1519de39b8b661a4ece40c2c1b"},"cell_type":"code","source":"def model5():\n    inp = Input(shape=(maxlen, ))\n    embed = Embedding(max_features, embed_size * 3, weights=[embedding_matrix], trainable=False)(inp)\n    x = embed\n    \n    x0 = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)\n    x1 = attention_3d_block(x0)\n    x2 = Bidirectional(CuDNNGRU(128, return_sequences=True))(x1)\n    x3 = Add()([x0, x2])\n    x4 = Bidirectional(CuDNNGRU(64, return_sequences=True))(x3)\n    x5 = AttLayer(64)(x4)\n    #x5 = Capsule(num_capsule=5, dim_capsule=32, routings=5, share_weights=True)(x4)\n    \n    x = Dropout(0.3)(x5)\n    x = Dense(128, activation='relu')(x)\n    outp = Dense(1, activation=\"sigmoid\")(x)\n    model = Model(inputs=inp, outputs=outp)\n    model.compile(loss='binary_crossentropy',\n                  optimizer='adam',\n                  metrics=['accuracy'])    \n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad99792e499940f8a9d1269a2d6fb81daf775750","scrolled":true},"cell_type":"code","source":"MODEL5 = model5()\nMODEL5.summary()\n\nbatch_size = 1536\nepochs = 3\n\nearly_stopping = EarlyStopping(patience=3, verbose=1, monitor='val_loss', mode='min')\nmodel_checkpoint = ModelCheckpoint('./model5.model', save_best_only=True, verbose=1, monitor='val_loss', mode='min')\nreduce_lr = ReduceLROnPlateau(factor=0.5, patience=3, min_lr=0.0001, verbose=1)\n\nhist = MODEL5.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val), verbose=True)\nMODEL5.save('./model5.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12a188d1690129be8d56c0100ed3030c43382ca5"},"cell_type":"code","source":"pred_val_y_5 = MODEL5.predict([X_val], batch_size=1024, verbose=1)\nthresholds = []\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    res = metrics.f1_score(y_val, (pred_val_y_5 > thresh).astype(int))\n    thresholds.append([thresh, res])\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, res))\n    \nthresholds.sort(key=lambda x: x[1], reverse=True)\nbest_thresh_5 = thresholds[0][0]\nprint(\"Best threshold: \", best_thresh_5)\n\ny_pred_5 = MODEL5.predict(x_test, batch_size=1024, verbose=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"92c1f06de657421e831c59865616cf7774f1607a"},"cell_type":"markdown","source":"# Concat Result & Best Threshold"},{"metadata":{"trusted":true,"_uuid":"9dd8810a5348d5a26c63b2aa1b00642d09515120"},"cell_type":"code","source":"pred_val_y = (3*pred_val_y_2 + 4*pred_val_y_4 + 3*pred_val_y_5)/10\n# pred_val_y = pred_val_y_2\n\nthresholds = []\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    res = metrics.f1_score(y_val, (pred_val_y > thresh).astype(int))\n    thresholds.append([thresh, res])\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, res))\n    \nthresholds.sort(key=lambda x: x[1], reverse=True)\nbest_thresh = thresholds[0][0]\nprint(\"Best threshold: \", best_thresh)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"37901249694b0c14f9ec5a4b6586f52686c92e2f"},"cell_type":"markdown","source":"# Submission File"},{"metadata":{"trusted":true,"_uuid":"404a671dca09cebb0a014796063bd59d82f98714"},"cell_type":"code","source":"y_pred = (3*y_pred_2 + 4*y_pred_4 + 3*y_pred_5)/10\n# y_pred = y_pred_2\ny_te = (y_pred[:,0] > best_thresh).astype(np.int)\n\nsubmit_df = pd.DataFrame({\"qid\": test_df[\"qid\"], \"prediction\": y_te})\nsubmit_df.to_csv(\"sample_submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3c861699e0ae612641cc3eb7d288500c2168cd1"},"cell_type":"code","source":"# from IPython.display import HTML\n# import base64  \n# import pandas as pd  \n\n# def create_download_link( df, title = \"Download CSV file\", filename = \"data.csv\"):  \n#     csv = df.to_csv(index =False)\n#     b64 = base64.b64encode(csv.encode())\n#     payload = b64.decode()\n#     html = '<a download=\"{filename}\" href=\"data:text/csv;base64,{payload}\" target=\"_blank\">{title}</a>'\n#     html = html.format(payload=payload,title=title,filename=filename)\n#     return HTML(html)\n\n# create_download_link(submit_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"48313b5e086fd1dd08d43e51190db353a689cbbc"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}