MAMI_Task_A.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "MAMI Task A Github",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "gpu_info = !nvidia-smi\n",
        "gpu_info = '\\n'.join(gpu_info)\n",
        "if gpu_info.find('failed') >= 0:\n",
        "  print('Not connected to a GPU')\n",
        "else:\n",
        "  print(gpu_info)"
      ],
      "metadata": {
        "id": "b0hWT40aVYyp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "amJ-AHG-7JAF"
      },
      "outputs": [],
      "source": [
        "# 4.15.0\n",
        "!pip install transformers==4.15.0\n",
        "!pip install ekphrasis\n",
        "!pip install emoji"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "id": "EJyQiKhNVWQj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import transformers\n",
        "from transformers import ViTFeatureExtractor, TFViTModel, TFViTForImageClassification, BertTokenizerFast, TFBertModel\n",
        "from PIL import Image\n",
        "\n",
        "import tensorflow as tf\n",
        "from tensorflow.data import Dataset\n",
        "from tensorflow import keras\n",
        "from collections import Counter\n",
        "from tensorflow.keras.callbacks import ModelCheckpoint\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import os\n",
        "\n",
        "transformers.__version__"
      ],
      "metadata": {
        "id": "xS-AUdar7sIx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "os.mkdir('images')\n",
        "import zipfile\n",
        "with zipfile.ZipFile('/content/drive/MyDrive/mami/training.zip', 'r') as zip_ref:\n",
        "    zip_ref.extractall('/content/images')"
      ],
      "metadata": {
        "id": "p2Qvvaej_hED"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_csv('/content/training.csv', sep='delimiter', header=None, skiprows=1)"
      ],
      "metadata": {
        "id": "fAohk4hg8A_D"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.info()"
      ],
      "metadata": {
        "id": "l4CX_hEY-Weh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_data = df[0].apply(lambda string: np.array(string.split('\\t'), dtype='str'))\n",
        "train_data = np.asarray(train_data)\n",
        "train_array = np.stack(train_data)\n",
        "print(np.shape(train_array))"
      ],
      "metadata": {
        "id": "_lCFNsD1-Z4D"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "filenames = train_array[:,0]\n",
        "text = train_array[:,6]\n",
        "Y_misogyny = train_array[:,1]\n",
        "Y_misogyny = Y_misogyny.astype(np.int)\n",
        "Counter(Y_misogyny)"
      ],
      "metadata": {
        "id": "xlC0BbxzZBDx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h6>Text Train Split</h6>"
      ],
      "metadata": {
        "id": "XZVWyVGya3Mt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split"
      ],
      "metadata": {
        "id": "J7UMBcpeZx2T"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "filenames_train, filenames_val, text_train, text_val, Y_train_misogyny, Y_val_misogyny = train_test_split(filenames, text, Y_misogyny, test_size=0.05, random_state=3)"
      ],
      "metadata": {
        "id": "V2lGHNy5ZVQI"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"Train\")\n",
        "print(np.shape(filenames_train), type(filenames_train))\n",
        "print(np.shape(text_train), type(text_train))\n",
        "print(np.shape(Y_train_misogyny), type(Y_train_misogyny))\n",
        "print(\"Val\")\n",
        "print(np.shape(filenames_val))\n",
        "print(np.shape(text_val))\n",
        "print(np.shape(Y_val_misogyny))"
      ],
      "metadata": {
        "id": "GP3jB7OIZVMg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h6>Text pre-processing</h6>"
      ],
      "metadata": {
        "id": "ixx5hs5Za2JT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from ekphrasis.classes.preprocessor import TextPreProcessor\n",
        "from ekphrasis.classes.tokenizer import SocialTokenizer\n",
        "from ekphrasis.dicts.emoticons import emoticons"
      ],
      "metadata": {
        "id": "s0VuTFUiZVJS"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "text_processor = TextPreProcessor(\n",
        "    # terms that will be normalized\n",
        "    normalize=['url', 'email', 'percent', 'money', 'phone', 'user',\n",
        "        'time', 'url', 'date', 'number'],\n",
        "    # terms that will be annotated\n",
        "    annotate={\"hashtag\", \"allcaps\", \"elongated\", \"repeated\",\n",
        "        'emphasis', 'censored'},\n",
        "    fix_html=True,  # fix HTML tokens\n",
        "    \n",
        "    # corpus from which the word statistics are going to be used \n",
        "    # for word segmentation \n",
        "    segmenter=\"twitter\", \n",
        "    \n",
        "    # corpus from which the word statistics are going to be used \n",
        "    # for spell correction\n",
        "    corrector=\"twitter\", \n",
        "    \n",
        "    unpack_hashtags=True,  # perform word segmentation on hashtags\n",
        "    unpack_contractions=True,  # Unpack contractions (can't -> can not)\n",
        "    spell_correct_elong=True,  # spell correction for elongated words\n",
        "    \n",
        "    # select a tokenizer. You can use SocialTokenizer, or pass your own\n",
        "    # the tokenizer, should take as input a string and return a list of tokens\n",
        "    tokenizer=SocialTokenizer(lowercase=True).tokenize,\n",
        "    \n",
        "    # list of dictionaries, for replacing tokens extracted from the text,\n",
        "    # with other expressions. You can pass more than one dictionaries.\n",
        "    dicts=[emoticons]\n",
        ")"
      ],
      "metadata": {
        "id": "EuT4bqHJbB8l"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def print_text(texts,i,j):\n",
        "    for u in range(i,j):\n",
        "        print(texts[u])\n",
        "        print()"
      ],
      "metadata": {
        "id": "Hvr1XOf-bB5E"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print_text(text_train,0,5)\n",
        "print(\"##############################################################################################################\")\n",
        "print_text(text_val,0,5)"
      ],
      "metadata": {
        "id": "2SHN3YeybB2Z"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Functions for chat word conversion\n",
        "f = open(\"slang.txt\", \"r\")\n",
        "chat_words_str = f.read()\n",
        "chat_words_map_dict = {}\n",
        "chat_words_list = []\n",
        "\n",
        "for line in chat_words_str.split(\"\\n\"):\n",
        "    if line != \"\":\n",
        "        cw = line.split(\"=\")[0]\n",
        "        cw_expanded = line.split(\"=\")[1]\n",
        "        chat_words_list.append(cw)\n",
        "        chat_words_map_dict[cw] = cw_expanded\n",
        "chat_words_list = set(chat_words_list)\n",
        "\n",
        "def chat_words_conversion(text):\n",
        "    new_text = []\n",
        "    for w in text.split():\n",
        "        if w.upper() in chat_words_list:\n",
        "            new_text.append(chat_words_map_dict[w.upper()])\n",
        "        else:\n",
        "            new_text.append(w)\n",
        "    return \" \".join(new_text)"
      ],
      "metadata": {
        "id": "gQ4cYQKubBzZ"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "text_train = pd.Series(text_train)\n",
        "text_val = pd.Series(text_val)"
      ],
      "metadata": {
        "id": "p-RGux_eb4rC"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Chat word conversion\n",
        "# Training set\n",
        "text_train = text_train.apply(lambda text: chat_words_conversion(text))\n",
        "print_text(text_train,0,5)\n",
        "\n",
        "print(\"********************************************************************************\")\n",
        "\n",
        "# Validation set\n",
        "text_val = text_val.apply(lambda text: chat_words_conversion(text))\n",
        "print_text(text_val,0,5)\n",
        "\n",
        "# Test set\n",
        "# text_test = text_test.apply(lambda text: chat_words_conversion(text))\n",
        "# print_text(text_test,0,10)"
      ],
      "metadata": {
        "id": "B45gKA2ZbBw3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def ekphrasis_pipe(sentence):\n",
        "    cleaned_sentence = \" \".join(text_processor.pre_process_doc(sentence))\n",
        "    return cleaned_sentence"
      ],
      "metadata": {
        "id": "jnyAeJgLbBtp"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Training set\n",
        "text_train = text_train.apply(lambda text: ekphrasis_pipe(text))\n",
        "print(\"Training set completed.......\")\n",
        "#Validation set\n",
        "text_val = text_val.apply(lambda text: ekphrasis_pipe(text))\n",
        "print(\"Validation set completed.......\")\n",
        "#Test set\n",
        "# text_test = text_test.apply(lambda text: ekphrasis_pipe(text))\n",
        "# print(\"Test set completed.......\")"
      ],
      "metadata": {
        "id": "O3VSPXylbBrA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "u = lambda text: len(text.split(\" \"))\n",
        "sentence_lengths = []\n",
        "for x in text_train:\n",
        "    sentence_lengths.append(u(x))\n",
        "print(sorted(sentence_lengths)[-500:])\n",
        "print(len(sentence_lengths))"
      ],
      "metadata": {
        "id": "ObiKEw2ucqSr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "</h6>Text processing complete</h6>"
      ],
      "metadata": {
        "id": "VCqoO60Rcr1p"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import RobertaTokenizerFast, TFBertModel, TFRobertaModel, MPNetTokenizerFast, TFMPNetModel, ElectraTokenizerFast, TFElectraModel, XLNetTokenizerFast, TFXLNetModel, AlbertTokenizerFast, TFAlbertModel, DebertaTokenizer, TFDebertaModel"
      ],
      "metadata": {
        "id": "qTlMhqMkoNaD"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Use the tokenizer as per the model being used\n",
        "tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')"
      ],
      "metadata": {
        "id": "bFp70daooPCI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_encodings = tokenizer(list(text_train), max_length=80, truncation=True, padding=\"max_length\", return_tensors='tf')\n",
        "print(np.shape(train_encodings[\"input_ids\"]))\n",
        "\n",
        "val_encodings = tokenizer(list(text_val), max_length=80, truncation=True, padding=\"max_length\", return_tensors='tf')\n",
        "print(np.shape(val_encodings[\"input_ids\"]))"
      ],
      "metadata": {
        "id": "4Pxcc2BxAs2Q"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h6>Image features pipeline</h6>"
      ],
      "metadata": {
        "id": "D1SM6eFBXFX3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def image_processing(filename):\n",
        "  # Read Image (Change path as per the directory structure)\n",
        "  image_string = tf.io.read_file('/content/images/TRAINING/' + filename)\n",
        "  img = tf.io.decode_jpeg(image_string, channels=3)\n",
        "  # Resize image\n",
        "  img = tf.image.resize(img, [224,224], method='bilinear')\n",
        "  # Normalise image\n",
        "  img = tf.cast(img, tf.float32)\n",
        "  img = tf.math.divide(img, 255.0)\n",
        "  img = tf.math.subtract(img, 0.5)\n",
        "  img = tf.math.divide(img, 0.5)\n",
        "  # Move channel axis\n",
        "  img = tf.experimental.numpy.moveaxis(img, -1, 0)\n",
        "\n",
        "  return img"
      ],
      "metadata": {
        "id": "7bSxXlZc-xlE"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "batch_size = 16 # for tpu 128\n",
        "def configure_for_performance(ds):\n",
        "  # ds = ds.cache()\n",
        "  ds = ds.shuffle(buffer_size=1000)\n",
        "  ds = ds.batch(batch_size)\n",
        "  ds = ds.prefetch(buffer_size=tf.data.AUTOTUNE)\n",
        "  return ds"
      ],
      "metadata": {
        "id": "Vj0CxeY7JEIN"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_dataset(filenames, encodings, Y_misogyny):\n",
        "  image_names = Dataset.from_tensor_slices(filenames) # ==> 3x2 tensor\n",
        "  image_features = image_names.map(image_processing)\n",
        "\n",
        "  text_features = Dataset.from_tensor_slices(encodings[\"input_ids\"])\n",
        "  text_masks = Dataset.from_tensor_slices(encodings[\"attention_mask\"])\n",
        "\n",
        "  labels_misogyny = Dataset.from_tensor_slices(Y_misogyny)\n",
        "\n",
        "  features = Dataset.zip((image_features, text_features, text_masks))\n",
        "\n",
        "  dataset = Dataset.zip((features, labels_misogyny))\n",
        "\n",
        "  return dataset"
      ],
      "metadata": {
        "id": "nxLce-8Fdl-z"
      },
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dataset_train = get_dataset(filenames_train, train_encodings, Y_train_misogyny)\n",
        "dataset_train = configure_for_performance(dataset_train)"
      ],
      "metadata": {
        "id": "oK2vMAdPezHt"
      },
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "images_features_val = np.zeros((500,3,224,224))\n",
        "for i in range(0, 500):\n",
        "  images_features_val[i] = image_processing(filenames_val[i])"
      ],
      "metadata": {
        "id": "vXeqVId4xXJH"
      },
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h6>Simple Concatenation based model</h6>"
      ],
      "metadata": {
        "id": "ht4NEGQ-P4xg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# # Concat Model\n",
        "# def mami_concat(input_shape):\n",
        "#     img_input = keras.Input(shape=(3, 224, 224), dtype='float32')\n",
        "#     txt_input = keras.Input(shape=input_shape, dtype='int32')\n",
        "#     input_masks = keras.Input(shape=input_shape, dtype='int32')\n",
        "\n",
        "#     # Text\n",
        "#     model_txt = TFMPNetModel.from_pretrained(\"microsoft/mpnet-base\")\n",
        "#     layer_txt = model_txt.layers[0]\n",
        "#     embeddings_txt = layer_txt([txt_input, input_masks])[0][:,0,:]\n",
        "#     embeddings_txt = keras.layers.Dense(256,activation='relu')(embeddings_txt)\n",
        "\n",
        "#     # Images\n",
        "#     model_images = TFViTModel.from_pretrained('google/vit-base-patch16-224-in21k')\n",
        "#     embeddings_img = model_images(img_input)[0][:,0,:]\n",
        "#     embeddings_img = keras.layers.Dense(256,activation='relu')(embeddings_img)\n",
        "    \n",
        "#     X =  keras.layers.Concatenate()([embeddings_txt, embeddings_img])\n",
        "\n",
        "#     X = keras.layers.Dense(128,activation='relu')(X)\n",
        "\n",
        "#     X = keras.layers.BatchNormalization()(X)\n",
        "\n",
        "#     X = keras.layers.Dense(64,activation='relu')(X)\n",
        "    \n",
        "#     X = keras.layers.Dense(1,activation='sigmoid')(X)\n",
        "    \n",
        "#     model = keras.Model(inputs=[img_input, txt_input, input_masks], outputs=[X])    \n",
        "#     return model"
      ],
      "metadata": {
        "id": "I9xzpXMmAvXO"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h6>Attention based model</h6>"
      ],
      "metadata": {
        "id": "3yItX7HaW0ST"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Attention Model base models\n",
        "def mami_attention(input_shape):\n",
        "    img_input = keras.Input(shape=(3, 224, 224), dtype='float32')\n",
        "    txt_input = keras.Input(shape=input_shape, dtype='int32')\n",
        "    input_masks = keras.Input(shape=input_shape, dtype='int32')\n",
        "\n",
        "    # Text\n",
        "    model_txt = TFBertModel.from_pretrained(\"bert-base-uncased\") # Change model definition as per requirement\n",
        "    layer_txt = model_txt.layers[0]\n",
        "    text_seq = layer_txt([txt_input, input_masks])[0]\n",
        "\n",
        "    # Images\n",
        "    model_images = TFViTModel.from_pretrained('google/vit-base-patch16-224-in21k')\n",
        "    layer_img = model_images.layers[0]\n",
        "    img_seq = layer_img(img_input)[0]\n",
        "    \n",
        "    joint_features = tf.keras.layers.Attention()([img_seq, text_seq])\n",
        "\n",
        "    joint_features = tf.keras.layers.Conv1D(32, 30, 15)(joint_features)\n",
        "    \n",
        "    joint_features = tf.keras.layers.Flatten()(joint_features)\n",
        "\n",
        "    X = keras.layers.BatchNormalization()(joint_features)\n",
        "\n",
        "    X = keras.layers.Dense(64,activation='relu')(X)\n",
        "    \n",
        "    X = keras.layers.Dense(1,activation='sigmoid')(X)\n",
        "    \n",
        "    model = keras.Model(inputs=[img_input, txt_input, input_masks], outputs=[X])    \n",
        "    return model"
      ],
      "metadata": {
        "id": "mCV99RjFLyvy"
      },
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import roc_curve\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "class EvaluationMetric(keras.callbacks.Callback):   \n",
        "    \n",
        "    def __init__(self, val_img_features, val_encodings, val_masks, Y_val):\n",
        "        super(EvaluationMetric, self).__init__()\n",
        "        self.val_encodings = val_encodings\n",
        "        self.val_masks = val_masks\n",
        "        self.Y_val = Y_val\n",
        "        self.val_img_features = val_img_features\n",
        "    \n",
        "    def on_epoch_begin(self, epoch, logs={}):\n",
        "        print(\"\\nTraining...\")\n",
        "\n",
        "    def on_epoch_end(self, epoch, logs={}):\n",
        "        print(\"\\nEvaluating...\")\n",
        "        val_prediction = self.model.predict([self.val_img_features, self.val_encodings, self.val_masks])\n",
        "        fpr, tpr, threshold = roc_curve(self.Y_val, val_prediction)\n",
        "        minima = np.argmin(np.abs(fpr + tpr -1))\n",
        "        threshold_final = threshold[minima] \n",
        "        print(\"Threshold is:\", threshold_final)\n",
        "\n",
        "        pred_ = []\n",
        "        for i in range(0,len(self.Y_val)):\n",
        "            num = val_prediction[i]\n",
        "            if(num > 0.5):\n",
        "              num = 1\n",
        "            else:\n",
        "              num = 0\n",
        "            pred_.append(num)\n",
        "        \n",
        "        from sklearn.metrics import classification_report\n",
        "        print(\"With Threshold\")\n",
        "        print(classification_report(self.Y_val, pred, digits=3))\n",
        "        print(\"Without Threshold\")\n",
        "        print(classification_report(self.Y_val, pred_, digits=3))\n",
        "        \n",
        "evaluation_metric = EvaluationMetric(images_features_val, val_encodings[\"input_ids\"], val_encodings[\"attention_mask\"], Y_val_misogyny)"
      ],
      "metadata": {
        "id": "Ieg_DTqrX8KH"
      },
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = mami_attention((80,))\n",
        "optimizer = keras.optimizers.Adam(learning_rate=4e-5)\n",
        "loss_fun = tf.keras.losses.BinaryCrossentropy(from_logits=False)\n",
        "metric = [tf.metrics.BinaryAccuracy(), tf.metrics.Precision(), tf.metrics.Recall()]\n",
        "model.compile(optimizer=optimizer, loss=loss_fun, metrics=metric)"
      ],
      "metadata": {
        "id": "8IEuSqSqK_oi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.summary()"
      ],
      "metadata": {
        "id": "PA1Cu7CJNm41"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tf.keras.utils.plot_model(\n",
        "    model)"
      ],
      "metadata": {
        "id": "G11KmoRuOARu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "checkpoint = ModelCheckpoint(filepath='/content/bert.{epoch:03d}.h5',\n",
        "                                 verbose = 0,\n",
        "                                 save_weights_only=True,\n",
        "                                 epoch=1)"
      ],
      "metadata": {
        "id": "uRslZCZUkCCn"
      },
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# ATTN BERT\n",
        "history = model.fit(\n",
        "    dataset_train,\n",
        "    callbacks = [evaluation_metric, checkpoint],\n",
        "    epochs=1\n",
        ")"
      ],
      "metadata": {
        "id": "hjwB1x8O2bGR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# model.load_weights(\"/content/att-vit-bert.003.h5\")\n",
        "# # model.save(\"/content/att-vit-bert-model.002.h5\")"
      ],
      "metadata": {
        "id": "xHB1W0OiODW1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# import gc\n",
        "# gc.collect()"
      ],
      "metadata": {
        "id": "HVYb4mJFlqkO"
      },
      "execution_count": 44,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h6>Test Set Predictions</h6>"
      ],
      "metadata": {
        "id": "sg8thWcvkZvp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model.load_weights('/content/bert.001.h5')"
      ],
      "metadata": {
        "id": "pgOsauAjjEYD"
      },
      "execution_count": 67,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "os.mkdir('images_test')\n",
        "import zipfile\n",
        "with zipfile.ZipFile('/content/drive/MyDrive/mami/test.zip', 'r') as zip_ref:\n",
        "    zip_ref.extractall('/content/images_test')"
      ],
      "metadata": {
        "id": "UehYWrpwcnyP"
      },
      "execution_count": 45,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_test = pd.read_csv('/content/test.csv', sep='delimiter', header=None, skiprows=1)"
      ],
      "metadata": {
        "id": "2DnGqzAtky5c"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_test.info()"
      ],
      "metadata": {
        "id": "-hnVprr_kzxC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test_data = df_test[0].apply(lambda string: np.array(string.split('\\t'), dtype='str'))\n",
        "test_data = np.asarray(test_data)\n",
        "test_array = np.stack(test_data)\n",
        "print(np.shape(test_array))"
      ],
      "metadata": {
        "id": "EwV-MhD0lAzy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "filenames_test = test_array[:,0]\n",
        "text_test = test_array[:,1]"
      ],
      "metadata": {
        "id": "rsfQtKrWZYdd"
      },
      "execution_count": 49,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "text_test = pd.Series(text_test)\n",
        "text_test = text_test.apply(lambda text: chat_words_conversion(text))\n",
        "text_test = text_test.apply(lambda text: ekphrasis_pipe(text))"
      ],
      "metadata": {
        "id": "PRUxIgUTZZ_x"
      },
      "execution_count": 50,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def image_processing_test(filename):\n",
        "  # Read Image\n",
        "  image_string = tf.io.read_file('/content/images_test/test/' + filename)\n",
        "  img = tf.io.decode_jpeg(image_string, channels=3)\n",
        "  # Resize image\n",
        "  img = tf.image.resize(img, [224,224], method='bilinear')\n",
        "  # Normalise image\n",
        "  img = tf.cast(img, tf.float32)\n",
        "  img = tf.math.divide(img, 255.0)\n",
        "  img = tf.math.subtract(img, 0.5)\n",
        "  img = tf.math.divide(img, 0.5)\n",
        "  # Move channel axis\n",
        "  img = tf.experimental.numpy.moveaxis(img, -1, 0)\n",
        "\n",
        "  return img"
      ],
      "metadata": {
        "id": "B9J69fJKmDqN"
      },
      "execution_count": 51,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "images_features_test = np.zeros((1000,3,224,224))\n",
        "for i in range(0, 1000):\n",
        "  images_features_test[i] = image_processing_test(filenames_test[i])"
      ],
      "metadata": {
        "id": "Dm-9YN_EZeWk"
      },
      "execution_count": 52,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test_encodings = tokenizer(list(text_test), max_length=80, truncation=True, padding=\"max_length\", return_tensors='tf')\n",
        "print(np.shape(test_encodings[\"input_ids\"]))"
      ],
      "metadata": {
        "id": "wxYfJ9EBZbpw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "answer_test = model.predict([images_features_test, test_encodings[\"input_ids\"], test_encodings[\"attention_mask\"]])"
      ],
      "metadata": {
        "id": "motKWOSIZd45"
      },
      "execution_count": 54,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_answer_list(answer):\n",
        "  print(np.shape(answer))\n",
        "  final = (np.where(answer_test > 0.5, 1, 0)).astype(np.int)\n",
        "  print(Counter(final[:,0]))\n",
        "  answer_list = final[:,0]\n",
        "  return answer_list"
      ],
      "metadata": {
        "id": "GDnAGxDAZhrw"
      },
      "execution_count": 68,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "final_answer = get_answer_list(answer_test)"
      ],
      "metadata": {
        "id": "yORY7S2QZjI-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Counter(final_answer)"
      ],
      "metadata": {
        "id": "ElafF9h5Wa6c"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "answer_list = final_answer\n",
        "answer_names = list(filenames_test)"
      ],
      "metadata": {
        "id": "x1gXn70bZmXc"
      },
      "execution_count": 61,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "with open('answer.txt', 'w') as outf:\n",
        "  for i in range(0, len(answer_list)-1):\n",
        "    outf.write(answer_names[i] + '\\t' + str(answer_list[i]) + '\\n')\n",
        "  outf.write(answer_names[999] + '\\t' + str(answer_list[999]))"
      ],
      "metadata": {
        "id": "5quao_41Znv4"
      },
      "execution_count": 62,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import zipfile\n",
        "zipfile.ZipFile('bert.zip', mode='w').write(\"answer.txt\")"
      ],
      "metadata": {
        "id": "q6gqpi3CZpRp"
      },
      "execution_count": 64,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "_3_DqQviUsJY"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}