ds4_sentiment_analysis_for_rating_calculation.py

# -*- coding: utf-8 -*-
"""Fourth_Sentiment_Analysis.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1JDwpb11IrN-uB1nBfsbVDCprd3yDWVV7

Importing Libraries
--
"""

import numpy as np
import pandas as pd

"""Dataset
--
"""

from google.colab import drive
drive.mount('/content/drive')

dataset = pd.read_csv('/content/drive/MyDrive/SQA/Project Works/Datasets/App_Reviews.csv', delimiter = ',', nrows=224689)

dataset.shape

dataset.tail()

from sklearn.utils import shuffle
df = shuffle(dataset)

df.head()

"""BERT
--
"""

!pip install transformers

!pip install torch==1.8.1+cu111 torchvision==0.9.1+cu111 torchaudio===0.8.1 -f https://download.pytorch.org/whl/torch_stable.html

from transformers import AutoTokenizer, AutoModelForSequenceClassification
import torch

tokenizer = AutoTokenizer.from_pretrained('nlptown/bert-base-multilingual-uncased-sentiment')

model = AutoModelForSequenceClassification.from_pretrained('nlptown/bert-base-multilingual-uncased-sentiment')

def sentiment_score(review):
    tokens = tokenizer.encode(review, return_tensors='pt')
    result = model(tokens)
    return int(torch.argmax(result.logits))+1

X_data = []
Y_data = []
Y_score = []
for i in range(2,len(df)):
  try:
    Y_out = sentiment_score(df['Reviews'].iloc[i])
    Y_true = df['Rating'].iloc[i];
    if Y_out>=3 and Y_true>=3:
      X_data.append(df['Reviews'].iloc[i])
      Y_data.append(Y_true)
      Y_score.append(Y_out)
    elif Y_out<3 and Y_true<3:
      X_data.append(df['Reviews'].iloc[i])
      Y_data.append(Y_true)
      Y_score.append(Y_out)
  except Exception as e:
      print(e, i)
      pass

X_data = np.array(X_data)
Y_data = np.array(Y_data)
Y_score = np.array(Y_score)

from sklearn.metrics import confusion_matrix, accuracy_score
cm = confusion_matrix(Y_data, Y_score)
print(cm)

accuracy_score(Y_data, Y_score)

"""Pickle Data
--
"""

import pickle

pickle_out = open("/content/drive/MyDrive/SQA/Project Works/Datasets/Pickle Data/Model_4/X.pickle", "wb")
pickle.dump(X_data, pickle_out, protocol=4) # protocol=4 is used for new version of pickle which can serialize more than 4GB data
pickle_out.close()

pickle_out = open("/content/drive/MyDrive/SQA/Project Works/Datasets/Pickle Data/Model_4/Y.pickle", "wb")
pickle.dump(Y_data, pickle_out, protocol=4) # protocol=4 is used for new version of pickle which can serialize more than 4GB data
pickle_out.close()

pickle_out = open("/content/drive/MyDrive/SQA/Project Works/Datasets/Pickle Data/Model_4/Y_score.pickle", "wb")
pickle.dump(Y_score, pickle_out, protocol=4) # protocol=4 is used for new version of pickle which can serialize more than 4GB data
pickle_out.close()

"""Data Preprocessing
--
"""

import pickle

pickle_in = open("/content/drive/MyDrive/SQA/Project Works/Datasets/Pickle Data/Model_4/X.pickle", "rb")
X = pickle.load(pickle_in)

pickle_in = open("/content/drive/MyDrive/SQA/Project Works/Datasets/Pickle Data/Model_4/Y.pickle", "rb")
Y = pickle.load(pickle_in)

len(X)

len(Y)

import re
import nltk

nltk.download('stopwords')

from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer
ps = PorterStemmer()

all_stopwords = stopwords.words('english')
all_stopwords.remove('not')

corpus=[]

for i in range(0, len(X)):
  review = re.sub('[^a-zA-Z]', ' ', str(X[i]))
  review = review.lower()
  review = review.split()
  review = [ps.stem(word) for word in review if not word in set(all_stopwords)]
  review = ' '.join(review)
  corpus.append(review)

corpus

len(corpus)

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.preprocessing.text import Tokenizer
tokens_model = Tokenizer(num_words = 64)

tokens_model.fit_on_texts(corpus)

seq = tokens_model.texts_to_sequences(corpus)

word_index = tokens_model.word_index
print(tokens_model.word_index)

len(word_index)

print(seq)

max_length = 0
for review_num in range(len(seq)):
  num_words = len(seq[review_num])
  if num_words > max_length:
    max_length = num_words
print(max_length)

from tensorflow.keras.preprocessing.sequence import pad_sequences
import numpy as np
X_vec = pad_sequences(seq, maxlen=max_length)
Y = np.asarray(Y)

X_vec.shape

Y.shape

Y = Y - 1

Y

"""Training and Test Splitting
--
"""

from sklearn.model_selection import train_test_split

X_train, X_test, Y_train, Y_test = train_test_split(X_vec, Y, test_size=0.2, random_state=1)
X_test, X_val, Y_test, Y_val = train_test_split(X_test, Y_test, test_size=0.5, random_state=1)

print('Train:', len(X_train), '\nTest:', len(X_test), '\nValidation:', len(X_val))

pickle_out = open("/content/drive/MyDrive/SQA/Project Works/Datasets/Pickle Data/Model_4/X_train.pickle", "wb")
pickle.dump(X_train, pickle_out, protocol=4) # protocol=4 is used for new version of pickle which can serialize more than 4GB data
pickle_out.close()

pickle_out = open("/content/drive/MyDrive/SQA/Project Works/Datasets/Pickle Data/Model_4/Y_train.pickle", "wb")
pickle.dump(Y_train, pickle_out, protocol=4) # protocol=4 is used for new version of pickle which can serialize more than 4GB data
pickle_out.close()

pickle_out = open("/content/drive/MyDrive/SQA/Project Works/Datasets/Pickle Data/Model_4/X_test.pickle", "wb")
pickle.dump(X_test, pickle_out, protocol=4) # protocol=4 is used for new version of pickle which can serialize more than 4GB data
pickle_out.close()

pickle_out = open("/content/drive/MyDrive/SQA/Project Works/Datasets/Pickle Data/Model_4/Y_test.pickle", "wb")
pickle.dump(Y_test, pickle_out, protocol=4) # protocol=4 is used for new version of pickle which can serialize more than 4GB data
pickle_out.close()

pickle_out = open("/content/drive/MyDrive/SQA/Project Works/Datasets/Pickle Data/Model_4/X_val.pickle", "wb")
pickle.dump(X_val, pickle_out, protocol=4) # protocol=4 is used for new version of pickle which can serialize more than 4GB data
pickle_out.close()

pickle_out = open("/content/drive/MyDrive/SQA/Project Works/Datasets/Pickle Data/Model_4/Y_val.pickle", "wb")
pickle.dump(Y_val, pickle_out, protocol=4) # protocol=4 is used for new version of pickle which can serialize more than 4GB data
pickle_out.close()

"""Load Data
--
"""

import pickle

pickle_in = open("/content/drive/MyDrive/SQA/Project Works/Datasets/Pickle Data/Model_4/X_train.pickle", "rb")
X_train = pickle.load(pickle_in)

pickle_in = open("/content/drive/MyDrive/SQA/Project Works/Datasets/Pickle Data/Model_4/Y_train.pickle", "rb")
Y_train = pickle.load(pickle_in)

pickle_in = open("/content/drive/MyDrive/SQA/Project Works/Datasets/Pickle Data/Model_4/X_test.pickle", "rb")
X_test = pickle.load(pickle_in)

pickle_in = open("/content/drive/MyDrive/SQA/Project Works/Datasets/Pickle Data/Model_4/Y_test.pickle", "rb")
Y_test = pickle.load(pickle_in)

pickle_in = open("/content/drive/MyDrive/SQA/Project Works/Datasets/Pickle Data/Model_4/X_val.pickle", "rb")
X_val = pickle.load(pickle_in)

pickle_in = open("/content/drive/MyDrive/SQA/Project Works/Datasets/Pickle Data/Model_4/Y_val.pickle", "rb")
Y_val = pickle.load(pickle_in)

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, models
from keras.callbacks import ModelCheckpoint

import sklearn
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix

total_words = len(word_index)+1
embedding_dim = 4

model = models.Sequential()
model.add(layers.Embedding(total_words, embedding_dim, input_length=max_length, name='embedding_layer'))
model.add(layers.SimpleRNN(256, activation='relu', return_sequences=True, name='rnn1'))
model.add(layers.LSTM(512, dropout=0.2, recurrent_dropout=0.2, return_sequences=True, name='lstm'))
model.add(layers.SimpleRNN(1024, activation='relu', name='rnn2'))
model.add(layers.Dense(5, activation='softmax'))

model.summary()

from keras.callbacks import ModelCheckpoint

model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
epochs = 15

filepath = '/content/drive/MyDrive/SQA/Project Works/Datasets/Pickle Data/Model_4/model-{epoch:02d}-performance-{val_accuracy:.2f}-{val_loss:.2f}.hdf5'

checkpoint_acc = ModelCheckpoint(filepath=filepath, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')
checkpoint_loss = ModelCheckpoint(filepath=filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='min')

callbacks = [checkpoint_acc, checkpoint_loss]

history = model.fit(X_train, Y_train, validation_data=(X_val, Y_val), epochs=epochs, callbacks=callbacks)

"""Re_Train
--
"""

r_model = tf.keras.models.load_model('/content/drive/MyDrive/SQA/Project Works/Datasets/Pickle Data/Model_4/model-02-performance-0.53-1.17.hdf5')

from keras.callbacks import ModelCheckpoint

r_model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
epochs = 15

filepath = '/content/drive/MyDrive/SQA/Project Works/Datasets/Pickle Data/Model_4/model-v2-{epoch:02d}-performance-{val_accuracy:.2f}-{val_loss:.2f}.hdf5'

checkpoint_acc = ModelCheckpoint(filepath=filepath, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')
checkpoint_loss = ModelCheckpoint(filepath=filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='min')

callbacks = [checkpoint_acc, checkpoint_loss]

history = r_model.fit(X_train, Y_train, validation_data=(X_val, Y_val), epochs=epochs, callbacks=callbacks)

"""Learning Visualization
--
"""

import matplotlib.pyplot as plt

train_acc = [0.5159, 0.5299, 0.5334, 0.5326, 0.5371, 0.5369, 0.5397, 0.5408, 0.5424, 0.5435]
val_acc = [0.5, 0.5304, 0.5323, 0.5319, 0.5326, 0.5365, 0.5358, 0.5337, 0.5360, 0.5353]

train_loss = [1.2184, 1.1770, 1.1684, 1.1772, 1.1591, 1.1568, 1.1522, 1.1482, 1.1441, 1.1399]
val_loss = [1.1912, 1.1732, 1.1699, 1.1687, 1.1676, 1.1611, 1.1607, 1.1648, 1.1609, 1.1628]

epochs_range = range(10)

plt.figure(figsize=(18, 6))
plt.subplot(1, 2, 1)
plt.plot(epochs_range, train_acc, label='Training Accuracy')
plt.plot(epochs_range, val_acc, label='Validation Accuracy')
plt.legend(loc='lower right')
plt.title('Training and Validation Accuracy')

plt.subplot(1, 2, 2)
plt.plot(epochs_range, train_loss, label='Training Loss')
plt.plot(epochs_range, val_loss, label='Validation Loss')
plt.legend(loc='upper right')
plt.title('Training and Validation Loss')
plt.show()

"""Performance Analysis
--
"""

model = tf.keras.models.load_model('/content/drive/MyDrive/SQA/Project Works/Datasets/Pickle Data/Model_4/model-v2-04-performance-0.54-1.16.hdf5')

records = model.evaluate(X_test, Y_test)

predictions = model.predict(X_test) # predict output for all test data

scores = tf.nn.softmax(predictions)
Y_pred = []
for score in scores:
  Y_pred.append(np.argmax(score))
Y_pred = np.array(Y_pred) # predicted labels

# Defining function for confusion matrix plot
def plot_confusion_matrix(Y_test, Y_pred, Classes, normalize=False, title=None, cmap=plt.cm.Blues):

    # Compute the confusion matrix
    conf_mat = confusion_matrix(Y_test, Y_pred)
    if normalize:
        conf_mat = conf_mat.astype('float32') / conf_mat.sum(axis=1)[:, np.newaxis]
        print(title)
    else:
        print(title)

    fig, ax = plt.subplots(figsize=(6,6))
    im = ax.imshow(conf_mat, interpolation='nearest', cmap=cmap)
    ax.figure.colorbar(im, ax=ax)

    # We want to show all ticks...
    ax.set(xticks=np.arange(conf_mat.shape[1]), yticks=np.arange(conf_mat.shape[0]),
           xticklabels=Classes, yticklabels=Classes,
           title=title, ylabel='True label', xlabel='Predicted label')

    # Rotate the tick labels and set their alignment.
    plt.setp(ax.get_xticklabels(), rotation=45, ha="right", rotation_mode="anchor")
   
    # Loop over data dimensions and create text annotations.
    fmt = '.2f' if normalize else 'd'
    thresh = conf_mat.max() / 2.
    for i in range(conf_mat.shape[0]):
        for j in range(conf_mat.shape[1]):
            ax.text(j, i, format(conf_mat[i, j], fmt),
                    ha="center", va="center",
                    color="white" if conf_mat[i, j] > thresh else "black")
    fig.tight_layout()

    return ax

np.set_printoptions(precision=2)

from sklearn.metrics import confusion_matrix

#Plotting the confusion matrix
confusion_mtx = confusion_matrix(Y_test, Y_pred)

# Plotting non-normalized confusion matrix
plot_confusion_matrix(Y_test, Y_pred, Classes = ['*    ', '**   ', '***  ', '**** ', '*****'], title='Confusion Matrix')

#Plotting normalized confusion matrix
plot_confusion_matrix(Y_test, Y_pred, Classes = ['*    ', '**   ', '***  ', '**** ', '*****'], normalize = True, title = 'Confusion Matrix - Normalized')

from sklearn import metrics

def get_metrics(true_labels, predicted_labels):
  print('Accuracy:', np.round(metrics.accuracy_score(true_labels, predicted_labels), 4))
  print('Precision:', np.round(metrics.precision_score(true_labels, predicted_labels, average='weighted'),4))
  print('Recall:', np.round(metrics.recall_score(true_labels, predicted_labels, average='weighted'), 4))
  print('F1 Score:', np.round(metrics.f1_score(true_labels, predicted_labels, average='weighted'), 4))

get_metrics(Y_test, Y_pred)
  
def display_classification_report(true_labels, predicted_labels, Classes):
  report = metrics.classification_report(y_true=true_labels, y_pred=predicted_labels, target_names=Classes)
  print("\nReport:\n"+report)

display_classification_report(Y_test, Y_pred, Classes=['*    ', '**   ', '***  ', '**** ', '*****'])

import matplotlib.pyplot as plt
from sklearn.preprocessing import LabelBinarizer
from sklearn.metrics import roc_curve, auc, roc_auc_score

target= ['*    ', '**   ', '***  ', '**** ', '*****']

# set plot figure size
fig, c_ax = plt.subplots(1,1, figsize = (12, 8))

# function for scoring roc auc score for multi-class
def multiclass_roc_auc_score(y_test, y_pred, average="macro"):
  lb = LabelBinarizer()
  lb.fit(y_test)
  y_test = lb.transform(y_test)
  y_pred = lb.transform(y_pred)
  
  for (idx, c_label) in enumerate(target):
    fpr, tpr, thresholds = roc_curve(y_test[:,idx].astype(int), y_pred[:,idx])
    c_ax.plot(fpr, tpr, label = '%s (AUC:%0.2f)' % (c_label, auc(fpr, tpr)))
  
  c_ax.plot(fpr, fpr, 'b-', label = 'Random Guessing')
  return roc_auc_score(y_test, y_pred, average=average)

print('ROC AUC score:', multiclass_roc_auc_score(Y_test, Y_pred))

c_ax.legend()
c_ax.set_xlabel('False Positive Rate')
c_ax.set_ylabel('True Positive Rate')
plt.show()