DATA_441_Final_Project_Neural_Net.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"authorship_tag":"ABX9TyNnYevFUVITGWNI4sgVqbe9"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"accelerator":"GPU","gpuClass":"standard"},"cells":[{"cell_type":"code","source":["from keras.activations import leaky_relu\n","from sklearn.model_selection import train_test_split as tts\n","from tensorflow.keras.callbacks import EarlyStopping\n","from tensorflow.keras.models import Model, Sequential\n","from tensorflow.keras.utils import to_categorical\n","from tensorflow.keras.optimizers import Adam, SGD, RMSprop\n","from tensorflow.keras.regularizers import l1_l2, l2, l1\n","from tensorflow.keras.optimizers.schedules import ExponentialDecay\n","# for images, use Conv2D, for regression, use Conv1Dr\n","from tensorflow.keras.layers import Input, BatchNormalization, Dense, Activation, Dropout, LeakyReLU"],"metadata":{"id":"6MVEqcjVSWSN"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["\n","\n","import numpy as np\n","import pandas as pd\n","import matplotlib.pyplot as plt\n","import os\n","from sklearn.metrics import accuracy_score\n","from sklearn.model_selection import cross_val_score, KFold"],"metadata":{"id":"sTHYjPvS1dVA"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["from google.colab import drive\n","drive.mount('/content/drive')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Pq6HH-_51_u6","executionInfo":{"status":"ok","timestamp":1683190579121,"user_tz":240,"elapsed":18012,"user":{"displayName":"Matthew Cheng","userId":"00814230256052145695"}},"outputId":"078eec96-29fe-45d1-9f17-8e3c783cd403"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/drive\n"]}]},{"cell_type":"code","source":["data = pd.read_csv('drive/MyDrive/Colab Notebooks/data/telecom_churn.csv')"],"metadata":{"id":"zilsPACO1wdS"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["X = data.loc[:, 'AccountWeeks':].values\n","#y = data['Churn'].values\n","y = to_categorical(data['Churn'].values,2)"],"metadata":{"id":"Otuh6PDaZowV"},"execution_count":null,"outputs":[]},{"cell_type":"code","execution_count":null,"metadata":{"id":"AIlwhcHaemmb"},"outputs":[],"source":["lr_schedule = ExponentialDecay(\n","    initial_learning_rate=1e-3,\n","    decay_steps=10000,\n","    decay_rate=1e-3/200)\n","optimizer = Adam(learning_rate=lr_schedule)\n","\n","model = Sequential()\n","# model.add(Dense(16,activation=LeakyReLU(alpha=0.08),use_bias=True,kernel_regularizer=l1_l2(l1=0.1,l2=0.2)))\n","# model.add(Dense(16,activation='gelu',use_bias=True,kernel_regularizer=l1_l2(l1=0.01,l2=0.02)))\n","model.add(BatchNormalization())\n","model.add(Dense(32,activation=LeakyReLU(alpha=0.07),kernel_initializer='he_normal',use_bias=True,kernel_regularizer=l1_l2(l1=0.001,l2=0.01)))\n","model.add(Dropout(0.2))\n","model.add(BatchNormalization())\n","model.add(Dense(64,activation='gelu',use_bias=True,kernel_regularizer=l1_l2(l1=0.001,l2=0.01)))\n","model.add(Dropout(0.2))\n","model.add(BatchNormalization())\n","model.add(Dense(32,activation='gelu',use_bias=True,kernel_regularizer=l1_l2(l1=0.001,l2=0.01)))\n","model.add(Dropout(0.2))\n","model.add(BatchNormalization())\n","model.add(Dense(16,activation='gelu',use_bias=True,kernel_regularizer=l1_l2(l1=0.001,l2=0.01)))\n","# model.add(Dense(1, activation=\"linear\"))\n","model.add(Dense(2, activation=\"softmax\"))\n","callback = EarlyStopping(monitor='val_loss', patience=2)\n","model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=[ 'accuracy' ])"]},{"cell_type":"code","source":["X_train, X_test, y_train, y_test = tts(X, y, test_size = 0.1)"],"metadata":{"id":"PzoDqSz6bUTZ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["batch_size=16\n","history = model.fit(X, y, validation_split=0.25,batch_size=batch_size,epochs=100, callbacks=[callback])"],"metadata":{"id":"b4pQ_YWmFijO","executionInfo":{"status":"ok","timestamp":1683191699504,"user_tz":240,"elapsed":45413,"user":{"displayName":"Matthew Cheng","userId":"00814230256052145695"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"661b0f5e-139b-4eae-aabb-f299200a7764"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Epoch 1/100\n","157/157 [==============================] - 6s 10ms/step - loss: 2.4989 - accuracy: 0.8035 - val_loss: 2.0538 - val_accuracy: 0.8417\n","Epoch 2/100\n","157/157 [==============================] - 1s 8ms/step - loss: 1.6974 - accuracy: 0.8607 - val_loss: 1.4762 - val_accuracy: 0.8441\n","Epoch 3/100\n","157/157 [==============================] - 2s 12ms/step - loss: 1.3104 - accuracy: 0.8647 - val_loss: 1.1701 - val_accuracy: 0.8525\n","Epoch 4/100\n","157/157 [==============================] - 2s 11ms/step - loss: 1.0842 - accuracy: 0.8659 - val_loss: 0.9849 - val_accuracy: 0.8597\n","Epoch 5/100\n","157/157 [==============================] - 1s 8ms/step - loss: 0.9382 - accuracy: 0.8727 - val_loss: 0.8699 - val_accuracy: 0.8621\n","Epoch 6/100\n","157/157 [==============================] - 1s 8ms/step - loss: 0.8361 - accuracy: 0.8731 - val_loss: 0.7789 - val_accuracy: 0.8741\n","Epoch 7/100\n","157/157 [==============================] - 1s 8ms/step - loss: 0.7862 - accuracy: 0.8647 - val_loss: 0.7249 - val_accuracy: 0.8729\n","Epoch 8/100\n","157/157 [==============================] - 1s 8ms/step - loss: 0.7391 - accuracy: 0.8703 - val_loss: 0.6855 - val_accuracy: 0.8777\n","Epoch 9/100\n","157/157 [==============================] - 1s 8ms/step - loss: 0.7056 - accuracy: 0.8691 - val_loss: 0.6522 - val_accuracy: 0.8753\n","Epoch 10/100\n","157/157 [==============================] - 1s 8ms/step - loss: 0.6761 - accuracy: 0.8715 - val_loss: 0.6272 - val_accuracy: 0.8789\n","Epoch 11/100\n","157/157 [==============================] - 1s 8ms/step - loss: 0.6584 - accuracy: 0.8760 - val_loss: 0.6083 - val_accuracy: 0.8777\n","Epoch 12/100\n","157/157 [==============================] - 1s 8ms/step - loss: 0.6356 - accuracy: 0.8707 - val_loss: 0.5961 - val_accuracy: 0.8801\n","Epoch 13/100\n","157/157 [==============================] - 2s 11ms/step - loss: 0.6212 - accuracy: 0.8747 - val_loss: 0.5826 - val_accuracy: 0.8861\n","Epoch 14/100\n","157/157 [==============================] - 2s 12ms/step - loss: 0.6077 - accuracy: 0.8768 - val_loss: 0.5731 - val_accuracy: 0.8897\n","Epoch 15/100\n","157/157 [==============================] - 1s 8ms/step - loss: 0.5962 - accuracy: 0.8760 - val_loss: 0.5669 - val_accuracy: 0.8837\n","Epoch 16/100\n","157/157 [==============================] - 1s 8ms/step - loss: 0.5927 - accuracy: 0.8756 - val_loss: 0.5610 - val_accuracy: 0.8861\n","Epoch 17/100\n","157/157 [==============================] - 1s 8ms/step - loss: 0.5834 - accuracy: 0.8784 - val_loss: 0.5536 - val_accuracy: 0.8885\n","Epoch 18/100\n","157/157 [==============================] - 1s 8ms/step - loss: 0.5791 - accuracy: 0.8784 - val_loss: 0.5491 - val_accuracy: 0.8897\n","Epoch 19/100\n","157/157 [==============================] - 1s 8ms/step - loss: 0.5821 - accuracy: 0.8735 - val_loss: 0.5463 - val_accuracy: 0.8897\n","Epoch 20/100\n","157/157 [==============================] - 1s 8ms/step - loss: 0.5749 - accuracy: 0.8804 - val_loss: 0.5433 - val_accuracy: 0.8909\n","Epoch 21/100\n","157/157 [==============================] - 1s 8ms/step - loss: 0.5819 - accuracy: 0.8768 - val_loss: 0.5421 - val_accuracy: 0.8873\n","Epoch 22/100\n","157/157 [==============================] - 1s 8ms/step - loss: 0.5747 - accuracy: 0.8768 - val_loss: 0.5389 - val_accuracy: 0.8909\n","Epoch 23/100\n","157/157 [==============================] - 2s 12ms/step - loss: 0.5765 - accuracy: 0.8784 - val_loss: 0.5391 - val_accuracy: 0.8861\n","Epoch 24/100\n","157/157 [==============================] - 2s 12ms/step - loss: 0.5698 - accuracy: 0.8743 - val_loss: 0.5354 - val_accuracy: 0.8945\n","Epoch 25/100\n","157/157 [==============================] - 1s 8ms/step - loss: 0.5753 - accuracy: 0.8747 - val_loss: 0.5345 - val_accuracy: 0.8921\n","Epoch 26/100\n","157/157 [==============================] - 1s 8ms/step - loss: 0.5720 - accuracy: 0.8760 - val_loss: 0.5342 - val_accuracy: 0.8897\n","Epoch 27/100\n","157/157 [==============================] - 1s 8ms/step - loss: 0.5621 - accuracy: 0.8804 - val_loss: 0.5360 - val_accuracy: 0.8873\n","Epoch 28/100\n","157/157 [==============================] - 1s 8ms/step - loss: 0.5731 - accuracy: 0.8707 - val_loss: 0.5320 - val_accuracy: 0.8957\n","Epoch 29/100\n","157/157 [==============================] - 1s 8ms/step - loss: 0.5661 - accuracy: 0.8739 - val_loss: 0.5321 - val_accuracy: 0.8933\n","Epoch 30/100\n","157/157 [==============================] - 1s 8ms/step - loss: 0.5730 - accuracy: 0.8792 - val_loss: 0.5341 - val_accuracy: 0.8873\n"]}]},{"cell_type":"code","source":["accuracy_score(y_test, np.around(model.predict(X_test)))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"TXCRYIs3ewtE","executionInfo":{"status":"ok","timestamp":1683191217142,"user_tz":240,"elapsed":516,"user":{"displayName":"Matthew Cheng","userId":"00814230256052145695"}},"outputId":"786c933d-4207-4dd4-9c99-8f8dca97dde4"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["11/11 [==============================] - 0s 2ms/step\n"]},{"output_type":"execute_result","data":{"text/plain":["0.9011976047904192"]},"metadata":{},"execution_count":27}]},{"cell_type":"code","source":["results = model.evaluate(x_test, y_test, batch_size=batch_size)\n","print( 'Loss = {} and Accuracy = {} %'.format( results[0] , results[1] * 100 ) )"],"metadata":{"id":"vr5ueL8ABEBN"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plt.plot(history.history['loss'], label='train')\n","plt.plot(history.history['val_loss'], label='test')\n","plt.legend()\n","plt.show()"],"metadata":{"id":"WxnnI3qxFkN_"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["k = 10\n","kf = KFold(n_splits=k,shuffle=True,random_state=1234)\n","acc = np.empty(k)\n","for i,(idxtrain, idxtest) in enumerate(kf.split(x)):\n","  xtrain = x[idxtrain]\n","  xtest = x[idxtest]\n","  ytrain = y[idxtrain]\n","  ytest = y[idxtest]\n","  model = Sequential()\n","  model.add(BatchNormalization())\n","  model.add((Dense(32,activation=LeakyReLU(alpha=0.07),kernel_initializer='he_normal',use_bias=True,kernel_regularizer=l1_l2(l1=0.001,l2=0.01))))\n","  model.add(BatchNormalization())\n","  model.add(Dense(64,activation='gelu',use_bias=True,kernel_regularizer=l1_l2(l1=0.001,l2=0.01)))\n","  model.add(BatchNormalization())\n","  model.add(Dense(32,activation='gelu',use_bias=True,kernel_regularizer=l1_l2(l1=0.001,l2=0.01)))\n","  model.add(BatchNormalization())\n","  model.add(Dense(16,activation='gelu',use_bias=True,kernel_regularizer=l1_l2(l1=0.001,l2=0.01)))\n","  model.add(Dense(1, activation=\"linear\")) # Here we collect and add the outputs produced by th eprevious layer\n","  # Compile model: The model is initialized with the Adam optimizer and then it is compiled.\n","  callback = EarlyStopping(monitor='metrics', patience=3)\n","  model.compile(optimizer=optimizer, loss='mean_squared_error')\n","  model.fit(xtrain,ytrain,validation_split=0.25,batch_size=25,epochs=100,verbose=0)\n","  acc[i] = accuracy_score(ytest,model.predict(xtest))\n","  print(acc[i])\n","print(acc.mean())"],"metadata":{"id":"3BbcF7jscpnp"},"execution_count":null,"outputs":[]}]}