Notebook - analyses.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import csv\n",
    "import numpy as np\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.neural_network import MLPRegressor\n",
    "from geopy.distance import vincenty\n",
    "from pyproj import Proj\n",
    "from math import radians, cos, sin, asin, sqrt\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.preprocessing import normalize"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# All functions used"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def haversine(lon1, lat1, lon2, lat2):\n",
    "    \"\"\"\n",
    "    Calculate the great circle distance between two points \n",
    "    on the earth (specified in decimal degrees)\n",
    "    \"\"\"\n",
    "    # convert decimal degrees to radians \n",
    "    lon1, lat1, lon2, lat2 = map(radians, [lon1, lat1, lon2, lat2])\n",
    "\n",
    "    # haversine formula \n",
    "    dlon = lon2 - lon1 \n",
    "    dlat = lat2 - lat1 \n",
    "    a = sin(dlat/2)**2 + cos(lat1) * cos(lat2) * sin(dlon/2)**2\n",
    "    c = 2 * asin(sqrt(a)) \n",
    "    r = 6371 # Radius of earth in kilometers. Use 3956 for miles\n",
    "    return c * r *1000 #return in meters\n",
    "#---------------------------------------------------------------------------------------------------------------      \n",
    "def regression_allset(Y_test_lon,Y_test_lat,X_test,ml_lon,ml_lat): #Only for tests\n",
    "                              \t\t\t\t\t\n",
    "\t#Turn into list\n",
    "\tpredicts_lon = ml_lon.predict(X_test).tolist()\n",
    "\tpredicts_lat = ml_lat.predict(X_test).tolist()\n",
    "\n",
    "\tY_test_lon = Y_test_lon.values.tolist()\n",
    "\tY_test_lat = Y_test_lat.values.tolist()\n",
    "\n",
    "\terror = []\n",
    "\n",
    "\tfor j in range(len(X_test)):\n",
    "\t\t\t\n",
    "\t\t#change the latitude and longitude unit\n",
    "\t\tmyProj = Proj(\"+proj=utm +zone=23K, +south +ellps=WGS84 +datum=WGS84 +units=m +no_defs\")\n",
    "\t\tlon_pred,lat_pred = myProj(predicts_lon[j], predicts_lat[j], inverse=True)\n",
    "\t\tlon_Y, lat_Y = myProj(Y_test_lon[j], Y_test_lat[j], inverse=True)\n",
    "\n",
    "\t\t#join in a unique list\n",
    "\t\tY = []\n",
    "\t\tY.append(lon_Y)\n",
    "\t\tY.append(lat_Y)\n",
    "\t\tpredict = []\n",
    "\t\tpredict.append(lon_pred)\n",
    "\t\tpredict.append(lat_pred)\t\t\t\n",
    "\n",
    "\t\t#The distance between the two latitudes is the error\n",
    "\t\tdistance = vincenty(Y, predict).meters \n",
    "\n",
    "\t\t#If you want to use haversine distance, uncomment the line below\n",
    "#\t\tdistance = haversine(lon_Y, lat_Y, lon_pred, lat_pred)\t\t\n",
    "\n",
    "\n",
    "\t\terror.append(distance)\t\n",
    "\n",
    "\t\n",
    "\treturn np.mean(error)\n",
    "#--------------------------------------------------------------------------------------------------------------\n",
    "#Calculate how many measurements each cell phone has\n",
    "def show_number_measurements(grouped_df):\n",
    "\tfor i in range(len(grouped_df)):\n",
    "\t\tprint (\"Measures:\" + str(len(grouped_df[i][1])) + \", PHONEID\" + str((grouped_df[i][1]).PHONEID.unique()))\n",
    "\tprint (\"\\n\")\n",
    "#---------------------------------------------------------------------------------------------------------------\n",
    "\n",
    "#Create a list of data frames. Each smartphone has its own data frame\n",
    "def create_phone_df(df,grouped_df):\n",
    "\tlist_phones = df.PHONEID.unique()\n",
    "\tdf_phone = []\n",
    "\n",
    "\tj=0\n",
    "\tfor i in range(0,24):\n",
    "\t\tif (i in list_phones):\n",
    "\t\t\tdf_phone.append(grouped_df[j][1])\n",
    "\t\t\tj=j+1\n",
    "\t\telse:\n",
    "\t\t\tdf_phone.append([])\n",
    "\n",
    "\treturn df_phone, list_phones\n",
    "#---------------------------------------------------------------------------------------------------------------\n",
    "\n",
    "def undersampling(df_phone, phones_used):\n",
    "\n",
    "\tminimum = 10000000\n",
    "\tund_df_phone = []\n",
    "\n",
    "\tfor i in phones_used:\n",
    "\t\t\n",
    "\t\t#find the smaller data frame\n",
    "\t\tif(len(df_phone[i]) < minimum):\n",
    "\t\t\tminimum = len(df_phone[i])\n",
    "\t\t\tind_min = i\n",
    "\n",
    "\t#unsampling the others data frames so they are the same size\t\t\n",
    "\tfor i in phones_used:\n",
    "\t\tif(i != ind_min):\n",
    "\t\t\tund_df_phone.append(df_phone[i].sample(n=minimum))\n",
    "\t\telse:\n",
    "\t\t\tund_df_phone.append(df_phone[i])\t\n",
    "\n",
    "\treturn und_df_phone\t\n",
    "\n",
    "#---------------------------------------------------------------------------------------------------------------\n",
    "def shuffle(und_df_phone):\n",
    "\n",
    "\tfor i in range(len(und_df_phone)):\n",
    "\t\tund_df_phone[i] = und_df_phone[i].sample(frac=1)\n",
    "\n",
    "\treturn und_df_phone\t \n",
    "\n",
    "#---------------------------------------------------------------------------------------------------------------\n",
    "def init_list_of_objects(size):\n",
    "    list_of_objects = list()\n",
    "    for i in range(0,size):\n",
    "        list_of_objects.append( list() ) #different object reference each time\n",
    "    return list_of_objects\n",
    "#---------------------------------------------------------------------------------------------------------------\n",
    "#return the number of hits\n",
    "def compare(Y_test_build, predictions_build, Y_test_floor, predictions_floor):\n",
    "\n",
    "\thits = 0\n",
    "\t#if tests and predictions have the same number of building and the same number of floor, the algorithm hit\n",
    "\tfor i in range(len(Y_test_floor)):\n",
    "\t\tif(Y_test_build[i] == predictions_build[i] and Y_test_floor[i] == predictions_floor[i]):\n",
    "\t\t\thits = hits +1\n",
    "\n",
    "\treturn hits\n",
    "\n",
    "#---------------------------------------------------------------------------------------------------------------\n",
    "def floor_classifier(predictions,train,test,method):\n",
    "\t\t\n",
    "\tsuccessful_amount = 0\n",
    "\tpred_floor_ordered = init_list_of_objects(len(predictions))\n",
    "\n",
    "\tif(method==1):\n",
    "\t\tmachine_learn = KNeighborsClassifier(n_neighbors=5, weights = 'distance')\n",
    "\telif(method==2):\n",
    "\t\t#machine_learn = MLPClassifier(solver='sgd',learning_rate = 'adaptive',verbose='true',activation='tanh',alpha=1e-5)\t\t\n",
    "\t\tmachine_learn = MLPClassifier(solver='sgd',learning_rate = 'adaptive',verbose='false',activation='tanh',alpha=1e-5,max_iter=400) #THE BEST\n",
    "\t\t#machine_learn = MLPClassifier(hidden_layer_sizes=(100,5), solver='sgd',learning_rate = 'adaptive',verbose='true',activation='tanh',alpha=1e-5,max_iter=500)\n",
    "\t\t#model = MLPClassifier(learning_rate = 'adaptive')\n",
    "\t\t#solvers = ['lbfgs', 'sgd', 'adam']\n",
    "\t\t#activations = ['identity', 'logistic', 'tanh', 'relu']\n",
    "\t\t#max_its = [200,400,600]\n",
    "\t\t#machine_learn = GridSearchCV(estimator=model, param_grid=dict(activation =activations,max_iter=max_its),n_jobs=7) #GRID\n",
    "\n",
    "\n",
    "\t#for each building\n",
    "\tfor i in range(3):\n",
    "\t\t\n",
    "\t\tnew_train = train.loc[train['BUILDINGID'] == i] #select for training only buildings with that label (0,1, or 2)\n",
    "\t\tindexes = [x for x in range(len(predictions)) if predictions[x]==i] #get the position of the samples that have building == i\n",
    "\t\t\n",
    "\t\tif (indexes): #if list is not empty\n",
    "\t\t\t#training, samples with building == i \n",
    "\t\t\tX_train = new_train.ix[:,0:519] # 0:519 -> columns containing the measures\n",
    "\t\t\tY_train = new_train['FLOOR']\n",
    "\t\t\tmachine_learn.fit(X_train,Y_train)                                   \n",
    "\t\t\t\n",
    "\t\t\t#testing samples w ith prediction building == i\n",
    "\t\t\tnew_test = test.iloc[indexes,:]\n",
    "\t\t\tX_test = new_test.ix[:,0:519] \n",
    "\n",
    "\t\t\tY_test_floor = new_test['FLOOR']\n",
    "\t\t\tY_test_build = new_test['BUILDINGID']\n",
    "\t\t\t#if(method ==2):\n",
    "\t\t\t\t#print \"best score:\"\n",
    "\t\t\t\t#print machine_learn.best_score_\n",
    "\t\t\tpredictions_floor = machine_learn.predict(X_test)\n",
    "\t\t\tpred_floor_ordered = put_list(predictions_floor, indexes, pred_floor_ordered)\n",
    "\n",
    "\t\t\t#Accumulate the number of hits \n",
    "\t\t\tsuccessful_amount = compare(Y_test_build.tolist(), predictions[indexes].tolist(), Y_test_floor.tolist(), predictions_floor.tolist()) + successful_amount\n",
    "\t\n",
    "\treturn successful_amount/float(len(test)), pred_floor_ordered\t\t\n",
    "\n",
    "#---------------------------------------------------------------------------------------------------------------\n",
    "\n",
    "def save_vec(hit_rate_build_mlp,hit_rate_floor_mlp,hit_rate_build_knn, hit_rate_floor_knn):\n",
    "\n",
    "\tnp.save(\"build_mlp.npy\",hit_rate_build_mlp)\n",
    "\tnp.save(\"floor_mlp.npy\",hit_rate_floor_mlp)\n",
    "\n",
    "\tnp.save(\"build_knn.npy\",hit_rate_build_knn)\n",
    "\tnp.save(\"floor_knn.npy\",hit_rate_floor_knn)\n",
    "#---------------------------------------------------------------------------------------------------------------\n",
    "def load_vec():\n",
    "\thit_rate_build_mlp = np.load(\"build_mlp.npy\")\n",
    "\thit_rate_floor_mlp = np.load(\"floor_mlp.npy\")\n",
    "\n",
    "\thit_rate_build_knn = np.load(\"build_knn.npy\")\n",
    "\thit_rate_floor_knn = np.load(\"floor_knn.npy\")\n",
    "\n",
    "#---------------------------------------------------------------------------------------------------------------\n",
    "\n",
    "def KFold(k, und_df_phone):\n",
    "\n",
    "\t#und_df_phone = shuffle(und_df_phone)\n",
    "\tphone = []\n",
    "\terror = []\n",
    "\t\n",
    "\t#split the data frame of each smartphone\n",
    "\tfor j in range(len(und_df_phone)): \n",
    "\t\tphone.append(np.array_split(und_df_phone[j],k)) #the first dimension of \"phone\" is each phone, the second is the splits data frames from that smatphone\n",
    "\n",
    "\t#GridSearch\t\n",
    "#\tmodel= KNeighborsRegressor(n_neighbors=5, weights = 'distance')\n",
    "#\tks = [1,3,5,7,10]\n",
    "#\tknn_lon= GridSearchCV(estimator=model, param_grid=dict(n_neighbors = ks),n_jobs=2) #GRID\n",
    "#\tknn_lat = GridSearchCV(estimator=model, param_grid=dict(n_neighbors = ks),n_jobs=2) #GRID\n",
    "\tknn_lon = KNeighborsRegressor(n_neighbors=5, weights = 'distance')\n",
    "\tknn_lat = KNeighborsRegressor(n_neighbors=5, weights = 'distance')\n",
    "\n",
    "\t#creating a empty list with size len(und_df_phone)\n",
    "\tmean_error_knn = init_list_of_objects(len(und_df_phone))\n",
    "\n",
    "\tfor i in range(k):\n",
    "\t\t#separate each smartphone's data frame in test and train\n",
    "\t\ttest = [] #list of data frames\n",
    "\t\ttrain =pd.DataFrame()\t\t\n",
    "\t\tfor j in range(len(und_df_phone)):\n",
    "\t\t\ttest.append(phone[j][i])\n",
    "\t\t\t#Join the train set\n",
    "\t\t\tfor x in range(k):\n",
    "\t\t\t\tif x != i:\n",
    "\t\t\t\t\ttrain = pd.concat([train,phone[j][x]])\t\n",
    "\t\t\n",
    "\t\t#Training with total training set\t\t\t\t\n",
    "\t\tX_train = train.ix[:,0:519]\n",
    "\t\tY_train_lon = train['LONGITUDE']\n",
    "\t\tY_train_lat = train['LATITUDE']\n",
    "\n",
    "\t\tknn_lon.fit(X_train,Y_train_lon) \n",
    "\t\tknn_lat.fit(X_train,Y_train_lat)   \n",
    "\n",
    "\n",
    "\n",
    "\t\t#test all phones\n",
    "\t\tfor j in range(len(und_df_phone)):\n",
    "\t\t\t#only pick up from test set the phone that you will be evaluated\n",
    "\t\t\tdata_test = test[j].ix[:,0:519] \n",
    "\t\t\tY_test_lon = test[j]['LONGITUDE']\n",
    "\t\t\tY_test_lat = test[j]['LATITUDE']\t\n",
    "\n",
    "\n",
    "\t\t\tmean_error_knn[j].append( regression_allset(Y_test_lon,Y_test_lat,data_test,knn_lon,knn_lat) )\n",
    "\n",
    "\n",
    "\tnp.save(\"mean_error_knn.npy\", mean_error_knn)\t\t\n",
    "\n",
    "\"\"\"\t\n",
    "\tprint \"mean error regression knn\"\n",
    "\tprint str(np.mean(mean_error_knn[0])) + \" - \" +  str(np.std(mean_error_knn[0]))\n",
    "\tprint str(np.mean(mean_error_knn[1])) + \" - \" +  str(np.std(mean_error_knn[1]))\n",
    "\tprint str(np.mean(mean_error_knn[2])) + \" - \" +  str(np.std(mean_error_knn[2]))\n",
    "\tprint str(np.mean(mean_error_knn[3])) + \" - \" +  str(np.std(mean_error_knn[3]))\n",
    "\tprint \" \"\n",
    "\n",
    "\tprint \"Best Params Lon\"\n",
    "\tprint knn_lon.best_params_\n",
    "\n",
    "\n",
    "\tprint \"Best Params Lat\"\n",
    "\tprint knn_lat.best_params_\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "#--------------------------------------------------------------------------------------------------------------\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "\t#defines\n",
    "\tphones_used = [6,7,13,14]\n",
    "\tk=10\n",
    "\n",
    "\t#convert csv file in an data frame\n",
    "\tdf = pd.read_csv('trainingData.csv')\n",
    "\n",
    "\t#group by pohneID\n",
    "\tgrouped_df = list(df.groupby(['PHONEID']))\n",
    "\n",
    "\t#show_number_measurements(grouped_df)\n",
    "\n",
    "\t#create a data frame for each phone\n",
    "\tdf_phone, list_phones = create_phone_df(df,grouped_df)\n",
    "\t\n",
    "\t#Doing undersampling\n",
    "\tund_df_phone = undersampling(df_phone,phones_used)\n",
    "    #the index 1 is the smartphone 6, the index 2 is the smartphone 7...\n",
    "\n",
    "\t#KFold(k, und_df_phone)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1383, 529)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "und_df_phone[1].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}