Reboot_K_RF_final.ipynb

{"cells":[{"metadata":{"colab":{},"colab_type":"code","id":"KJoBqNk6ilc-","trusted":true},"cell_type":"code","source":"import sys\nimport sklearn\nimport numpy as np\nimport zipfile\nimport pandas as pd\nimport matplotlib as plt\nfrom sklearn.preprocessing import LabelEncoder,OneHotEncoder\n# np.set_printoptions(threshold=sys.maxsize)","execution_count":1,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train=pd.read_csv('../input/dataset/TrainingData.csv')\ntest=pd.read_csv('../input/dataset/TestData.csv')\nsample=pd.read_csv('../input/dataset/SubmissionFormat.csv')","execution_count":2,"outputs":[{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/IPython/core/interactiveshell.py:2785: DtypeWarning: Columns (5,11) have mixed types. Specify dtype option on import or set low_memory=False.\n  interactivity=interactivity, compiler=compiler, result=result)\n","name":"stderr"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"def mean_iou(y_true, y_pred):\n   score, up_opt = tf.metrics.mean_iou(y_true, y_pred, NUM_CLASSES)\n   K.get_session().run(tf.local_variables_initializer())\n   with tf.control_dependencies([up_opt]):\n       score = tf.identity(score)\n   return score","execution_count":3,"outputs":[]},{"metadata":{"colab":{},"colab_type":"code","id":"-vdZGKFXjA3J","trusted":true},"cell_type":"code","source":"training_data = train[['Facility_or_Department', 'Function_Description','Fund_Description',\n                       'Job_Title_Description', 'Location_Description','Object_Description',\n                       'Position_Extra', 'Program_Description', 'SubFund_Description',\n                       'Sub_Object_Description', 'Text_1', 'Text_2','Text_3','Text_4', 'Total','FTE']]\n\nlabels = (train[['Function','Object_Type','Operating_Status','Position_Type','Pre_K', 'Reporting',\n                'Sharing','Student_Type', 'Use']])\n\ntest_data = test[['Facility_or_Department', 'Function_Description','Fund_Description',\n                       'Job_Title_Description', 'Location_Description','Object_Description',\n                       'Position_Extra', 'Program_Description', 'SubFund_Description',\n                       'Sub_Object_Description', 'Text_1', 'Text_2','Text_3','Text_4', 'Total','FTE']]","execution_count":4,"outputs":[]},{"metadata":{"colab":{},"colab_type":"code","id":"x-qTKdxnjKBK","trusted":true},"cell_type":"code","source":"training_data=training_data.drop(['Total'],axis=1)\ntraining_data=training_data.drop(['FTE'],axis=1)","execution_count":5,"outputs":[]},{"metadata":{"colab":{},"colab_type":"code","id":"Wpu7xa2IjMP_","trusted":true},"cell_type":"code","source":"test_data=test_data.drop(['Total'], axis = 1)\ntest_data=test_data.drop(['FTE'], axis = 1)","execution_count":6,"outputs":[]},{"metadata":{"colab":{},"colab_type":"code","id":"j9HDX0q_jOsR","trusted":true},"cell_type":"code","source":"for i in range(training_data.shape[1]):\n    labelencoder_X=LabelEncoder();\n    training_data.iloc[:,i] = training_data.iloc[:,i].astype(str)\n    training_data.iloc[:,i] = training_data.iloc[:,i].str.replace('[^\\w\\s]','')\n    training_data.iloc[:,i] = labelencoder_X.fit_transform(training_data.iloc[:,i])","execution_count":7,"outputs":[]},{"metadata":{"colab":{},"colab_type":"code","id":"2pkBCYdejUuZ","trusted":true},"cell_type":"code","source":"for i in range(labels.shape[1]):\n    labelencoder_X=LabelEncoder();\n    labels.iloc[:,i] = labels.iloc[:,i].astype(str)\n    labels.iloc[:,i] = labels.iloc[:,i].str.replace('[^\\w\\s]','')\n    labels.iloc[:,i] = labelencoder_X.fit_transform(labels.iloc[:,i])","execution_count":8,"outputs":[{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py:630: SettingWithCopyWarning: \nA value is trying to be set on a copy of a slice from a DataFrame.\nTry using .loc[row_indexer,col_indexer] = value instead\n\nSee the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n  self.obj[item_labels[indexer[info_axis]]] = value\n/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py:543: SettingWithCopyWarning: \nA value is trying to be set on a copy of a slice from a DataFrame.\nTry using .loc[row_indexer,col_indexer] = value instead\n\nSee the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n  self.obj[item] = s\n","name":"stderr"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(test_data.shape[1]):\n    labelencoder_X=LabelEncoder();\n    test_data.iloc[:,i] = test_data.iloc[:,i].astype(str)\n    test_data.iloc[:,i] = test_data.iloc[:,i].str.replace('[^\\w\\s]','')\n    test_data.iloc[:,i] = labelencoder_X.fit_transform(test_data.iloc[:,i])","execution_count":9,"outputs":[]},{"metadata":{"colab":{},"colab_type":"code","id":"hQWYzyMkjYQP","trusted":true},"cell_type":"code","source":"import keras\nfrom keras.models import Sequential#to initialise our NN\nfrom keras.layers import Dense#to build layers of our ANN\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn import metrics\n\nx=training_data.iloc[:,0:14].values\ny=labels.iloc[:,0].values\np=np.unique(y)\ny=np.reshape(y,(400277,1))\nX_train,X_test,y_train,y_test=train_test_split(x,y,test_size=0.3,random_state=0)\nsc=StandardScaler()\nX_train=sc.fit_transform(X_train)\nX_test=sc.transform(X_test)\ntest_data=sc.fit_transform(test_data)\nnp.unique(y)\nmodel = RandomForestClassifier(n_estimators=200,random_state=0)\nmodel.fit(X_train, y_train)\nprob=np.array(model.predict_proba(test_data),dtype=object)\n# print(\"Accuracy:\",metrics.accuracy_score(y_test, prob))\nfor i in range(1,9):\n    x=training_data.iloc[:,0:14].values\n    y=labels.iloc[:,i].values\n    p=np.unique(y)\n    y=np.reshape(y,(400277,1))\n    X_train,X_test,y_train,y_test=train_test_split(x,y,test_size=0.3,random_state=0)\n    sc=StandardScaler()\n    X_train=sc.fit_transform(X_train)\n    X_test=sc.transform(X_test)\n    model = RandomForestClassifier(n_estimators=100, random_state=0)  \n    model.fit(X_train, y_train)\n    y_pred= np.array(model.predict_proba(test_data))\n    prob=np.concatenate((prob,y_pred),axis=1)\n    #     print(\"Accuracy:\",metrics.accuracy_score(y_test, y_pred))","execution_count":17,"outputs":[{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:595: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n  warnings.warn(msg, DataConversionWarning)\n/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:595: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n  warnings.warn(msg, DataConversionWarning)\n/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:595: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n  warnings.warn(msg, DataConversionWarning)\n/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:20: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:595: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n  warnings.warn(msg, DataConversionWarning)\n/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:595: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n  warnings.warn(msg, DataConversionWarning)\n/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:595: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n  warnings.warn(msg, DataConversionWarning)\n/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:33: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:595: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n  warnings.warn(msg, DataConversionWarning)\n/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:595: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n  warnings.warn(msg, DataConversionWarning)\n/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:595: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n  warnings.warn(msg, DataConversionWarning)\n/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:33: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:595: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n  warnings.warn(msg, DataConversionWarning)\n/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:595: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n  warnings.warn(msg, DataConversionWarning)\n/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:595: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n  warnings.warn(msg, DataConversionWarning)\n/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:33: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:595: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n  warnings.warn(msg, DataConversionWarning)\n/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:595: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n  warnings.warn(msg, DataConversionWarning)\n/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:595: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n  warnings.warn(msg, DataConversionWarning)\n/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:33: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:595: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n  warnings.warn(msg, DataConversionWarning)\n/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:595: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n  warnings.warn(msg, DataConversionWarning)\n/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:595: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n  warnings.warn(msg, DataConversionWarning)\n/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:33: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:595: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n  warnings.warn(msg, DataConversionWarning)\n/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:595: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n  warnings.warn(msg, DataConversionWarning)\n/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:595: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n  warnings.warn(msg, DataConversionWarning)\n/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:33: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:595: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n  warnings.warn(msg, DataConversionWarning)\n/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:595: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n  warnings.warn(msg, DataConversionWarning)\n/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:595: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n  warnings.warn(msg, DataConversionWarning)\n/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:33: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:595: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n  warnings.warn(msg, DataConversionWarning)\n/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:595: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n  warnings.warn(msg, DataConversionWarning)\n/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:595: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n  warnings.warn(msg, DataConversionWarning)\n/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:33: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n","name":"stderr"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.shape(prob)","execution_count":22,"outputs":[{"output_type":"execute_result","execution_count":22,"data":{"text/plain":"(50064, 104)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.savetxt(\"prob_RF.csv\",prob, delimiter=\",\")","execution_count":19,"outputs":[]}],"metadata":{"colab":{"collapsed_sections":[],"name":"Reboot_ANN.ipynb","provenance":[],"version":"0.3.2"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.3"}},"nbformat":4,"nbformat_minor":1}