From 2bd1b8e75e70b8191b987b0b3df0b59b6b143af4 Mon Sep 17 00:00:00 2001
From: qlmx <wenruichn@gmail.com>
Date: Tue, 13 Aug 2019 15:16:49 +0800
Subject: [PATCH] add pytorch

---
 .idea/data_mining_models.iml        |  12 --
 .idea/misc.xml                      |   7 -
 .idea/modules.xml                   |   8 -
 .idea/vcs.xml                       |   6 -
 .idea/workspace.xml                 | 256 ----------------------------
 README.md                           |  12 +-
 code/keras/binary_class.py          |  13 +-
 code/keras/multi_class.py           |  12 +-
 code/keras/regression.py            |  12 +-
 code/lgb/binary_class.py            |   4 +-
 code/lgb/multi_class.py             |   4 +-
 code/lgb/multi_class_custom_loss.py |   2 +-
 code/lgb/multi_class_weight_loss.py |   2 +-
 code/lgb/regression.py              |   2 +-
 code/pytorch/binary_class.py        | 196 +++++++++++++++++++++
 code/pytorch/multi_class.py         | 187 ++++++++++++++++++++
 code/pytorch/regression.py          | 188 ++++++++++++++++++++
 code/xgb/binary_class.py            |   4 +-
 code/xgb/multi_class.py             |   4 +-
 code/xgb/regression.py              |   4 +-
 requirements.txt                    |   1 +
 21 files changed, 613 insertions(+), 323 deletions(-)
 delete mode 100644 .idea/data_mining_models.iml
 delete mode 100644 .idea/misc.xml
 delete mode 100644 .idea/modules.xml
 delete mode 100644 .idea/vcs.xml
 delete mode 100644 .idea/workspace.xml
 create mode 100644 code/pytorch/binary_class.py
 create mode 100644 code/pytorch/multi_class.py
 create mode 100644 code/pytorch/regression.py

diff --git a/.idea/data_mining_models.iml b/.idea/data_mining_models.iml
deleted file mode 100644
index 7c9d48f..0000000
--- a/.idea/data_mining_models.iml
+++ /dev/null
@@ -1,12 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module type="PYTHON_MODULE" version="4">
-  <component name="NewModuleRootManager">
-    <content url="file://$MODULE_DIR$" />
-    <orderEntry type="inheritedJdk" />
-    <orderEntry type="sourceFolder" forTests="false" />
-  </component>
-  <component name="TestRunnerService">
-    <option name="projectConfiguration" value="pytest" />
-    <option name="PROJECT_TEST_RUNNER" value="pytest" />
-  </component>
-</module>
\ No newline at end of file
diff --git a/.idea/misc.xml b/.idea/misc.xml
deleted file mode 100644
index 3999087..0000000
--- a/.idea/misc.xml
+++ /dev/null
@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="JavaScriptSettings">
-    <option name="languageLevel" value="ES6" />
-  </component>
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6" project-jdk-type="Python SDK" />
-</project>
\ No newline at end of file
diff --git a/.idea/modules.xml b/.idea/modules.xml
deleted file mode 100644
index 0e430ff..0000000
--- a/.idea/modules.xml
+++ /dev/null
@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectModuleManager">
-    <modules>
-      <module fileurl="file://$PROJECT_DIR$/.idea/data_mining_models.iml" filepath="$PROJECT_DIR$/.idea/data_mining_models.iml" />
-    </modules>
-  </component>
-</project>
\ No newline at end of file
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
deleted file mode 100644
index 94a25f7..0000000
--- a/.idea/vcs.xml
+++ /dev/null
@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="VcsDirectoryMappings">
-    <mapping directory="$PROJECT_DIR$" vcs="Git" />
-  </component>
-</project>
\ No newline at end of file
diff --git a/.idea/workspace.xml b/.idea/workspace.xml
deleted file mode 100644
index ed2ff98..0000000
--- a/.idea/workspace.xml
+++ /dev/null
@@ -1,256 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ChangeListManager">
-    <list default="true" id="65cc020e-a97e-48c3-a557-398e6e6b3413" name="Default Changelist" comment="">
-      <change beforePath="$PROJECT_DIR$/README.md" beforeDir="false" afterPath="$PROJECT_DIR$/README.md" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/code/keras/mutil_class.py" beforeDir="false" />
-      <change beforePath="$PROJECT_DIR$/code/lgb/multi_class.py" beforeDir="false" afterPath="$PROJECT_DIR$/code/lgb/multi_class.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/code/xgb/multi_class.py" beforeDir="false" afterPath="$PROJECT_DIR$/code/xgb/multi_class.py" afterDir="false" />
-    </list>
-    <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
-    <option name="SHOW_DIALOG" value="false" />
-    <option name="HIGHLIGHT_CONFLICTS" value="true" />
-    <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
-    <option name="LAST_RESOLUTION" value="IGNORE" />
-  </component>
-  <component name="FileEditorManager">
-    <leaf>
-      <file pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/code/lgb/binary_class.py">
-          <provider selected="true" editor-type-id="text-editor" />
-        </entry>
-      </file>
-      <file pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/code/keras/regression.py">
-          <provider selected="true" editor-type-id="text-editor" />
-        </entry>
-      </file>
-      <file pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/code/lgb/multi_class.py">
-          <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="436">
-              <caret line="54" column="29" lean-forward="true" selection-start-line="54" selection-start-column="29" selection-end-line="54" selection-end-column="29" />
-            </state>
-          </provider>
-        </entry>
-      </file>
-      <file pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/code/lgb/multi_class_custom_loss.py">
-          <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="-1507" />
-          </provider>
-        </entry>
-      </file>
-      <file pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/code/lgb/regression.py">
-          <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="1636">
-              <caret line="97" column="47" lean-forward="true" selection-start-line="97" selection-start-column="47" selection-end-line="97" selection-end-column="47" />
-            </state>
-          </provider>
-        </entry>
-      </file>
-      <file pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/code/keras/binary_class.py">
-          <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="-336" />
-          </provider>
-        </entry>
-      </file>
-      <file pinned="false" current-in-tab="true">
-        <entry file="file://$PROJECT_DIR$/code/keras/multi_class.py">
-          <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="-1925" />
-          </provider>
-        </entry>
-      </file>
-    </leaf>
-  </component>
-  <component name="Git.Settings">
-    <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
-  </component>
-  <component name="IdeDocumentHistory">
-    <option name="CHANGED_PATHS">
-      <list>
-        <option value="$PROJECT_DIR$/code/lgb/multi_class.py" />
-        <option value="$PROJECT_DIR$/code/xgb/multi_class.py" />
-      </list>
-    </option>
-  </component>
-  <component name="ProjectConfigurationFiles">
-    <option name="files">
-      <list>
-        <option value="$PROJECT_DIR$/.idea/data_mining_models.iml" />
-        <option value="$PROJECT_DIR$/.idea/misc.xml" />
-        <option value="$PROJECT_DIR$/.idea/modules.xml" />
-        <option value="$PROJECT_DIR$/.idea/vcs.xml" />
-      </list>
-    </option>
-  </component>
-  <component name="ProjectFrameBounds" extendedState="6" fullScreen="true">
-    <option name="x" value="-322" />
-    <option name="y" value="-1080" />
-    <option name="width" value="1920" />
-    <option name="height" value="1080" />
-  </component>
-  <component name="ProjectView">
-    <navigator proportions="" version="1">
-      <foldersAlwaysOnTop value="true" />
-    </navigator>
-    <panes>
-      <pane id="Scope" />
-      <pane id="ProjectPane">
-        <subPane>
-          <expand>
-            <path>
-              <item name="data_mining_models" type="b2602c69:ProjectViewProjectNode" />
-              <item name="data_mining_models" type="462c0819:PsiDirectoryNode" />
-            </path>
-            <path>
-              <item name="data_mining_models" type="b2602c69:ProjectViewProjectNode" />
-              <item name="data_mining_models" type="462c0819:PsiDirectoryNode" />
-              <item name="code" type="462c0819:PsiDirectoryNode" />
-            </path>
-            <path>
-              <item name="data_mining_models" type="b2602c69:ProjectViewProjectNode" />
-              <item name="data_mining_models" type="462c0819:PsiDirectoryNode" />
-              <item name="code" type="462c0819:PsiDirectoryNode" />
-              <item name="keras" type="462c0819:PsiDirectoryNode" />
-            </path>
-            <path>
-              <item name="data_mining_models" type="b2602c69:ProjectViewProjectNode" />
-              <item name="data_mining_models" type="462c0819:PsiDirectoryNode" />
-              <item name="code" type="462c0819:PsiDirectoryNode" />
-              <item name="lgb" type="462c0819:PsiDirectoryNode" />
-            </path>
-            <path>
-              <item name="data_mining_models" type="b2602c69:ProjectViewProjectNode" />
-              <item name="data_mining_models" type="462c0819:PsiDirectoryNode" />
-              <item name="code" type="462c0819:PsiDirectoryNode" />
-              <item name="xgb" type="462c0819:PsiDirectoryNode" />
-            </path>
-          </expand>
-          <select />
-        </subPane>
-      </pane>
-    </panes>
-  </component>
-  <component name="PropertiesComponent">
-    <property name="WebServerToolWindowFactoryState" value="true" />
-    <property name="last_opened_file_path" value="$PROJECT_DIR$" />
-    <property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
-    <property name="nodejs_npm_path_reset_for_default_project" value="true" />
-    <property name="settings.editor.selected.configurable" value="com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable" />
-  </component>
-  <component name="RunDashboard">
-    <option name="ruleStates">
-      <list>
-        <RuleState>
-          <option name="name" value="ConfigurationTypeDashboardGroupingRule" />
-        </RuleState>
-        <RuleState>
-          <option name="name" value="StatusDashboardGroupingRule" />
-        </RuleState>
-      </list>
-    </option>
-  </component>
-  <component name="SvnConfiguration">
-    <configuration />
-  </component>
-  <component name="TaskManager">
-    <task active="true" id="Default" summary="Default task">
-      <changelist id="65cc020e-a97e-48c3-a557-398e6e6b3413" name="Default Changelist" comment="" />
-      <created>1564841730565</created>
-      <option name="number" value="Default" />
-      <option name="presentableId" value="Default" />
-      <updated>1564841730565</updated>
-      <workItem from="1564841734554" duration="571000" />
-    </task>
-    <servers />
-  </component>
-  <component name="TimeTrackingManager">
-    <option name="totallyTimeSpent" value="571000" />
-  </component>
-  <component name="ToolWindowManager">
-    <frame x="-337" y="-1080" width="1920" height="1080" extended-state="0" />
-    <editor active="true" />
-    <layout>
-      <window_info id="Favorites" side_tool="true" />
-      <window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.24973376" />
-      <window_info id="Structure" order="1" side_tool="true" weight="0.25" />
-      <window_info anchor="bottom" id="Docker" show_stripe_button="false" />
-      <window_info anchor="bottom" id="Database Changes" />
-      <window_info anchor="bottom" id="Version Control" />
-      <window_info anchor="bottom" id="Python Console" />
-      <window_info anchor="bottom" id="Terminal" />
-      <window_info anchor="bottom" id="Event Log" side_tool="true" />
-      <window_info anchor="bottom" id="Message" order="0" />
-      <window_info anchor="bottom" id="Find" order="1" />
-      <window_info anchor="bottom" id="Run" order="2" />
-      <window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
-      <window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
-      <window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
-      <window_info anchor="bottom" id="TODO" order="6" />
-      <window_info anchor="right" id="Remote Host" />
-      <window_info anchor="right" id="SciView" />
-      <window_info anchor="right" id="Database" />
-      <window_info anchor="right" id="Commander" order="0" weight="0.4" />
-      <window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
-      <window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
-    </layout>
-  </component>
-  <component name="TypeScriptGeneratedFilesManager">
-    <option name="version" value="1" />
-  </component>
-  <component name="editorHistoryManager">
-    <entry file="file://$PROJECT_DIR$/code/keras/mutil_class.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="822">
-          <caret line="157" column="42" selection-start-line="157" selection-start-column="9" selection-end-line="157" selection-end-column="42" />
-        </state>
-      </provider>
-    </entry>
-    <entry file="file://$PROJECT_DIR$/code/xgb/multi_class.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="651">
-          <caret line="96" selection-start-line="96" selection-end-line="96" />
-        </state>
-      </provider>
-    </entry>
-    <entry file="file://$PROJECT_DIR$/code/lgb/regression.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="1636">
-          <caret line="97" column="47" lean-forward="true" selection-start-line="97" selection-start-column="47" selection-end-line="97" selection-end-column="47" />
-        </state>
-      </provider>
-    </entry>
-    <entry file="file://$PROJECT_DIR$/code/lgb/binary_class.py">
-      <provider selected="true" editor-type-id="text-editor" />
-    </entry>
-    <entry file="file://$PROJECT_DIR$/code/keras/regression.py">
-      <provider selected="true" editor-type-id="text-editor" />
-    </entry>
-    <entry file="file://$PROJECT_DIR$/code/lgb/multi_class.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="436">
-          <caret line="54" column="29" lean-forward="true" selection-start-line="54" selection-start-column="29" selection-end-line="54" selection-end-column="29" />
-        </state>
-      </provider>
-    </entry>
-    <entry file="file://$PROJECT_DIR$/code/lgb/multi_class_custom_loss.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="-1507" />
-      </provider>
-    </entry>
-    <entry file="file://$PROJECT_DIR$/code/keras/binary_class.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="-336" />
-      </provider>
-    </entry>
-    <entry file="file://$PROJECT_DIR$/code/keras/multi_class.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="-1925" />
-      </provider>
-    </entry>
-  </component>
-</project>
\ No newline at end of file
diff --git a/README.md b/README.md
index 8224c78..288be98 100644
--- a/README.md
+++ b/README.md
@@ -18,6 +18,10 @@
   * [binary_class.py](./code/keras/binary_class.py):keras实现的mlp,做二分类任务
   * [multi_class.py](./code/keras/multi_class.py):keras实现的mlp,做多分类任务
   * [regression.py](./code/keras/regression.py):keras实现的mlp,做回归任务
+* pytorch实现的mlp
+  * [binary_class.py](./code/pytorch/binary_class.py):pytorch实现的mlp,做二分类任务
+  * [multi_class.py](./code/pytorch/multi_class.py):pytorch实现的mlp,做多分类任务
+  * [regression.py](./code/pytorch/regression.py):实现的mlp,做回归任务
 
 ## 2.环境设置
 
@@ -30,6 +34,7 @@ matplotlib
 sklearn
 tensorflow==1.12.0
 keras==2.2.4
+pytorch
 seaborn
 lightgbm==2.2.1
 xgboost==0.90
@@ -40,14 +45,15 @@ xgboost==0.90
 * [基于lightgbm实现的二分类、多分类和回归任务](https://mp.weixin.qq.com/s/t6EpWmLWP81DcJ7AUro3Ng)
 * [基于xgboost实现的二分类、多分类和回归任务]()
 * [基于keras实现的二分类、多分类和回归任务]()
+* [基于pytorch实现的二分类、多分类和回归任务]()
 
 ## 4. 最后
 
-目前只整理了几个常用的模型，下一步会将用到的不错的代码同步进来，如果有什么问题可以直接指出来，大家一块进步，如果对您有帮助，麻烦点击一下star,鼓励一下作者。如果有最新内容更新会第一时间在微信公众号公布，希望能和大家一块成长。
+目前只整理了几个常用的模型，下一步会将用到的不错的代码同步进来，如果有问题或者发现有什么错误的地方需要交流可以通过微信公众号或者代码中的邮箱联系作者，也可以通过Issues提出问题。如果该项目对你有帮助，麻烦点击一下star，鼓励一下作者。如果有最新内容更新会第一时间在微信公众号内发布，希望能和你一起在AI方面前进，成长。
 
-作者的知乎：[QLMX](https://www.zhihu.com/people/qlmx-61/activities)
+作者的知乎：[QLMX](https://www.zhihu.com/people/qlmx-61/activities), 知乎专栏：[ML与DL成长之路](https://zhuanlan.zhihu.com/c_1138029910563020800)
 
-微信公号：AI成长社![wx](./result/wx.jpg)
+微信公号：AI成长社<img src="./result/wx.jpg" width = "80" height = "80" />
 
 
 
diff --git a/code/keras/binary_class.py b/code/keras/binary_class.py
index 1b80b27..a9bc785 100644
--- a/code/keras/binary_class.py
+++ b/code/keras/binary_class.py
@@ -6,7 +6,7 @@
 @contact: wenruichn@gmail.com
 @time: 2019-08-02 10:26
 公众号：AI成长社
-知乎：https://www.zhihu.com/people/qlmx-61/activities
+知乎：https://www.zhihu.com/people/qlmx-61/columns
 """
 import pandas as pd
 import numpy as np
@@ -42,7 +42,7 @@
 train = data[data['label'] != -1]
 test = data[data['label'] == -1]
 
-#del dataframe
+## Clean up the memory
 del data, train_data, test_data
 gc.collect()
 
@@ -61,8 +61,8 @@
 for i in train_x.columns:
     # print(i, train_x[i].isnull().sum(), test[i].isnull().sum())
     if train_x[i].isnull().sum() != 0:
-        train_x[i] = train_x[i].fillna(0)
-        test[i] = test[i].fillna(0)
+        train_x[i] = train_x[i].fillna(-1)
+        test[i] = test[i].fillna(-1)
 
 ## normalized
 scaler = StandardScaler()
@@ -71,7 +71,7 @@
 
 ## simple mlp model
 K.clear_session()
-def build_model(dropout_rate=0.25, activation='relu'):
+def MLP(dropout_rate=0.25, activation='relu'):
     start_neurons = 512
     model = Sequential()
     model.add(Dense(start_neurons, input_dim=train_X.shape[1], activation=activation))
@@ -128,7 +128,7 @@ def plot_loss_acc(history, fold):
     x_valid, y_valid = train_X[val_], train_y[val_]
 
 
-    model = build_model(dropout_rate=0.5, activation='tanh')
+    model = MLP(dropout_rate=0.5, activation='tanh')
     model.compile(optimizer='adam', loss='binary_crossentropy',  metrics=['accuracy'])
     history = model.fit(x_train, y_train,
                         validation_data=[x_valid, y_valid],
@@ -148,3 +148,4 @@ def plot_loss_acc(history, fold):
 result = []
 for pred in NN_predictions:
     result.append(1 if pred > threshold else 0)
+print(result)
diff --git a/code/keras/multi_class.py b/code/keras/multi_class.py
index 7e86c17..4138561 100644
--- a/code/keras/multi_class.py
+++ b/code/keras/multi_class.py
@@ -6,7 +6,7 @@
 @contact: wenruichn@gmail.com
 @time: 2019-08-01 15:28
 公众号：AI成长社
-知乎：https://www.zhihu.com/people/qlmx-61/activities
+知乎：https://www.zhihu.com/people/qlmx-61/columns
 """
 import pandas as pd
 import numpy as np
@@ -45,7 +45,7 @@
 train = data[data['label'] != -1]
 test = data[data['label'] == -1]
 
-#del dataframe
+##Clean up the memory
 del data, train_data, test_data
 gc.collect()
 
@@ -62,8 +62,8 @@
 for i in train_x.columns:
     # print(i, train_x[i].isnull().sum(), test[i].isnull().sum())
     if train_x[i].isnull().sum() != 0:
-        train_x[i].fillna(0, inplace=True)
-        test[i].fillna(0, inplace=True)
+        train_x[i].fillna(-1, inplace=True)
+        test[i].fillna(-1, inplace=True)
 
 ## normalized
 scaler = StandardScaler()
@@ -75,7 +75,7 @@
 
 ## simple mlp model
 K.clear_session()
-def build_model(dropout_rate=0.25, activation='relu'):
+def MLP(dropout_rate=0.25, activation='relu'):
     start_neurons = 512
     model = Sequential()
     model.add(Dense(start_neurons, input_dim=train_X.shape[1], activation=activation))
@@ -139,7 +139,7 @@ def plot_loss_acc(history, fold):
     x_valid, y_valid = train_X[val_], y_categorical[val_]
 
 
-    model = build_model(dropout_rate=0.5, activation='relu')
+    model = MLP(dropout_rate=0.5, activation='relu')
     model.compile(optimizer='adam', loss='categorical_crossentropy',  metrics=['accuracy'])
     history = model.fit(x_train, y_train,
                         validation_data=[x_valid, y_valid],
diff --git a/code/keras/regression.py b/code/keras/regression.py
index 79243a4..ed87c30 100644
--- a/code/keras/regression.py
+++ b/code/keras/regression.py
@@ -6,7 +6,7 @@
 @contact: wenruichn@gmail.com
 @time: 2019-08-01 23:10
 公众号：AI成长社
-知乎：https://www.zhihu.com/people/qlmx-61/activities
+知乎：https://www.zhihu.com/people/qlmx-61/columns
 """
 import pandas as pd
 import numpy as np
@@ -46,7 +46,7 @@
 train = data[data['label'] != -1]
 test = data[data['label'] == -1]
 
-#del dataframe
+##Clean up the memory
 del data, train_data, test_data
 gc.collect()
 
@@ -62,8 +62,8 @@
 for i in train_x.columns:
     # print(i, train_x[i].isnull().sum(), test[i].isnull().sum())
     if train_x[i].isnull().sum() != 0:
-        train_x[i].fillna(0, inplace=True)
-        test[i].fillna(0, inplace=True)
+        train_x[i].fillna(-1, inplace=True)
+        test[i].fillna(-1, inplace=True)
 
 ## normalized
 scaler = StandardScaler()
@@ -71,7 +71,7 @@
 test_X = scaler.transform(test)
 
 K.clear_session()
-def build_model(dropout_rate=0.25, activation='relu'):
+def MLP(dropout_rate=0.25, activation='relu'):
     start_neurons = 512
     model = Sequential()
     model.add(Dense(start_neurons, input_dim=train_X.shape[1], activation=activation))
@@ -128,7 +128,7 @@ def plot_loss_acc(history, fold):
     x_valid, y_valid = train_X[val_], train_y[val_]
 
 
-    model = build_model(dropout_rate=0.5, activation='relu')
+    model = MLP(dropout_rate=0.5, activation='relu')
     model.compile(optimizer='adam', loss='mse', metrics=['mae', 'acc'])
     history = model.fit(x_train, y_train,
                         validation_data=[x_valid, y_valid],
diff --git a/code/lgb/binary_class.py b/code/lgb/binary_class.py
index 4a40efd..d38df89 100644
--- a/code/lgb/binary_class.py
+++ b/code/lgb/binary_class.py
@@ -6,7 +6,7 @@
 @contact: wenruichn@gmail.com
 @time: 2019-08-01 00:44
 公众号：AI成长社
-知乎：https://www.zhihu.com/people/qlmx-61/activities
+知乎：https://www.zhihu.com/people/qlmx-61/columns
 """
 import pandas as pd
 import numpy as np
@@ -32,7 +32,7 @@
 train = data[data['label'] != -1]
 test = data[data['label'] == -1]
 
-## Clean up the memory
+##Clean up the memory
 del data, train_data, test_data
 gc.collect()
 
diff --git a/code/lgb/multi_class.py b/code/lgb/multi_class.py
index 075595c..b1eb23b 100644
--- a/code/lgb/multi_class.py
+++ b/code/lgb/multi_class.py
@@ -6,7 +6,7 @@
 @contact: wenruichn@gmail.com
 @time: 2019-07-29 08:56
 公众号：AI成长社
-知乎：https://www.zhihu.com/people/qlmx-61/activities
+知乎：https://www.zhihu.com/people/qlmx-61/columns
 """
 import pandas as pd
 import numpy as np
@@ -32,7 +32,7 @@
 train = data[data['label'] != -1]
 test = data[data['label'] == -1]
 
-## Clean up the memory
+##Clean up the memory
 del data, train_data, test_data
 gc.collect()
 
diff --git a/code/lgb/multi_class_custom_loss.py b/code/lgb/multi_class_custom_loss.py
index e453b9c..33bcad1 100644
--- a/code/lgb/multi_class_custom_loss.py
+++ b/code/lgb/multi_class_custom_loss.py
@@ -6,7 +6,7 @@
 @contact: wenruichn@gmail.com
 @time: 2019-07-31 22:00
 公众号：AI成长社
-知乎：https://www.zhihu.com/people/qlmx-61/activities
+知乎：https://www.zhihu.com/people/qlmx-61/columns
 """
 import pandas as pd
 import numpy as np
diff --git a/code/lgb/multi_class_weight_loss.py b/code/lgb/multi_class_weight_loss.py
index 5c65806..66007c5 100644
--- a/code/lgb/multi_class_weight_loss.py
+++ b/code/lgb/multi_class_weight_loss.py
@@ -6,7 +6,7 @@
 @contact: wenruichn@gmail.com
 @time: 2019-07-31 20:30
 公众号：AI成长社
-知乎：https://www.zhihu.com/people/qlmx-61/activities
+知乎：https://www.zhihu.com/people/qlmx-61/columns
 """
 import pandas as pd
 import numpy as np
diff --git a/code/lgb/regression.py b/code/lgb/regression.py
index 65a3f7f..6db1870 100644
--- a/code/lgb/regression.py
+++ b/code/lgb/regression.py
@@ -6,7 +6,7 @@
 @contact: wenruichn@gmail.com
 @time: 2019-07-31 20:02
 公众号：AI成长社
-知乎：https://www.zhihu.com/people/qlmx-61/activities
+知乎：https://www.zhihu.com/people/qlmx-61/columns
 """
 import pandas as pd
 import numpy as np
diff --git a/code/pytorch/binary_class.py b/code/pytorch/binary_class.py
new file mode 100644
index 0000000..af4b964
--- /dev/null
+++ b/code/pytorch/binary_class.py
@@ -0,0 +1,196 @@
+#!usr/bin/env python  
+#-*- coding:utf-8 _*- 
+"""
+@version: python3.6
+@author: QLMX
+@contact: wenruichn@gmail.com
+@time: 2019-08-07 00:44
+公众号：AI成长社
+知乎：https://www.zhihu.com/people/qlmx-61/columns
+"""
+import pandas as pd
+import numpy as np
+from sklearn.preprocessing import LabelEncoder, StandardScaler
+from sklearn.model_selection import KFold
+import gc
+import time
+
+import torch
+import torch.nn.functional as F
+from torch.utils.data import TensorDataset, DataLoader
+from sklearn.metrics import roc_auc_score
+from torch.autograd import Variable
+
+
+## load data
+train_data = pd.read_csv('../../data/train.csv')
+test_data = pd.read_csv('../../data/test.csv')
+epochs = 5
+batch_size = 1024
+classes = 1
+learning_rate = 0.01
+
+
+## category feature one_hot
+test_data['label'] = -1
+data = pd.concat([train_data, test_data])
+cate_feature = ['gender', 'cell_province', 'id_province', 'id_city', 'rate', 'term']
+for item in cate_feature:
+    data[item] = LabelEncoder().fit_transform(data[item])
+    item_dummies = pd.get_dummies(data[item])
+    item_dummies.columns = [item + str(i + 1) for i in range(item_dummies.shape[1])]
+    data = pd.concat([data, item_dummies], axis=1)
+data.drop(cate_feature,axis=1,inplace=True)
+
+train = data[data['label'] != -1]
+test = data[data['label'] == -1]
+
+##Clean up the memory
+del data, train_data, test_data
+gc.collect()
+
+## get train feature
+del_feature = ['auditing_date', 'due_date', 'label']
+features = [i for i in train.columns if i not in del_feature]
+
+
+## Convert the label to two categories
+train['label'] = train['label'].apply(lambda x: 1 if x==32 else 0)
+train_x = train[features]
+train_y = train['label'].values
+test = test[features]
+
+## Fill missing value
+for i in train_x.columns:
+    # print(i, train_x[i].isnull().sum(), test[i].isnull().sum())
+    if train_x[i].isnull().sum() != 0:
+        train_x[i] = train_x[i].fillna(-1)
+        test[i] = test[i].fillna(-1)
+
+## normalized
+scaler = StandardScaler()
+train_X = scaler.fit_transform(train_x)
+test_X = scaler.transform(test)
+
+class MLP(torch.nn.Module):
+    def __init__(self, n_feature, n_hidden, n_output, dropout=0.5):
+        super(MLP, self).__init__()
+        self.dropout = torch.nn.Dropout(dropout)
+
+        self.hidden_1 = torch.nn.Linear(n_feature, n_hidden)  # hidden layer
+        self.bn1 = torch.nn.BatchNorm1d(n_hidden)
+
+        self.hidden_2 = torch.nn.Linear(n_hidden, n_hidden//2)
+        self.bn2 = torch.nn.BatchNorm1d(n_hidden//2)
+
+        self.hidden_3 = torch.nn.Linear(n_hidden//2, n_hidden//4)  # hidden layer
+        self.bn3 = torch.nn.BatchNorm1d(n_hidden//4)
+
+        self.hidden_4 = torch.nn.Linear(n_hidden // 4, n_hidden // 8)  # hidden layer
+        self.bn4 = torch.nn.BatchNorm1d(n_hidden // 8)
+
+        self.out = torch.nn.Linear(n_hidden//8, n_output)  # output layer
+
+    def forward(self, x):
+        x = F.relu(self.hidden_1(x))  # activation function for hidden layer
+        x = self.dropout(self.bn1(x))
+        x = F.relu(self.hidden_2(x))  # activation function for hidden layer
+        x = self.dropout(self.bn2(x))
+        x = F.relu(self.hidden_3(x))  # activation function for hidden layer
+        x = self.dropout(self.bn3(x))
+        x = F.relu(self.hidden_4(x))  # activation function for hidden layer
+        x = self.dropout(self.bn4(x))
+        x = self.out(x)
+        return x
+
+def sigmoid(x):
+    return 1 / (1 + np.exp(-x))
+
+
+folds = KFold(n_splits=5, shuffle=True, random_state=2019)
+NN_predictions = np.zeros((test_X.shape[0], ))
+oof_preds = np.zeros((train_X.shape[0], ))
+
+x_test = np.array(test_X)
+x_test = torch.tensor(x_test, dtype=torch.float)
+if torch.cuda.is_available():
+    x_test = x_test.cuda()
+test = TensorDataset(x_test)
+test_loader = DataLoader(test, batch_size=batch_size, shuffle=False)
+
+avg_losses_f = []
+avg_val_losses_f = []
+
+for fold_, (trn_, val_) in enumerate(folds.split(train_x)):
+    print("fold {}".format(fold_ + 1))
+
+    x_train = Variable(torch.Tensor(train_X[trn_.astype(int)]))
+    y_train = Variable(torch.Tensor(train_y[trn_.astype(int), np.newaxis]))
+
+    x_valid = Variable(torch.Tensor(train_X[val_.astype(int)]))
+    y_valid = Variable(torch.Tensor(train_y[val_.astype(int), np.newaxis]))
+
+    model = MLP(x_train.shape[1], 512, classes, dropout=0.3)
+
+    if torch.cuda.is_available():
+        x_train, y_train = x_train.cuda(), y_train.cuda()
+        x_valid, y_valid = x_valid.cuda(), y_valid.cuda()
+        model = model.cuda()
+
+    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=1e-5)
+    loss_fn = torch.nn.BCEWithLogitsLoss()  # Combined with the sigmoid
+
+    train = TensorDataset(x_train, y_train)
+    valid = TensorDataset(x_valid, y_valid)
+
+    train_loader = DataLoader(train, batch_size=batch_size, shuffle=True)
+    valid_loader = DataLoader(valid, batch_size=batch_size, shuffle=False)
+
+    for epoch in range(epochs):
+        start_time = time.time()
+        model.train()
+        avg_loss = 0.
+        for i, (x_batch, y_batch) in enumerate(train_loader):
+            y_pred = model(x_batch)
+
+            loss = loss_fn(y_pred, y_batch)
+            optimizer.zero_grad()       # clear gradients for next train
+            loss.backward()             # -> accumulates the gradient (by addition) for each parameter
+            optimizer.step()            # -> update weights and biases
+            avg_loss += loss.item() / len(train_loader)
+            # avg_auc += round(roc_auc_score(y_batch.cpu(),y_pred.detach().cpu()),4) / len(train_loader)
+        model.eval()
+
+        valid_preds_fold = np.zeros((x_valid.size(0)))
+        test_preds_fold = np.zeros((len(test_X)))
+
+        avg_val_loss = 0.
+        # avg_val_auc = 0.
+        for i, (x_batch, y_batch) in enumerate(valid_loader):
+            y_pred = model(x_batch).detach()
+
+            # avg_val_auc += round(roc_auc_score(y_batch.cpu(),sigmoid(y_pred.cpu().numpy())[:, 0]),4) / len(valid_loader)
+            avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)
+            valid_preds_fold[i * batch_size:(i + 1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
+
+        elapsed_time = time.time() - start_time
+        print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(epoch + 1, epochs, avg_loss, avg_val_loss, elapsed_time))
+
+    avg_losses_f.append(avg_loss)
+    avg_val_losses_f.append(avg_val_loss)
+
+    for i, (x_batch,) in enumerate(test_loader):
+        y_pred = model(x_batch).detach()
+
+        test_preds_fold[i * batch_size:(i + 1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
+
+    oof_preds[val_] = valid_preds_fold
+    NN_predictions += test_preds_fold / folds.n_splits
+
+auc = round(roc_auc_score(train_y, oof_preds), 4)
+print('All \t loss={:.4f} \t val_loss={:.4f} \t auc={:.4f}'.format(np.average(avg_losses_f), np.average(avg_val_losses_f), auc))
+
+threshold = 0.5
+result = []
+for pred in NN_predictions:
+    result.append(1 if pred > threshold else 0)
\ No newline at end of file
diff --git a/code/pytorch/multi_class.py b/code/pytorch/multi_class.py
new file mode 100644
index 0000000..7a62088
--- /dev/null
+++ b/code/pytorch/multi_class.py
@@ -0,0 +1,187 @@
+#!usr/bin/env python  
+#-*- coding:utf-8 _*- 
+"""
+@version: python3.6
+@author: QLMX
+@contact: wenruichn@gmail.com
+@time: 2019-08-08 09:38
+公众号：AI成长社
+知乎：https://www.zhihu.com/people/qlmx-61/columns
+"""
+import pandas as pd
+import numpy as np
+from sklearn.preprocessing import LabelEncoder, StandardScaler
+from sklearn.model_selection import KFold
+import gc
+import time
+
+import torch
+import torch.nn.functional as F
+from torch.utils.data import TensorDataset, DataLoader
+from sklearn.metrics import roc_auc_score
+from torch.autograd import Variable
+
+
+## load data
+train_data = pd.read_csv('../../data/train.csv')
+test_data = pd.read_csv('../../data/test.csv')
+epochs = 10
+batch_size = 1024
+classes = 33
+learning_rate = 0.01
+
+
+## category feature one_hot
+test_data['label'] = -1
+data = pd.concat([train_data, test_data])
+cate_feature = ['gender', 'cell_province', 'id_province', 'id_city', 'rate', 'term']
+for item in cate_feature:
+    data[item] = LabelEncoder().fit_transform(data[item])
+    item_dummies = pd.get_dummies(data[item])
+    item_dummies.columns = [item + str(i + 1) for i in range(item_dummies.shape[1])]
+    data = pd.concat([data, item_dummies], axis=1)
+data.drop(cate_feature,axis=1,inplace=True)
+
+train = data[data['label'] != -1]
+test = data[data['label'] == -1]
+
+##Clean up the memory
+del data, train_data, test_data
+gc.collect()
+
+## get train feature
+del_feature = ['auditing_date', 'due_date', 'label']
+features = [i for i in train.columns if i not in del_feature]
+
+
+## Convert the label to two categories
+train_x = train[features]
+train_y = train['label'].astype(int).values
+test = test[features]
+
+## Fill missing value
+for i in train_x.columns:
+    # print(i, train_x[i].isnull().sum(), test[i].isnull().sum())
+    if train_x[i].isnull().sum() != 0:
+        train_x[i] = train_x[i].fillna(-1)
+        test[i] = test[i].fillna(-1)
+
+## normalized
+scaler = StandardScaler()
+train_X = scaler.fit_transform(train_x)
+test_X = scaler.transform(test)
+
+class MLP(torch.nn.Module):
+    def __init__(self, n_feature, n_hidden, n_output, dropout=0.5):
+        super(MLP, self).__init__()
+        self.dropout = torch.nn.Dropout(dropout)
+
+        self.hidden_1 = torch.nn.Linear(n_feature, n_hidden)  # hidden layer
+        self.bn1 = torch.nn.BatchNorm1d(n_hidden)
+
+        self.hidden_2 = torch.nn.Linear(n_hidden, n_hidden//2)
+        self.bn2 = torch.nn.BatchNorm1d(n_hidden//2)
+
+        self.hidden_3 = torch.nn.Linear(n_hidden//2, n_hidden//4)  # hidden layer
+        self.bn3 = torch.nn.BatchNorm1d(n_hidden//4)
+
+        self.hidden_4 = torch.nn.Linear(n_hidden // 4, n_hidden // 8)  # hidden layer
+        self.bn4 = torch.nn.BatchNorm1d(n_hidden // 8)
+
+        self.out = torch.nn.Linear(n_hidden//8, n_output)  # output layer
+
+    def forward(self, x):
+        x = F.relu(self.hidden_1(x))  # activation function for hidden layer
+        x = self.dropout(self.bn1(x))
+        x = F.relu(self.hidden_2(x))  # activation function for hidden layer
+        x = self.dropout(self.bn2(x))
+        x = F.relu(self.hidden_3(x))  # activation function for hidden layer
+        x = self.dropout(self.bn3(x))
+        x = F.relu(self.hidden_4(x))  # activation function for hidden layer
+        x = self.dropout(self.bn4(x))
+        x = self.out(x)
+        return x
+
+def softmax(x):
+    """Compute the softmax of vector x."""
+    return np.exp(x) / np.sum(np.exp(x), axis=1).reshape(-1, 1)
+
+
+folds = KFold(n_splits=5, shuffle=True, random_state=2019)
+NN_predictions = np.zeros((test_X.shape[0], classes))
+oof_preds = np.zeros((train_X.shape[0], classes))
+
+x_test = np.array(test_X)
+x_test = torch.tensor(x_test, dtype=torch.float)
+if torch.cuda.is_available():
+    x_test = x_test.cuda()
+test = TensorDataset(x_test)
+test_loader = DataLoader(test, batch_size=batch_size, shuffle=False)
+
+avg_losses_f = []
+avg_val_losses_f = []
+
+for fold_, (trn_, val_) in enumerate(folds.split(train_x)):
+    print("fold {}".format(fold_ + 1))
+
+    x_train, y_train = torch.Tensor(train_X[trn_]), torch.LongTensor(train_y[trn_])
+    x_valid, y_valid = torch.Tensor(train_X[val_]), torch.LongTensor(train_y[val_])
+
+    model = MLP(x_train.shape[1], 512, classes, dropout=0.3)
+
+    if torch.cuda.is_available():
+        x_train, y_train = x_train.cuda(), y_train.cuda()
+        x_valid, y_valid = x_valid.cuda(), y_valid.cuda()
+        model = model.cuda()
+
+    x_train, y_train = Variable(x_train), Variable(y_train)
+    x_valid, y_valid = Variable(x_valid), Variable(y_valid)
+
+    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=1e-5)
+    loss_fn = torch.nn.CrossEntropyLoss()  # Combined with the softmax and the target label is NOT an one-hotted
+
+    train = TensorDataset(x_train, y_train)
+    valid = TensorDataset(x_valid, y_valid)
+
+    train_loader = DataLoader(train, batch_size=batch_size, shuffle=True)
+    valid_loader = DataLoader(valid, batch_size=batch_size, shuffle=False)
+
+    for epoch in range(epochs):
+        start_time = time.time()
+        model.train()
+        avg_loss = 0.
+        for i, (x_batch, y_batch) in enumerate(train_loader):
+            y_pred = model(x_batch)
+
+            loss = loss_fn(y_pred, y_batch)
+            optimizer.zero_grad()       # clear gradients for next train
+            loss.backward()             # -> accumulates the gradient (by addition) for each parameter
+            optimizer.step()            # -> update weights and biases
+            avg_loss += loss.item() / len(train_loader)
+            # avg_auc += round(roc_auc_score(y_batch.cpu(),y_pred.detach().cpu()),4) / len(train_loader)
+        model.eval()
+
+        valid_preds_fold = np.zeros((x_valid.size(0), classes))
+        test_preds_fold = np.zeros((len(test_X), classes))
+
+        avg_val_loss = 0.
+        for i, (x_batch, y_batch) in enumerate(valid_loader):
+            y_pred = model(x_batch).detach()
+
+            avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)
+            valid_preds_fold[i * batch_size:(i + 1) * batch_size] = softmax(y_pred.cpu().numpy())
+
+        elapsed_time = time.time() - start_time
+        print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(epoch + 1, epochs, avg_loss, avg_val_loss, elapsed_time))
+
+    avg_losses_f.append(avg_loss)
+    avg_val_losses_f.append(avg_val_loss)
+
+    for i, (x_batch,) in enumerate(test_loader):
+        y_pred = model(x_batch).detach()
+        test_preds_fold[i * batch_size:(i + 1) * batch_size] = softmax(y_pred.cpu().numpy())
+
+    oof_preds[val_] = valid_preds_fold
+    NN_predictions += test_preds_fold / folds.n_splits
+
+result = np.argmax(NN_predictions, axis=1)
\ No newline at end of file
diff --git a/code/pytorch/regression.py b/code/pytorch/regression.py
new file mode 100644
index 0000000..548c6f9
--- /dev/null
+++ b/code/pytorch/regression.py
@@ -0,0 +1,188 @@
+#!usr/bin/env python  
+#-*- coding:utf-8 _*- 
+"""
+@version: python3.6
+@author: QLMX
+@contact: wenruichn@gmail.com
+@time: 2019-08-08 13:35
+公众号：AI成长社
+知乎：https://www.zhihu.com/people/qlmx-61/columns
+"""
+import pandas as pd
+import numpy as np
+from sklearn.preprocessing import LabelEncoder, StandardScaler
+from sklearn.model_selection import KFold
+import gc
+import time
+
+import torch
+import torch.nn.functional as F
+from torch.utils.data import TensorDataset, DataLoader
+from sklearn.metrics import roc_auc_score
+from torch.autograd import Variable
+
+
+## load data
+train_data = pd.read_csv('../../data/train.csv')
+test_data = pd.read_csv('../../data/test.csv')
+epochs = 20
+batch_size = 1024
+learning_rate = 0.002
+
+
+## category feature one_hot
+test_data['label'] = -1
+data = pd.concat([train_data, test_data])
+cate_feature = ['gender', 'cell_province', 'id_province', 'id_city', 'rate', 'term']
+for item in cate_feature:
+    data[item] = LabelEncoder().fit_transform(data[item])
+    item_dummies = pd.get_dummies(data[item])
+    item_dummies.columns = [item + str(i + 1) for i in range(item_dummies.shape[1])]
+    data = pd.concat([data, item_dummies], axis=1)
+data.drop(cate_feature,axis=1,inplace=True)
+
+train = data[data['label'] != -1]
+test = data[data['label'] == -1]
+
+##Clean up the memory
+del data, train_data, test_data
+gc.collect()
+
+## get train feature
+del_feature = ['auditing_date', 'due_date', 'label']
+features = [i for i in train.columns if i not in del_feature]
+
+
+## Convert the label to two categories
+train_x = train[features]
+train_y = train['label'].values
+test = test[features]
+
+## Fill missing value
+for i in train_x.columns:
+    # print(i, train_x[i].isnull().sum(), test[i].isnull().sum())
+    if train_x[i].isnull().sum() != 0:
+        train_x[i] = train_x[i].fillna(-1)
+        test[i] = test[i].fillna(-1)
+
+## normalized
+scaler = StandardScaler()
+train_X = scaler.fit_transform(train_x)
+test_X = scaler.transform(test)
+
+class MLP(torch.nn.Module):
+    def __init__(self, n_feature, n_hidden, n_output, dropout=0.5):
+        super(MLP, self).__init__()
+        self.dropout = torch.nn.Dropout(dropout)
+
+        self.hidden_1 = torch.nn.Linear(n_feature, n_hidden)  # hidden layer
+        self.bn1 = torch.nn.BatchNorm1d(n_hidden)
+
+        self.hidden_2 = torch.nn.Linear(n_hidden, n_hidden//2)
+        self.bn2 = torch.nn.BatchNorm1d(n_hidden//2)
+
+        self.hidden_3 = torch.nn.Linear(n_hidden//2, n_hidden//4)  # hidden layer
+        self.bn3 = torch.nn.BatchNorm1d(n_hidden//4)
+
+        self.hidden_4 = torch.nn.Linear(n_hidden // 4, n_hidden // 8)  # hidden layer
+        self.bn4 = torch.nn.BatchNorm1d(n_hidden // 8)
+
+        self.out = torch.nn.Linear(n_hidden//8, n_output)  # output layer
+
+    def forward(self, x):
+        x = F.relu(self.hidden_1(x))  # activation function for hidden layer
+        x = self.dropout(self.bn1(x))
+        x = F.relu(self.hidden_2(x))  # activation function for hidden layer
+        x = self.dropout(self.bn2(x))
+        x = F.relu(self.hidden_3(x))  # activation function for hidden layer
+        x = self.dropout(self.bn3(x))
+        x = F.relu(self.hidden_4(x))  # activation function for hidden layer
+        x = self.dropout(self.bn4(x))
+        x = self.out(x)
+        return x
+
+folds = KFold(n_splits=5, shuffle=True, random_state=2019)
+NN_predictions = np.zeros((test_X.shape[0], 1))
+oof_preds = np.zeros((train_X.shape[0], 1))
+
+x_test = np.array(test_X)
+x_test = torch.tensor(x_test, dtype=torch.float)
+if torch.cuda.is_available():
+    x_test = x_test.cuda()
+test = TensorDataset(x_test)
+test_loader = DataLoader(test, batch_size=batch_size, shuffle=False)
+
+avg_losses_f = []
+avg_val_losses_f = []
+train_y = np.log1p(train_y)   # Data smoothing
+
+
+for fold_, (trn_, val_) in enumerate(folds.split(train_x)):
+    print("fold {}".format(fold_ + 1))
+
+    x_train = Variable(torch.Tensor(train_X[trn_.astype(int)]))
+    y_train = Variable(torch.Tensor(train_y[trn_.astype(int), np.newaxis]))
+
+    x_valid = Variable(torch.Tensor(train_X[val_.astype(int)]))
+    y_valid = Variable(torch.Tensor(train_y[val_.astype(int), np.newaxis]))
+
+    model = MLP(x_train.shape[1], 512, 1, dropout=0.3)
+
+    if torch.cuda.is_available():
+        x_train, y_train = x_train.cuda(), y_train.cuda()
+        x_valid, y_valid = x_valid.cuda(), y_valid.cuda()
+        model = model.cuda()
+
+    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=1e-5)
+    loss_fn = torch.nn.L1Loss()
+
+    train = TensorDataset(x_train, y_train)
+    valid = TensorDataset(x_valid, y_valid)
+
+    train_loader = DataLoader(train, batch_size=batch_size, shuffle=True)
+    valid_loader = DataLoader(valid, batch_size=batch_size, shuffle=False)
+
+    for epoch in range(epochs):
+        start_time = time.time()
+        model.train()
+        avg_loss = 0.
+        for i, (x_batch, y_batch) in enumerate(train_loader):
+            y_pred = model(x_batch)
+
+            loss = loss_fn(y_pred, y_batch)
+            optimizer.zero_grad()       # clear gradients for next train
+            loss.backward()             # -> accumulates the gradient (by addition) for each parameter
+            optimizer.step()            # -> update weights and biases
+            avg_loss += loss.item() / len(train_loader)
+            # avg_auc += round(roc_auc_score(y_batch.cpu(),y_pred.detach().cpu()),4) / len(train_loader)
+        model.eval()
+
+        valid_preds_fold = np.zeros((x_valid.size(0), 1))
+        test_preds_fold = np.zeros((len(test_X), 1))
+
+        avg_val_loss = 0.
+        # avg_val_auc = 0.
+        for i, (x_batch, y_batch) in enumerate(valid_loader):
+            y_pred = model(x_batch).detach()
+
+            # avg_val_auc += round(roc_auc_score(y_batch.cpu(),sigmoid(y_pred.cpu().numpy())[:, 0]),4) / len(valid_loader)
+            avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)
+            tt = y_pred.cpu().numpy()
+            valid_preds_fold[i * batch_size:(i + 1) * batch_size] = y_pred.cpu().numpy()
+
+        elapsed_time = time.time() - start_time
+        print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(epoch + 1, epochs, avg_loss, avg_val_loss, elapsed_time))
+
+    avg_losses_f.append(avg_loss)
+    avg_val_losses_f.append(avg_val_loss)
+
+    for i, (x_batch,) in enumerate(test_loader):
+        y_pred = model(x_batch).detach()
+
+        test_preds_fold[i * batch_size:(i + 1) * batch_size] = y_pred.cpu().numpy()
+
+    oof_preds[val_] = valid_preds_fold
+    NN_predictions += test_preds_fold / folds.n_splits
+
+result = NN_predictions[:, 0]
+result = np.expm1(result)  #reduction
diff --git a/code/xgb/binary_class.py b/code/xgb/binary_class.py
index 80a99d8..e5e444a 100644
--- a/code/xgb/binary_class.py
+++ b/code/xgb/binary_class.py
@@ -6,7 +6,7 @@
 @contact: wenruichn@gmail.com
 @time: 2019-08-01 01:30
 公众号：AI成长社
-知乎：https://www.zhihu.com/people/qlmx-61/activities
+知乎：https://www.zhihu.com/people/qlmx-61/columns
 """
 import pandas as pd
 import numpy as np
@@ -36,7 +36,7 @@
 train = data[data['label'] != -1]
 test = data[data['label'] == -1]
 
-#del dataframe
+##Clean up the memory
 del data, train_data, test_data
 gc.collect()
 
diff --git a/code/xgb/multi_class.py b/code/xgb/multi_class.py
index 0c7dc30..1737adc 100644
--- a/code/xgb/multi_class.py
+++ b/code/xgb/multi_class.py
@@ -6,7 +6,7 @@
 @contact: wenruichn@gmail.com
 @time: 2019-08-01 01:32
 公众号：AI成长社
-知乎：https://www.zhihu.com/people/qlmx-61/activities
+知乎：https://www.zhihu.com/people/qlmx-61/columns
 """
 import pandas as pd
 import numpy as np
@@ -36,7 +36,7 @@
 train = data[data['label'] != -1]
 test = data[data['label'] == -1]
 
-#del dataframe
+##Clean up the memory
 del data, train_data, test_data
 gc.collect()
 
diff --git a/code/xgb/regression.py b/code/xgb/regression.py
index e87df3d..24798e0 100644
--- a/code/xgb/regression.py
+++ b/code/xgb/regression.py
@@ -6,7 +6,7 @@
 @contact: wenruichn@gmail.com
 @time: 2019-08-01 01:33
 公众号：AI成长社
-知乎：https://www.zhihu.com/people/qlmx-61/activities
+知乎：https://www.zhihu.com/people/qlmx-61/columns
 """
 import pandas as pd
 import numpy as np
@@ -37,7 +37,7 @@
 train = data[data['label'] != -1]
 test = data[data['label'] == -1]
 
-#del dataframe
+##Clean up the memory
 del data, train_data, test_data
 gc.collect()
 
diff --git a/requirements.txt b/requirements.txt
index 035f4a1..215b24b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,7 @@
 pandas
 numpy
 matplotlib
+pytorch
 sklearn
 keras==2.2.4
 seaborn