deploy_test.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3",
   "language": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "2021-01-12 15:42:43,404 featuretools - WARNING    Featuretools failed to load plugin nlp_primitives from library nlp_primitives. For a full stack trace, set logging to debug.\n"
     ]
    }
   ],
   "source": [
    "from semi_auto_ml import AutoCreate,AutoSelect,DataCheck,ModelSelect\n",
    "import featuretools as ft\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import sklearn as sk\n",
    "from itertools import chain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = r'E:/新建文件夹/'\n",
    "target_df = pd.read_csv(path+'target_df.csv').drop('Unnamed: 0',axis=1)\n",
    "fact1_df = pd.read_csv(path+'fact1_df.csv').drop('Unnamed: 0',axis=1)\n",
    "dim1_df = pd.read_csv(path+'dim1_df.csv').drop('Unnamed: 0',axis=1)\n",
    "dim2_df = pd.read_csv(path+'dim2_df.csv').drop('Unnamed: 0',axis=1)"
   ]
  },
  {
   "source": [
    "## 原始数据清洗\n",
    "#### 使用Datacheck"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 通过checks 发现异常值，当进行预测时首先通过 checks model 如果标记为-1 则为异常值 不进入模型进行预测"
   ]
  },
  {
   "source": [
    "## 特征生成"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "auto_c = AutoCreate('ISL')\n",
    "auto_c.create_entity('target',target_df,index ='key_id')\n",
    "auto_c.create_entity('fact',fact1_df,index ='apply_id',make_index=True)\n",
    "auto_c.create_entity('dim1',dim1_df,index ='dim1_id',make_index=True)\n",
    "auto_c.create_entity('dim2',dim2_df,index ='dim2_id',make_index=True)\n",
    "relationships = ['target.key_id','fact.key_id','fact.work_id','dim1.work_id',\n",
    "                    'fact.customer_id','dim2.customer_id',]\n",
    "auto_c.add_relation(relationships)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "2021-01-08 17:39:29,001 featuretools - WARNING    Attempting to add feature <Feature: COUNT(fact)> which is already present. This is likely a bug.\n",
      "2021-01-08 17:39:29,001 featuretools - WARNING    Attempting to add feature <Feature: COUNT(fact)> which is already present. This is likely a bug.\n",
      "2021-01-08 17:39:29,018 featuretools - WARNING    Attempting to add feature <Feature: COUNT(dim1)> which is already present. This is likely a bug.\n",
      "2021-01-08 17:39:29,018 featuretools - WARNING    Attempting to add feature <Feature: COUNT(dim1)> which is already present. This is likely a bug.\n",
      "2021-01-08 17:39:29,020 featuretools - WARNING    Attempting to add feature <Feature: COUNT(dim2)> which is already present. This is likely a bug.\n",
      "2021-01-08 17:39:29,021 featuretools - WARNING    Attempting to add feature <Feature: COUNT(dim2)> which is already present. This is likely a bug.\n"
     ]
    }
   ],
   "source": [
    "#得到计算后的特征，feature_f作为model\n",
    "feature_m,feature_f=auto_c.make_features(target_entity=\"target\",drop_contains=['_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_y=feature_m['is_loan']\n",
    "train_matrix,train_def = auto_c.remove_features(['is_loan'],feature_m,feature_f)\n",
    "num_matrix,num_def = auto_c.clean_features(train_matrix,train_def)\n",
    "num_matrix,num_def = auto_c.remove_features(['MODE(dim2.cust_category)','MODE(dim2.issue)','MODE(dim2.sex)','COUNT(fact)','COUNT(dim1)','COUNT(dim2)'],num_matrix,features_enc=num_def)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_matrix,num_def=auto_c.get_final_data(num_matrix,num_def)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#保存特征生成模型\n",
    "auto_c.deploy_features_create(num_def,'make_features.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#对空值和负值进行处理\n",
    "from sklearn.preprocessing import FunctionTransformer\n",
    "def transforme(X,fill_value=0,map_func=abs):\n",
    "    X=X.astype('float').fillna(fill_value).applymap(map_func)\n",
    "    return X\n",
    "sp_clf = FunctionTransformer(transforme,kw_args={'fill_value':0,'map_func':abs})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_matrix=sp_clf.transform(num_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from semi_auto_ml.utils.extract_funcs import load_sk_model,save_sk_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_sk_model(sp_clf,'model_file/f1.joblib')"
   ]
  },
  {
   "source": [
    "## 特征选择"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "auto_s = AutoSelect(num_matrix,train_y)\n",
    "sk_fm = auto_s.sk_feature_importances(0.99)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "orientation": "h",
         "type": "bar",
         "x": [
          0.43078416573250117,
          0.3706622163855281,
          0.09130993764084892,
          0.05521377546026175,
          0.015124385502662435,
          0.00983620939001557,
          0.009199788952058871,
          0.006899506354956414,
          0.005611259443764626,
          0.00471789339224308,
          0.0005657364273536248,
          0.000058331502550677126,
          0.000011412329363764764,
          0.000004042067433039893,
          6.730079083555877e-7
         ],
         "y": [
          "SUM(dim2.fact.apply_amount)",
          "MAX(dim2.regis_date_key)",
          "STD(dim2.fact.apply_amount)",
          "MAX(fact.SUM(dim1.salary))",
          "SUM(dim1.salary)",
          "MAX(fact.apply_amount)",
          "MAX(dim1.fact.apply_amount)",
          "MAX(dim1.salary)",
          "STD(dim1.fact.apply_amount)",
          "MAX(fact.MEAN(dim2.regis_date_key))",
          "MAX(fact.MEAN(dim1.salary))",
          "SUM(dim2.age)",
          "SUM(dim2.regis_month)",
          "MAX(fact.COUNT(dim1))",
          "MAX(dim2.age)"
         ]
        }
       ],
       "layout": {
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "title": {
         "text": "Feature Importances"
        },
        "xaxis": {
         "range": [
          0,
          1
         ],
         "title": {
          "text": "Normalized Importance"
         }
        },
        "yaxis": {
         "categoryorder": "total ascending"
        }
       }
      }
     },
     "metadata": {}
    }
   ],
   "source": [
    "auto_s.plotly_feature_importances(sk_fm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "features_to_drop = set(chain(*auto_s.removed_features))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def drop_columns(X,remove_features):\n",
    "    kf = [item for item in X.columns if item not in remove_features]\n",
    "    return X[kf]\n",
    "rm1_clf = FunctionTransformer(drop_columns,kw_args={'remove_features':features_to_drop})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_matrix = rm1_clf.transform(num_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_sk_model(rm1_clf,'model_file/f2.joblib')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "        MAX(fact.apply_amount)  MAX(dim1.salary)  SUM(dim1.salary)  \\\n",
       "key_id                                                               \n",
       "1.0                   100000.0           20000.0           20000.0   \n",
       "2.0                    20000.0            6000.0            6000.0   \n",
       "4.0                   150000.0           15000.0           15000.0   \n",
       "6.0                    70000.0           10500.0           10500.0   \n",
       "7.0                    50000.0            6000.0            6000.0   \n",
       "\n",
       "        MAX(dim2.regis_date_key)  MAX(fact.SUM(dim1.salary))  \\\n",
       "key_id                                                         \n",
       "1.0                   20201020.0                      6300.0   \n",
       "2.0                   20200316.0                      7000.0   \n",
       "4.0                   20201126.0                         0.0   \n",
       "6.0                   20200710.0                      3500.0   \n",
       "7.0                   20201008.0                      4420.0   \n",
       "\n",
       "        MAX(dim1.fact.apply_amount)  STD(dim2.fact.apply_amount)  \\\n",
       "key_id                                                             \n",
       "1.0                        100000.0                          0.0   \n",
       "2.0                         20000.0                          0.0   \n",
       "4.0                        150000.0                          0.0   \n",
       "6.0                         70000.0                          0.0   \n",
       "7.0                         50000.0                          0.0   \n",
       "\n",
       "        SUM(dim2.fact.apply_amount)  \n",
       "key_id                               \n",
       "1.0                        100000.0  \n",
       "2.0                         20000.0  \n",
       "4.0                        150000.0  \n",
       "6.0                         70000.0  \n",
       "7.0                         50000.0  "
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>MAX(fact.apply_amount)</th>\n      <th>MAX(dim1.salary)</th>\n      <th>SUM(dim1.salary)</th>\n      <th>MAX(dim2.regis_date_key)</th>\n      <th>MAX(fact.SUM(dim1.salary))</th>\n      <th>MAX(dim1.fact.apply_amount)</th>\n      <th>STD(dim2.fact.apply_amount)</th>\n      <th>SUM(dim2.fact.apply_amount)</th>\n    </tr>\n    <tr>\n      <th>key_id</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>1.0</th>\n      <td>100000.0</td>\n      <td>20000.0</td>\n      <td>20000.0</td>\n      <td>20201020.0</td>\n      <td>6300.0</td>\n      <td>100000.0</td>\n      <td>0.0</td>\n      <td>100000.0</td>\n    </tr>\n    <tr>\n      <th>2.0</th>\n      <td>20000.0</td>\n      <td>6000.0</td>\n      <td>6000.0</td>\n      <td>20200316.0</td>\n      <td>7000.0</td>\n      <td>20000.0</td>\n      <td>0.0</td>\n      <td>20000.0</td>\n    </tr>\n    <tr>\n      <th>4.0</th>\n      <td>150000.0</td>\n      <td>15000.0</td>\n      <td>15000.0</td>\n      <td>20201126.0</td>\n      <td>0.0</td>\n      <td>150000.0</td>\n      <td>0.0</td>\n      <td>150000.0</td>\n    </tr>\n    <tr>\n      <th>6.0</th>\n      <td>70000.0</td>\n      <td>10500.0</td>\n      <td>10500.0</td>\n      <td>20200710.0</td>\n      <td>3500.0</td>\n      <td>70000.0</td>\n      <td>0.0</td>\n      <td>70000.0</td>\n    </tr>\n    <tr>\n      <th>7.0</th>\n      <td>50000.0</td>\n      <td>6000.0</td>\n      <td>6000.0</td>\n      <td>20201008.0</td>\n      <td>4420.0</td>\n      <td>50000.0</td>\n      <td>0.0</td>\n      <td>50000.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 18
    }
   ],
   "source": [
    "train_matrix.head()"
   ]
  },
  {
   "source": [
    "## 模型检索"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "`X` passed was not a DataTable. EvalML will try to convert the input as a Woodwork DataTable and types will be inferred. To control this behavior, please pass in a Woodwork DataTable instead.\n",
      "`y` passed was not a DataColumn. EvalML will try to convert the input as a Woodwork DataTable and types will be inferred. To control this behavior, please pass in a Woodwork DataTable instead.\n",
      "Generating pipelines to search over...\n",
      "*****************************\n",
      "* Beginning pipeline search *\n",
      "*****************************\n",
      "\n",
      "Optimizing for Log Loss Binary. \n",
      "Lower score is better.\n",
      "\n",
      "Searching up to 2 batches for a total of 14 pipelines. \n",
      "Allowed model families: extra_trees, random_forest, linear_model, catboost, decision_tree, xgboost, lightgbm\n",
      "\n",
      "Batch 1: (1/14) Mode Baseline Binary Classification P... Elapsed:00:00\n",
      "\tStarting cross validation\n",
      "\tFinished cross validation - mean Log Loss Binary: 7.045\n",
      "Batch 1: (2/14) Decision Tree Classifier w/ Imputer      Elapsed:00:00\n",
      "\tStarting cross validation\n",
      "\tFinished cross validation - mean Log Loss Binary: 0.490\n",
      "Batch 1: (3/14) LightGBM Classifier w/ Imputer           Elapsed:00:00\n",
      "\tStarting cross validation\n",
      "\tFinished cross validation - mean Log Loss Binary: 0.404\n",
      "Batch 1: (4/14) Extra Trees Classifier w/ Imputer        Elapsed:00:01\n",
      "\tStarting cross validation\n",
      "\tFinished cross validation - mean Log Loss Binary: 0.461\n",
      "Batch 1: (5/14) Elastic Net Classifier w/ Imputer + S... Elapsed:00:02\n",
      "\tStarting cross validation\n",
      "\tFinished cross validation - mean Log Loss Binary: 0.506\n",
      "Batch 1: (6/14) CatBoost Classifier w/ Imputer           Elapsed:00:02\n",
      "\tStarting cross validation\n",
      "\tFinished cross validation - mean Log Loss Binary: 0.587\n",
      "Batch 1: (7/14) XGBoost Classifier w/ Imputer            Elapsed:00:03\n",
      "\tStarting cross validation\n",
      "[17:41:21] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[17:41:21] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[17:41:21] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "\tFinished cross validation - mean Log Loss Binary: 0.400\n",
      "Batch 1: (8/14) Random Forest Classifier w/ Imputer      Elapsed:00:04\n",
      "\tStarting cross validation\n",
      "\tFinished cross validation - mean Log Loss Binary: 0.409\n",
      "Batch 1: (9/14) Logistic Regression Classifier w/ Imp... Elapsed:00:06\n",
      "\tStarting cross validation\n",
      "\tFinished cross validation - mean Log Loss Binary: 0.432\n",
      "Batch 2: (10/14) XGBoost Classifier w/ Imputer            Elapsed:00:08\n",
      "\tStarting cross validation\n",
      "[17:41:26] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[17:41:27] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[17:41:29] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "\tFinished cross validation - mean Log Loss Binary: 0.716\n",
      "Batch 2: (11/14) XGBoost Classifier w/ Imputer            Elapsed:00:12\n",
      "\tStarting cross validation\n",
      "[17:41:30] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[17:41:31] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[17:41:32] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "\tFinished cross validation - mean Log Loss Binary: 0.659\n",
      "Batch 2: (12/14) XGBoost Classifier w/ Imputer            Elapsed:00:16\n",
      "\tStarting cross validation\n",
      "[17:41:34] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[17:41:35] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[17:41:36] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "\tFinished cross validation - mean Log Loss Binary: 0.727\n",
      "Batch 2: (13/14) XGBoost Classifier w/ Imputer            Elapsed:00:20\n",
      "\tStarting cross validation\n",
      "[17:41:38] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[17:41:38] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[17:41:39] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "\tFinished cross validation - mean Log Loss Binary: 0.394\n",
      "Batch 2: (14/14) XGBoost Classifier w/ Imputer            Elapsed:00:21\n",
      "\tStarting cross validation\n",
      "[17:41:39] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[17:41:39] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[17:41:40] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "\tFinished cross validation - mean Log Loss Binary: 0.599\n",
      "\n",
      "Search finished after 00:22            \n",
      "Best pipeline: XGBoost Classifier w/ Imputer\n",
      "Best pipeline Log Loss Binary: 0.394142\n"
     ]
    }
   ],
   "source": [
    "ms = ModelSelect('binary',max_batches=2)\n",
    "ranking = ms.search(train_matrix,train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#获取最优模型\n",
    "bm = ms.auto_ml.best_pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "[17:41:40] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    }
   ],
   "source": [
    "pipline = bm.fit(train_matrix,train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipline.save('m3.joblib')"
   ]
  },
  {
   "source": [
    "## LOAD MODEL PREDICT"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get source data\n",
    "#完成model_deploy 部分将修改load model的代码\n",
    "auto_c2 = AutoCreate('PISL')\n",
    "auto_c2.create_entity('target',target_df.sample(n=200),index ='key_id')\n",
    "auto_c2.create_entity('fact',fact1_df,index ='apply_id',make_index=True)\n",
    "auto_c2.create_entity('dim1',dim1_df,index ='dim1_id',make_index=True)\n",
    "auto_c2.create_entity('dim2',dim2_df,index ='dim2_id',make_index=True)\n",
    "relationships = ['target.key_id','fact.key_id','fact.work_id','dim1.work_id',\n",
    "                    'fact.customer_id','dim2.customer_id',]\n",
    "auto_c2.add_relation(relationships)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_df_load = AutoCreate.load_features_create('make_features.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_m=auto_c2.make_features(target_entity=\"target\",features=feature_df_load)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "lsp_clf = load_sk_model('model_file/f1.joblib')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "lfs_clf = load_sk_model('model_file/f2.joblib')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "l_model = load_sk_model('m3.joblib')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_m = lsp_clf.transform(feature_m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_m = lfs_clf.transform(test_m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "已放款    169\n",
       "未放款     31\n",
       "dtype: int64"
      ]
     },
     "metadata": {},
     "execution_count": 35
    }
   ],
   "source": [
    "# 进行预测\n",
    "l_model.predict(test_m).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "# 根据path 取一次加载model,可以使用model_deploy 进行 模型保存与加载\n",
    "pipe = Pipeline([('f1', lsp_clf), ('f2', lfs_clf),('m1',l_model)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "0      已放款\n",
       "1      已放款\n",
       "2      已放款\n",
       "3      已放款\n",
       "4      已放款\n",
       "      ... \n",
       "195    已放款\n",
       "196    已放款\n",
       "197    未放款\n",
       "198    已放款\n",
       "199    已放款\n",
       "Length: 200, dtype: object"
      ]
     },
     "metadata": {},
     "execution_count": 37
    }
   ],
   "source": [
    "pipe.predict(feature_m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}