docs/source/Test_package_using_url.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Getting Started"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "## Data load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql.session import SparkSession\n",
    "\n",
    "spark = SparkSession.builder.getOrCreate()\n",
    "\n",
    "from pyspark import SparkFiles\n",
    "from pyspark.sql.functions import col\n",
    "\n",
    "url_X = 'https://raw.githubusercontent.com/seunghajeong/data_repo/master/simulation_data_x.csv'\n",
    "url_y = 'https://raw.githubusercontent.com/seunghajeong/data_repo/master/simulation_data_y.csv'\n",
    "spark.sparkContext.addFile(url_X)\n",
    "X_Sparkdataframe = spark.read.csv(SparkFiles.get(\"simulation_data_x.csv\"), header=True)\n",
    "X_Sparkdataframe = X_Sparkdataframe.select(*(col(c).cast(\"float\").alias(c) for c in X_Sparkdataframe.columns))\n",
    "X = X_Sparkdataframe.toPandas()\n",
    "spark.sparkContext.addFile(url_y)\n",
    "y_Sparkdataframe = spark.read.csv(SparkFiles.get(\"simulation_data_y.csv\"), header=True)\n",
    "y_Sparkdataframe = y_Sparkdataframe.select(*(col(c).cast(\"float\").alias(c) for c in y_Sparkdataframe.columns))\n",
    "y = y_Sparkdataframe.toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>V1</th>\n",
       "      <th>V2</th>\n",
       "      <th>V3</th>\n",
       "      <th>V4</th>\n",
       "      <th>V5</th>\n",
       "      <th>V6</th>\n",
       "      <th>V7</th>\n",
       "      <th>V8</th>\n",
       "      <th>V9</th>\n",
       "      <th>V10</th>\n",
       "      <th>...</th>\n",
       "      <th>V91</th>\n",
       "      <th>V92</th>\n",
       "      <th>V93</th>\n",
       "      <th>V94</th>\n",
       "      <th>V95</th>\n",
       "      <th>V96</th>\n",
       "      <th>V97</th>\n",
       "      <th>V98</th>\n",
       "      <th>V99</th>\n",
       "      <th>V100</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.101741</td>\n",
       "      <td>0.144909</td>\n",
       "      <td>0.235567</td>\n",
       "      <td>0.576186</td>\n",
       "      <td>0.299443</td>\n",
       "      <td>0.296395</td>\n",
       "      <td>0.902235</td>\n",
       "      <td>0.265811</td>\n",
       "      <td>0.420927</td>\n",
       "      <td>0.684045</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.058677</td>\n",
       "      <td>0.497420</td>\n",
       "      <td>-1.124986</td>\n",
       "      <td>0.338215</td>\n",
       "      <td>-0.942943</td>\n",
       "      <td>-1.257044</td>\n",
       "      <td>-0.531471</td>\n",
       "      <td>1.236317</td>\n",
       "      <td>0.405682</td>\n",
       "      <td>0.387636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.105054</td>\n",
       "      <td>-0.110128</td>\n",
       "      <td>-0.033311</td>\n",
       "      <td>-0.042925</td>\n",
       "      <td>-0.752605</td>\n",
       "      <td>-0.794815</td>\n",
       "      <td>-1.699739</td>\n",
       "      <td>-1.891533</td>\n",
       "      <td>-1.287547</td>\n",
       "      <td>-1.154547</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.614948</td>\n",
       "      <td>-1.337878</td>\n",
       "      <td>0.795742</td>\n",
       "      <td>1.101117</td>\n",
       "      <td>-0.920702</td>\n",
       "      <td>-0.098002</td>\n",
       "      <td>-0.269719</td>\n",
       "      <td>0.333092</td>\n",
       "      <td>-0.500367</td>\n",
       "      <td>1.340876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.478922</td>\n",
       "      <td>-0.058475</td>\n",
       "      <td>-0.620625</td>\n",
       "      <td>1.775435</td>\n",
       "      <td>0.935760</td>\n",
       "      <td>1.268173</td>\n",
       "      <td>-2.652118</td>\n",
       "      <td>-2.327299</td>\n",
       "      <td>-2.913926</td>\n",
       "      <td>-2.140256</td>\n",
       "      <td>...</td>\n",
       "      <td>0.081620</td>\n",
       "      <td>0.999657</td>\n",
       "      <td>-0.594758</td>\n",
       "      <td>0.057804</td>\n",
       "      <td>-1.259650</td>\n",
       "      <td>0.321864</td>\n",
       "      <td>0.992930</td>\n",
       "      <td>0.552269</td>\n",
       "      <td>1.253700</td>\n",
       "      <td>0.974150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.657057</td>\n",
       "      <td>-0.508105</td>\n",
       "      <td>-0.556453</td>\n",
       "      <td>0.057045</td>\n",
       "      <td>-0.344814</td>\n",
       "      <td>-0.557824</td>\n",
       "      <td>-0.814844</td>\n",
       "      <td>0.016355</td>\n",
       "      <td>-0.384234</td>\n",
       "      <td>-0.022224</td>\n",
       "      <td>...</td>\n",
       "      <td>0.077500</td>\n",
       "      <td>-1.489750</td>\n",
       "      <td>-0.151010</td>\n",
       "      <td>0.347814</td>\n",
       "      <td>2.281268</td>\n",
       "      <td>-1.275026</td>\n",
       "      <td>-0.141539</td>\n",
       "      <td>-0.335557</td>\n",
       "      <td>0.196004</td>\n",
       "      <td>-1.347782</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.939275</td>\n",
       "      <td>-0.780565</td>\n",
       "      <td>-0.495687</td>\n",
       "      <td>-0.308105</td>\n",
       "      <td>-0.604543</td>\n",
       "      <td>-0.272711</td>\n",
       "      <td>0.080763</td>\n",
       "      <td>0.542582</td>\n",
       "      <td>0.580669</td>\n",
       "      <td>0.162638</td>\n",
       "      <td>...</td>\n",
       "      <td>0.677662</td>\n",
       "      <td>0.188278</td>\n",
       "      <td>0.716616</td>\n",
       "      <td>-1.290398</td>\n",
       "      <td>-0.579556</td>\n",
       "      <td>-0.692827</td>\n",
       "      <td>-1.040820</td>\n",
       "      <td>-0.674525</td>\n",
       "      <td>1.355343</td>\n",
       "      <td>1.754870</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 100 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         V1        V2        V3        V4        V5        V6        V7  \\\n",
       "0  0.101741  0.144909  0.235567  0.576186  0.299443  0.296395  0.902235   \n",
       "1 -0.105054 -0.110128 -0.033311 -0.042925 -0.752605 -0.794815 -1.699739   \n",
       "2 -0.478922 -0.058475 -0.620625  1.775435  0.935760  1.268173 -2.652118   \n",
       "3 -0.657057 -0.508105 -0.556453  0.057045 -0.344814 -0.557824 -0.814844   \n",
       "4 -0.939275 -0.780565 -0.495687 -0.308105 -0.604543 -0.272711  0.080763   \n",
       "\n",
       "         V8        V9       V10  ...       V91       V92       V93       V94  \\\n",
       "0  0.265811  0.420927  0.684045  ... -0.058677  0.497420 -1.124986  0.338215   \n",
       "1 -1.891533 -1.287547 -1.154547  ... -1.614948 -1.337878  0.795742  1.101117   \n",
       "2 -2.327299 -2.913926 -2.140256  ...  0.081620  0.999657 -0.594758  0.057804   \n",
       "3  0.016355 -0.384234 -0.022224  ...  0.077500 -1.489750 -0.151010  0.347814   \n",
       "4  0.542582  0.580669  0.162638  ...  0.677662  0.188278  0.716616 -1.290398   \n",
       "\n",
       "        V95       V96       V97       V98       V99      V100  \n",
       "0 -0.942943 -1.257044 -0.531471  1.236317  0.405682  0.387636  \n",
       "1 -0.920702 -0.098002 -0.269719  0.333092 -0.500367  1.340876  \n",
       "2 -1.259650  0.321864  0.992930  0.552269  1.253700  0.974150  \n",
       "3  2.281268 -1.275026 -0.141539 -0.335557  0.196004 -1.347782  \n",
       "4 -0.579556 -0.692827 -1.040820 -0.674525  1.355343  1.754870  \n",
       "\n",
       "[5 rows x 100 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>V1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.811113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.049249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-4.496389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-3.846408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-2.805357</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         V1\n",
       "0  2.811113\n",
       "1  1.049249\n",
       "2 -4.496389\n",
       "3 -3.846408\n",
       "4 -2.805357"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## General Usage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Seungha\\anaconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\__init__.py:15: FutureWarning: sklearn.externals.joblib is deprecated in 0.21 and will be removed in 0.23. Please import this functionality directly from joblib, which can be installed with: pip install joblib. If this warning is raised when loading pickled models, you may need to re-serialize those models with scikit-learn 0.21+.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\Seungha\\anaconda3\\lib\\site-packages\\sklearn\\externals\\six.py:31: FutureWarning: The module is deprecated in version 0.21 and will be removed in version 0.23 since we've dropped support for Python 2.7. Please rely on the official version of six (https://pypi.org/project/six/).\n",
      "  \"(https://pypi.org/project/six/).\", FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Procedure_1_fin.\n",
      "Procedure_2_fin.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Hi_LASSO_pyspark.HiLASSO_Spark at 0x172ece75f48>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from Hi_LASSO_pyspark import HiLASSO_Spark\n",
    "\n",
    "model = HiLASSO_Spark(X, y, alpha=0.05, q1='auto', q2='auto', L=30, cv=5, node='auto', logistic=False)\n",
    "model.fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1.15905047e+00,  1.73054066e+00, -2.78546358e-01,  7.08538060e-01,\n",
       "        2.93630644e-01, -3.09802967e-02,  9.87172481e-01,  4.98625094e-01,\n",
       "        2.80438607e-01,  2.04938322e-03, -5.33703582e-01,  0.00000000e+00,\n",
       "       -2.25060983e-02,  0.00000000e+00, -5.63347651e-01,  7.20865170e-03,\n",
       "       -1.05214712e-02,  1.45435365e-02,  0.00000000e+00,  8.20807070e-03,\n",
       "        2.18642615e-01, -5.03987466e-02,  0.00000000e+00,  7.34933400e-03,\n",
       "        1.43663963e-01,  0.00000000e+00,  6.34305539e-03,  2.00344988e-02,\n",
       "        3.13616708e-05,  6.35422045e-02, -2.01727289e-03,  0.00000000e+00,\n",
       "        4.78273926e-02,  4.29157582e-01,  0.00000000e+00,  0.00000000e+00,\n",
       "        0.00000000e+00, -1.40783507e-02,  0.00000000e+00,  0.00000000e+00,\n",
       "        8.87616324e-02,  0.00000000e+00,  3.38209725e-01,  0.00000000e+00,\n",
       "       -4.18006928e-02,  3.83894299e-02, -3.01721439e-02,  0.00000000e+00,\n",
       "        0.00000000e+00, -1.45920949e-03,  0.00000000e+00, -3.20830882e-02,\n",
       "       -1.49872298e-01,  0.00000000e+00, -3.98133525e-03,  0.00000000e+00,\n",
       "        2.45903335e-03,  7.68685032e-03, -5.65818992e-02,  1.12274264e-01,\n",
       "        2.13740191e-03,  0.00000000e+00, -4.18373709e-04,  0.00000000e+00,\n",
       "       -7.87575129e-04,  0.00000000e+00,  4.93090821e-02,  0.00000000e+00,\n",
       "       -4.18499004e-04, -3.56030163e-04,  3.92035324e-02,  2.35306927e-01,\n",
       "        1.71041040e-02,  0.00000000e+00,  1.03596675e-02,  0.00000000e+00,\n",
       "        2.09837938e-02, -2.43313781e-03,  0.00000000e+00,  0.00000000e+00,\n",
       "       -2.72681876e-01,  1.64381733e-01,  1.56982030e-01,  1.43836815e-03,\n",
       "        4.66357974e-03,  0.00000000e+00, -1.12394212e+00, -8.07932918e-02,\n",
       "        0.00000000e+00,  0.00000000e+00, -1.70966580e-03,  0.00000000e+00,\n",
       "       -2.23018886e-03,  1.26032087e-01,  0.00000000e+00,  1.03304962e-01,\n",
       "        0.00000000e+00,  0.00000000e+00, -5.00787433e-02,  8.02132494e-03])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5.45546950e-131, 2.01625779e-164, 1.31670825e-004, 8.80886752e-097,\n",
       "       4.31004255e-027, 9.99999051e-001, 8.02834634e-123, 6.37878173e-048,\n",
       "       6.27334431e-017, 1.00000000e+000, 1.70639215e-121, 1.00000000e+000,\n",
       "       9.99733947e-001, 1.00000000e+000, 6.12580122e-098, 1.00000000e+000,\n",
       "       1.00000000e+000, 9.99999704e-001, 1.00000000e+000, 1.00000000e+000,\n",
       "       1.31517681e-007, 8.32544869e-001, 1.00000000e+000, 1.00000000e+000,\n",
       "       2.85508681e-007, 1.00000000e+000, 1.00000000e+000, 9.99948594e-001,\n",
       "       1.00000000e+000, 5.20943089e-003, 1.00000000e+000, 1.00000000e+000,\n",
       "       1.85618624e-002, 3.49112719e-075, 1.00000000e+000, 1.00000000e+000,\n",
       "       1.00000000e+000, 1.00000000e+000, 1.00000000e+000, 1.00000000e+000,\n",
       "       4.16256090e-004, 1.00000000e+000, 1.28847785e-054, 1.00000000e+000,\n",
       "       5.31842478e-001, 2.19719125e-001, 9.81054785e-001, 1.00000000e+000,\n",
       "       1.00000000e+000, 1.00000000e+000, 1.00000000e+000, 9.34796624e-001,\n",
       "       2.85508681e-007, 1.00000000e+000, 1.00000000e+000, 1.00000000e+000,\n",
       "       1.00000000e+000, 1.00000000e+000, 1.01690916e-001, 5.94884253e-008,\n",
       "       1.00000000e+000, 1.00000000e+000, 1.00000000e+000, 1.00000000e+000,\n",
       "       1.00000000e+000, 1.00000000e+000, 8.73984401e-001, 1.00000000e+000,\n",
       "       1.00000000e+000, 1.00000000e+000, 9.70377124e-001, 7.00544469e-023,\n",
       "       9.99948594e-001, 1.00000000e+000, 1.00000000e+000, 1.00000000e+000,\n",
       "       9.99997164e-001, 1.00000000e+000, 1.00000000e+000, 1.00000000e+000,\n",
       "       2.73297075e-068, 8.03655110e-012, 1.39764626e-010, 1.00000000e+000,\n",
       "       1.00000000e+000, 1.00000000e+000, 4.97483677e-211, 2.64252727e-008,\n",
       "       1.00000000e+000, 1.00000000e+000, 1.00000000e+000, 1.00000000e+000,\n",
       "       1.00000000e+000, 1.39764626e-010, 1.00000000e+000, 4.94110926e-009,\n",
       "       1.00000000e+000, 1.00000000e+000, 9.93078320e-001, 1.00000000e+000])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.p_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1.15905047,  1.73054066, -0.27854636,  0.70853806,  0.29363064,\n",
       "        0.        ,  0.98717248,  0.49862509,  0.28043861,  0.        ,\n",
       "       -0.53370358,  0.        ,  0.        ,  0.        , -0.56334765,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.21864261,  0.        ,  0.        ,  0.        ,  0.14366396,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.42915758,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.08876163,  0.        ,  0.33820973,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        , -0.1498723 ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.11227426,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.23530693,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "       -0.27268188,  0.16438173,  0.15698203,  0.        ,  0.        ,\n",
       "        0.        , -1.12394212, -0.08079329,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.12603209,  0.        ,\n",
       "        0.10330496,  0.        ,  0.        ,  0.        ,  0.        ])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.selected_var"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}