feature_selection.ipynb

{
 "metadata": {
  "name": ""
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Trying with differnt feature extractions. \n",
      "Note : These are not used in final submission"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pandas as pd\n",
      "from sklearn.feature_extraction.text import CountVectorizer\n",
      "import re\n",
      "from sklearn.metrics import mean_squared_error\n",
      "from sklearn import cross_validation\n",
      "from sklearn.ensemble import GradientBoostingClassifier,GradientBoostingRegressor\n",
      "from sklearn.decomposition import TruncatedSVD\n",
      "from sklearn.preprocessing import LabelEncoder,LabelBinarizer,OneHotEncoder\n",
      "\n",
      "def errorFunc(pred,actual):\n",
      "    #return np.mean(np.square(pred-actual).sum())\n",
      "    return mean_squared_error(pred,actual)\n",
      "    \n",
      "def y_binary_p(y,p):\n",
      "    if y <= p:\n",
      "        return 0    \n",
      "    else:\n",
      "        return 1\n",
      "\n",
      "\n",
      "def isItRetweet(df):\n",
      "    return [len(re.findall('^rt', tweet)) > 0 for tweet in df.tweet]\n",
      "\n",
      "def hashTagsCount(df):\n",
      "    return [len(re.findall('\\#.',tweet)) for tweet in df.tweet]\n",
      "\n",
      "def mentionCount(df):\n",
      "    return [len(re.findall('\\@.',tweet)) for tweet in df.tweet]\n",
      "\n",
      "def numericContains(df):\n",
      "    return [len(re.findall('[0-9]+[:.]*[0-9]*.',tweet)) for tweet in df.tweet]\n",
      "    \n",
      "def containsAMPM(df):\n",
      "    return [len(re.findall('[0-9]+\\s*am|pm',tweet)) for tweet in df.tweet]\n",
      "    \n",
      "def linksCount(df):\n",
      "    return [len(re.findall('{link}',tweet)) for tweet in df.tweet]\n",
      "\n",
      "def containsHashWeather(df):\n",
      "    return [len(re.findall('\\#weather',tweet)) for tweet in df.tweet]\n",
      "\n",
      "def tweetLength(df):\n",
      "    return [len(tweet) for tweet in df.tweet]\n",
      "\n",
      "def getHighFeqWords(df,col,percent = 0.85, stop_words = 'english',max_features = 50):\n",
      "    \n",
      "    df1 = df['tweet'][df[col] > percent]\n",
      "    print 'total_tweets, filtered tweets, perent_cover ###',df.shape[0],df1.shape[0], (1.0 * df1.shape[0])/df.shape[0]\n",
      "    countVect = CountVectorizer(strip_accents = 'ascii',analyzer='word',stop_words = stop_words,\n",
      "                                max_features = max_features,binary = True)    \n",
      "    \n",
      "    countVect.fit(df1.values)\n",
      "    maxFeatures = countVect.fit(df['tweet'][df[col] > percent]).get_feature_names()\n",
      "    print maxFeatures\n",
      "    countVect = CountVectorizer(vocabulary=maxFeatures)\n",
      "    return countVect.fit_transform(df.tweet).toarray()\n",
      "\n",
      "def getFeatureVecture(df,function_list = None,col_category=None):\n",
      "    \n",
      "    returnVals = None\n",
      "    \n",
      "    if(function_list == None):\n",
      "        function_chain = [isItRetweet,hashTagsCount,mentionCount,numericContains,containsAMPM,linksCount,\n",
      "                      containsHashWeather,tweetLength]\n",
      "        returnVals = np.hstack([np.matrix(func(df)).T for func in function_chain])\n",
      "        \n",
      "    if(col_category == 's'):\n",
      "        returnVals = np.hstack((returnVals,getHighFeqWords(df,'s2',percent = 0.8)))\n",
      "        returnVals = np.hstack((returnVals,getHighFeqWords(df,'s4',percent = 0.8)))\n",
      "        \n",
      "        \n",
      "    return returnVals\n",
      "\n",
      "\n",
      "def cleanData(s):    \n",
      "    \n",
      "    s = s.lower()\n",
      "    s = re.sub('n\\'t',' not',s)\n",
      "    s = re.sub('{link}',' ',s)\n",
      "    s = re.sub('#\\w+',' ',s)\n",
      "    s = re.sub('@\\w+',' ',s)\n",
      "    s = re.sub('[\\W]',' ',s)\n",
      "    s = re.sub('\\s+',' ',s)\n",
      "    s = re.sub('\\d+','00',s)\n",
      "    return s   \n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def main():\n",
      "    print 'reading the datsets...'\n",
      "    train = pd.read_csv('/home/trupti/data_local/kaggle/partly_sunny/train.csv',header = 0)\n",
      "    train['tweet'] = [tweet.lower() for tweet in train.tweet]\n",
      "    \n",
      "    test = pd.read_csv('/home/trupti/data_local/kaggle/partly_sunny/test.csv',header = 0)\n",
      "    test['tweet'] = [tweet.lower() for tweet in test.tweet]\n",
      "    \n",
      "    \n",
      "    #clf = GradientBoostingClassifier(n_estimators=50)\n",
      "    regg = GradientBoostingRegressor(n_estimators = 30)\n",
      "    lsa = TruncatedSVD(n_components= 30, n_iterations = 50)\n",
      "    \n",
      "    tweets_train = train['tweet']\n",
      "    tweets_test = test['tweet']\n",
      "    X_all = list(tweets_train)+list(tweets_test)\n",
      "    X_all = [cleanData(s) for s in X_all]\n",
      "   \n",
      "    tfidf = CountVectorizer(X_all,min_df=5,  max_features=None, strip_accents='unicode',  \n",
      "        analyzer='word',token_pattern=r'\\w{2,}',ngram_range=(1,1),stop_words = 'english')   \n",
      "    \n",
      "\n",
      "    print 'fitting TfIDF...'\n",
      "    trainLen = tweets_train.shape[0]\n",
      "\n",
      "\n",
      "    tfidf.fit(X_all)\n",
      "    X_all = tfidf.transform(X_all)    \n",
      "\n",
      "    print 'Fitting in LSA'\n",
      "    X_all = lsa.fit_transform(X_all)\n",
      "\n",
      "    X_train = X_all[:trainLen]\n",
      "    X_test = X_all[trainLen:]\n",
      "    \n",
      "    \n",
      "    #print OneHotEncoder().fit_transform(getFeatureVecture(train)).shape\n",
      "    \n",
      "    X_train = np.hstack((X_train,OneHotEncoder().fit_transform(getFeatureVecture(train)).toarray()))\n",
      "    X_train = np.hstack((X_train,getHighFeqWords(train,'s2',percent = 0.8),getHighFeqWords(train,'s4',percent = 0.8)))\n",
      "    X_train = np.hstack((X_train,LabelBinarizer().fit_transform(LabelEncoder().fit_transform(train.state))))\n",
      "    \n",
      "    \n",
      "    kf = cross_validation.KFold(X_train.shape[0],n_folds = 10)\n",
      "    \n",
      "    \n",
      "    final_error = []\n",
      "    j = 0\n",
      "    for train_idx,cv_idx in kf: \n",
      "        j +=1\n",
      "        print 'prcessing CV :',j\n",
      "        X_train_kf = X_train[train_idx,:]\n",
      "        X_cv_kf = X_train[cv_idx,:]\n",
      "        \n",
      "        y_cv_kf = train.ix[cv_idx,4:9]\n",
      "        #creating a place holder of cv predictions\n",
      "        pred_cv = pd.DataFrame(np.zeros(y_cv_kf.shape), columns = train.columns[4:9])\n",
      "        \n",
      "        error_pred = 100\n",
      "        \n",
      "        for y_col in train.columns[4:9]:\n",
      "            y = train[y_col]            \n",
      "            y_train_kf = y[train_idx]            \n",
      "           # y_train_kf = vec_y_binary_p(y_train_kf,0.5)\n",
      "            print 'Fitting for column ',y_col\n",
      "            regg.fit(X_train_kf,log(1 + y_train_kf))\n",
      "            pred_cv[y_col] = exp(regg.predict(X_cv_kf)) - 1\n",
      "            \n",
      "        #error = sqrt(mean_squared_error(y_cv_kf,pred_cv))\n",
      "        error = sqrt(errorFunc(y_cv_kf,pred_cv))\n",
      "        final_error.append(error)\n",
      "        print 'error for cv',j, error\n",
      "    \n",
      "    print 'Mean error for 10 fold CV is ',np.mean(final_error)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "    \n",
      "def sub_main():\n",
      "    print 'reading the datsets...'\n",
      "    train = pd.read_csv('/home/trupti/data_local/kaggle/partly_sunny/train.csv',header = 0)\n",
      "    train['tweet'] = [tweet.lower() for tweet in train.tweet]\n",
      "    \n",
      "    #test = pd.read_csv('/home/trupti/data_local/kaggle/partly_sunny/test.csv',header = 0)\n",
      "    #test['tweet'] = [tweet.lower() for tweet in test.tweet]\n",
      "    \n",
      "    \n",
      "    #clf = GradientBoostingClassifier(n_estimators=50)\n",
      "    regg = GradientBoostingRegressor(n_estimators = 100)\n",
      "    \n",
      "    X_train = getFeatureVecture(train,col_category = 's')\n",
      "    \n",
      "    kf = cross_validation.KFold(X_train.shape[0],n_folds = 10)\n",
      "    \n",
      "    \n",
      "    final_error = []\n",
      "    j = 0\n",
      "    for train_idx,cv_idx in kf: \n",
      "        j +=1\n",
      "        print 'prcessing CV :',j\n",
      "        X_train_kf = X_train[train_idx,:]\n",
      "        X_cv_kf = X_train[cv_idx,:]\n",
      "        \n",
      "        y_cv_kf = train.ix[cv_idx,4:9]\n",
      "        #creating a place holder of cv predictions\n",
      "        pred_cv = pd.DataFrame(np.zeros(y_cv_kf.shape), columns = train.columns[4:9])\n",
      "        \n",
      "        error_pred = 100\n",
      "        \n",
      "        for y_col in train.columns[4:9]:\n",
      "            y = train[y_col]            \n",
      "            y_train_kf = y[train_idx]            \n",
      "           # y_train_kf = vec_y_binary_p(y_train_kf,0.5)\n",
      "            print 'Fitting for column ',y_col\n",
      "            regg.fit(X_train_kf,log(1 + y_train_kf))\n",
      "            pred_cv[y_col] = exp(regg.predict(X_cv_kf)) - 1\n",
      "            \n",
      "        #error = sqrt(mean_squared_error(y_cv_kf,pred_cv))\n",
      "        error = sqrt(errorFunc(y_cv_kf,pred_cv))\n",
      "        final_error.append(error)\n",
      "        print 'error for cv',j, error\n",
      "    \n",
      "    print 'Mean error for 10 fold CV is ',np.mean(final_error)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "if __name__ == \"__main__\":\n",
      "    sub_main()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "train1 = pd.read_csv('/home/trupti/data_local/kaggle/partly_sunny/train.csv',header = 0)\n",
      "train1['tweet'] = [tweet.lower() for tweet in train1.tweet]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "getHighFeqWords(train1,'s5',percent = 0.90)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "total_tweets, filtered tweets, perent_cover ### 77946 855 0.0109691324763\n",
        "[u'blizzard', u'check', u'cold', u'come', u'day', u'degrees', u'don', u'freezing', u'getting', u'good', u'got', u'great', u'hot', u'just', u'know', u'let', u'like', u'link', u'll', u'lol', u'love', u'make', u'man', u'mention', u'need', u'new', u'night', u'oh', u'patrol', u'really', u'right', u'rt', u'shit', u'slush', u'snow', u'song', u'storm', u'sunny', u'sunshine', u'temp', u'time', u'today', u'twitter', u'video', u'want', u'warm', u'way', u'weather', u'white', u'world']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 14,
       "text": [
        "array([[0, 0, 0, ..., 0, 0, 0],\n",
        "       [0, 0, 0, ..., 0, 0, 0],\n",
        "       [0, 0, 0, ..., 0, 0, 0],\n",
        "       ..., \n",
        "       [0, 0, 0, ..., 0, 0, 0],\n",
        "       [0, 0, 1, ..., 1, 0, 0],\n",
        "       [0, 0, 0, ..., 0, 0, 0]])"
       ]
      }
     ],
     "prompt_number": 14
    }
   ],
   "metadata": {}
  }
 ]
}