getting_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "notes: \n",
    "    \n",
    "* source activate cpac\n",
    "* conda install jupyter\n",
    "* git clone https:nbpapaya\n",
    "* remember to do submodules\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline\n",
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('Phenotypic_V1_0b_preprocessed1.csv',\n",
       " <httplib.HTTPMessage instance at 0x7f167c4356c8>)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import urllib\n",
    "\n",
    "pheno_file_url=\"https://s3.amazonaws.com/fcp-indi/data/Projects/ABIDE_Initiative/Phenotypic_V1_0b_preprocessed1.csv\"\n",
    "pheno_file_local=\"Phenotypic_V1_0b_preprocessed1.csv\"\n",
    "\n",
    "urllib.urlretrieve(pheno_file_url, pheno_file_local)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>SUB_ID</th>\n",
       "      <th>X</th>\n",
       "      <th>subject</th>\n",
       "      <th>SITE_ID</th>\n",
       "      <th>FILE_ID</th>\n",
       "      <th>DX_GROUP</th>\n",
       "      <th>DSM_IV_TR</th>\n",
       "      <th>AGE_AT_SCAN</th>\n",
       "      <th>...</th>\n",
       "      <th>qc_notes_rater_1</th>\n",
       "      <th>qc_anat_rater_2</th>\n",
       "      <th>qc_anat_notes_rater_2</th>\n",
       "      <th>qc_func_rater_2</th>\n",
       "      <th>qc_func_notes_rater_2</th>\n",
       "      <th>qc_anat_rater_3</th>\n",
       "      <th>qc_anat_notes_rater_3</th>\n",
       "      <th>qc_func_rater_3</th>\n",
       "      <th>qc_func_notes_rater_3</th>\n",
       "      <th>SUB_IN_SMP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>50002</td>\n",
       "      <td>1</td>\n",
       "      <td>50002</td>\n",
       "      <td>PITT</td>\n",
       "      <td>no_filename</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>16.77</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>fail</td>\n",
       "      <td>ic-parietal-cerebellum</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>fail</td>\n",
       "      <td>ERROR #24</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>50003</td>\n",
       "      <td>2</td>\n",
       "      <td>50003</td>\n",
       "      <td>PITT</td>\n",
       "      <td>Pitt_0050003</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>24.45</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>50004</td>\n",
       "      <td>3</td>\n",
       "      <td>50004</td>\n",
       "      <td>PITT</td>\n",
       "      <td>Pitt_0050004</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>19.09</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>50005</td>\n",
       "      <td>4</td>\n",
       "      <td>50005</td>\n",
       "      <td>PITT</td>\n",
       "      <td>Pitt_0050005</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>13.73</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>maybe</td>\n",
       "      <td>ic-parietal-cerebellum</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>50006</td>\n",
       "      <td>5</td>\n",
       "      <td>50006</td>\n",
       "      <td>PITT</td>\n",
       "      <td>Pitt_0050006</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>13.37</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>maybe</td>\n",
       "      <td>ic-parietal slight</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 106 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  Unnamed: 0.1  SUB_ID  X  subject SITE_ID       FILE_ID  \\\n",
       "0           0             1   50002  1    50002    PITT   no_filename   \n",
       "1           1             2   50003  2    50003    PITT  Pitt_0050003   \n",
       "2           2             3   50004  3    50004    PITT  Pitt_0050004   \n",
       "3           3             4   50005  4    50005    PITT  Pitt_0050005   \n",
       "4           4             5   50006  5    50006    PITT  Pitt_0050006   \n",
       "\n",
       "   DX_GROUP  DSM_IV_TR  AGE_AT_SCAN     ...      qc_notes_rater_1  \\\n",
       "0         1          1        16.77     ...                   NaN   \n",
       "1         1          1        24.45     ...                   NaN   \n",
       "2         1          1        19.09     ...                   NaN   \n",
       "3         1          1        13.73     ...                   NaN   \n",
       "4         1          1        13.37     ...                   NaN   \n",
       "\n",
       "  qc_anat_rater_2  qc_anat_notes_rater_2  qc_func_rater_2  \\\n",
       "0              OK                    NaN             fail   \n",
       "1              OK                    NaN               OK   \n",
       "2              OK                    NaN               OK   \n",
       "3              OK                    NaN            maybe   \n",
       "4              OK                    NaN            maybe   \n",
       "\n",
       "    qc_func_notes_rater_2  qc_anat_rater_3 qc_anat_notes_rater_3  \\\n",
       "0  ic-parietal-cerebellum               OK                   NaN   \n",
       "1                     NaN               OK                   NaN   \n",
       "2                     NaN               OK                   NaN   \n",
       "3  ic-parietal-cerebellum               OK                   NaN   \n",
       "4      ic-parietal slight               OK                   NaN   \n",
       "\n",
       "  qc_func_rater_3 qc_func_notes_rater_3  SUB_IN_SMP  \n",
       "0            fail             ERROR #24           1  \n",
       "1              OK                   NaN           1  \n",
       "2              OK                   NaN           1  \n",
       "3              OK                   NaN           0  \n",
       "4              OK                   NaN           1  \n",
       "\n",
       "[5 rows x 106 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "pheno_df=pd.read_csv(pheno_file_local)\n",
    "pheno_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pipes=['ccs','cpac','dparsf','niak']\n",
    "strats=['filt_global','filt_noglobal','nofilt_global','nofilt_noglobal']\n",
    "derivatives=['alff']#,'degree_binarize','degree_weighted','eigenvector_binarize',\n",
    "    #'eigenvector_weighted','falff','func_mean','lfcd','reho','vmhc']\n",
    "file_id = [\"Pitt_0050003\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import itertools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "new_df = pd.DataFrame(columns = [\"pipe\", \"strat\", \"deriv\", \"file_id\"])\n",
    "for combo in itertools.product(pipes, strats, derivatives, file_id):\n",
    "    new_df = new_df.append(pd.Series({key: combo[i] for i, key in enumerate(new_df.columns)}), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pipe</th>\n",
       "      <th>strat</th>\n",
       "      <th>deriv</th>\n",
       "      <th>file_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ccs</td>\n",
       "      <td>filt_global</td>\n",
       "      <td>alff</td>\n",
       "      <td>Pitt_0050003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ccs</td>\n",
       "      <td>filt_noglobal</td>\n",
       "      <td>alff</td>\n",
       "      <td>Pitt_0050003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ccs</td>\n",
       "      <td>nofilt_global</td>\n",
       "      <td>alff</td>\n",
       "      <td>Pitt_0050003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ccs</td>\n",
       "      <td>nofilt_noglobal</td>\n",
       "      <td>alff</td>\n",
       "      <td>Pitt_0050003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cpac</td>\n",
       "      <td>filt_global</td>\n",
       "      <td>alff</td>\n",
       "      <td>Pitt_0050003</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pipe            strat deriv       file_id\n",
       "0   ccs      filt_global  alff  Pitt_0050003\n",
       "1   ccs    filt_noglobal  alff  Pitt_0050003\n",
       "2   ccs    nofilt_global  alff  Pitt_0050003\n",
       "3   ccs  nofilt_noglobal  alff  Pitt_0050003\n",
       "4  cpac      filt_global  alff  Pitt_0050003"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# configure out download string\n",
    "template_str=\"https://s3.amazonaws.com/fcp-indi/data/Projects/ABIDE_Initiative\\\n",
    "/Outputs/{pipeline}/{strategy}/{derivative}/{file_id}_{derivative}.nii.gz\"\n",
    "\n",
    "def get_data_str(pipe, strat, deriv, file_id):\n",
    "    data_str = template_str.format(pipeline=pipe, strategy=strat, derivative=deriv, file_id=file_id)\n",
    "    local_file = os.path.join(\"data\",os.path.basename(data_str))\n",
    "    if not os.path.isfile(local_file):\n",
    "        # now download the file\n",
    "        try:\n",
    "            urllib.urlretrieve(data_str, local_file)\n",
    "        except Exception as e:\n",
    "            print \"Could not download %s\"%(data_str)\n",
    "            pheno_df.loc[pheno_df[\"FILE_ID\"]==file_id,\"dl_error\"]=True\n",
    "            \n",
    "    return data_str, local_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "new_df[\"local_filename\"] = new_df.index.map(lambda i: get_data_str(new_df.loc[i].pipe,\n",
    "                                                             new_df.loc[i].strat, \n",
    "                                                             new_df.loc[i].deriv, \n",
    "                                                             new_df.loc[i].file_id)[1])\n",
    "new_df[\"remote_filename\"] = new_df.index.map(lambda i: get_data_str(new_df.loc[i].pipe,\n",
    "                                                             new_df.loc[i].strat, \n",
    "                                                             new_df.loc[i].deriv, \n",
    "                                                             new_df.loc[i].file_id)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pipe</th>\n",
       "      <th>strat</th>\n",
       "      <th>deriv</th>\n",
       "      <th>file_id</th>\n",
       "      <th>filename</th>\n",
       "      <th>local_filename</th>\n",
       "      <th>remote_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ccs</td>\n",
       "      <td>filt_global</td>\n",
       "      <td>alff</td>\n",
       "      <td>Pitt_0050003</td>\n",
       "      <td>data/Pitt_0050003_alff.nii.gz</td>\n",
       "      <td>data/Pitt_0050003_alff.nii.gz</td>\n",
       "      <td>https://s3.amazonaws.com/fcp-indi/data/Project...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ccs</td>\n",
       "      <td>filt_noglobal</td>\n",
       "      <td>alff</td>\n",
       "      <td>Pitt_0050003</td>\n",
       "      <td>data/Pitt_0050003_alff.nii.gz</td>\n",
       "      <td>data/Pitt_0050003_alff.nii.gz</td>\n",
       "      <td>https://s3.amazonaws.com/fcp-indi/data/Project...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ccs</td>\n",
       "      <td>nofilt_global</td>\n",
       "      <td>alff</td>\n",
       "      <td>Pitt_0050003</td>\n",
       "      <td>data/Pitt_0050003_alff.nii.gz</td>\n",
       "      <td>data/Pitt_0050003_alff.nii.gz</td>\n",
       "      <td>https://s3.amazonaws.com/fcp-indi/data/Project...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ccs</td>\n",
       "      <td>nofilt_noglobal</td>\n",
       "      <td>alff</td>\n",
       "      <td>Pitt_0050003</td>\n",
       "      <td>data/Pitt_0050003_alff.nii.gz</td>\n",
       "      <td>data/Pitt_0050003_alff.nii.gz</td>\n",
       "      <td>https://s3.amazonaws.com/fcp-indi/data/Project...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cpac</td>\n",
       "      <td>filt_global</td>\n",
       "      <td>alff</td>\n",
       "      <td>Pitt_0050003</td>\n",
       "      <td>data/Pitt_0050003_alff.nii.gz</td>\n",
       "      <td>data/Pitt_0050003_alff.nii.gz</td>\n",
       "      <td>https://s3.amazonaws.com/fcp-indi/data/Project...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pipe            strat deriv       file_id                       filename  \\\n",
       "0   ccs      filt_global  alff  Pitt_0050003  data/Pitt_0050003_alff.nii.gz   \n",
       "1   ccs    filt_noglobal  alff  Pitt_0050003  data/Pitt_0050003_alff.nii.gz   \n",
       "2   ccs    nofilt_global  alff  Pitt_0050003  data/Pitt_0050003_alff.nii.gz   \n",
       "3   ccs  nofilt_noglobal  alff  Pitt_0050003  data/Pitt_0050003_alff.nii.gz   \n",
       "4  cpac      filt_global  alff  Pitt_0050003  data/Pitt_0050003_alff.nii.gz   \n",
       "\n",
       "                  local_filename  \\\n",
       "0  data/Pitt_0050003_alff.nii.gz   \n",
       "1  data/Pitt_0050003_alff.nii.gz   \n",
       "2  data/Pitt_0050003_alff.nii.gz   \n",
       "3  data/Pitt_0050003_alff.nii.gz   \n",
       "4  data/Pitt_0050003_alff.nii.gz   \n",
       "\n",
       "                                     remote_filename  \n",
       "0  https://s3.amazonaws.com/fcp-indi/data/Project...  \n",
       "1  https://s3.amazonaws.com/fcp-indi/data/Project...  \n",
       "2  https://s3.amazonaws.com/fcp-indi/data/Project...  \n",
       "3  https://s3.amazonaws.com/fcp-indi/data/Project...  \n",
       "4  https://s3.amazonaws.com/fcp-indi/data/Project...  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nbpapaya import Brain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "doing checks /root/.jupyter/custom/\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<iframe src=\"https://ec2-52-90-103-243.compute-1.amazonaws.com:8888/files/papaya_data/tmp6FylIl.html\"\n",
       "                   width=\"600\"\n",
       "                   height=\"450\"\n",
       "                   scrolling=\"no\"\n",
       "                   frameBorder=\"0\">\n",
       "                   </iframe>"
      ],
      "text/plain": [
       "<nbpapaya.base.Brain at 0x7f167c66fc50>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Brain([os.path.abspath(\"./data/Pitt_0050003_alff.nii.gz\")], \n",
    "      host=\"https://ec2-52-90-103-243.compute-1.amazonaws.com\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}