1. import and export 0. Object Storage.ipynb

{
    "nbformat": 4, 
    "cells": [
        {
            "cell_type": "markdown", 
            "metadata": {}, 
            "source": "# In-and export from Object storage"
        }, 
        {
            "cell_type": "code", 
            "execution_count": 2, 
            "source": "#imports\nfrom io import StringIO\nimport requests\nimport json\n\nimport pandas as pd\nimport numpy as np", 
            "metadata": {
                "collapsed": true
            }, 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "metadata": {}, 
            "source": "Credentials"
        }, 
        {
            "cell_type": "code", 
            "execution_count": 3, 
            "source": "@hidden_cell\n\ncredentials_1= {\n  \"auth_url\": \"https://identity.open.softlayer.com\",\n  \"project\": \"object_storage_effacaaa_ad08_4ee4_bd59_b2e105bc9639\",\n  \"projectId\": \"e96165fa44c44a7d956507ebf4026cfb\",\n  \"region\": \"dallas\",\n  \"userId\": \"5002ddd00432452d8dd086e3f74ed3f1\",\n  \"username\": \"admin_bdffbeee0797e72fe86a4270ec23774a835cbd4b\",\n  \"password\": \"TAu!-k36VR[&xp9V\",\n  \"domainId\": \"193a321481be4f73b08e76a87e7d585a\",\n  \"container\":\"DSETraining101ObjectStorage\",\n  \"domainName\": \"1123181\",\n  \"role\": \"admin\"\n}\n", 
            "metadata": {
                "collapsed": true
            }, 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 8, 
            "source": "# create some data\nrandom_data=pd.DataFrame(np.random.random((1000, 100)))\nrandom_data.head()", 
            "metadata": {
                "collapsed": false
            }, 
            "outputs": [
                {
                    "execution_count": 8, 
                    "data": {
                        "text/html": "<div>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n      <th>4</th>\n      <th>5</th>\n      <th>6</th>\n      <th>7</th>\n      <th>8</th>\n      <th>9</th>\n      <th>...</th>\n      <th>90</th>\n      <th>91</th>\n      <th>92</th>\n      <th>93</th>\n      <th>94</th>\n      <th>95</th>\n      <th>96</th>\n      <th>97</th>\n      <th>98</th>\n      <th>99</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.218387</td>\n      <td>0.681619</td>\n      <td>0.127457</td>\n      <td>0.176184</td>\n      <td>0.208149</td>\n      <td>0.431590</td>\n      <td>0.436492</td>\n      <td>0.346246</td>\n      <td>0.632588</td>\n      <td>0.829314</td>\n      <td>...</td>\n      <td>0.432520</td>\n      <td>0.974076</td>\n      <td>0.894935</td>\n      <td>0.410939</td>\n      <td>0.487212</td>\n      <td>0.428583</td>\n      <td>0.598402</td>\n      <td>0.706644</td>\n      <td>0.385664</td>\n      <td>0.493396</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0.438189</td>\n      <td>0.477055</td>\n      <td>0.292263</td>\n      <td>0.737119</td>\n      <td>0.986888</td>\n      <td>0.571864</td>\n      <td>0.952132</td>\n      <td>0.589837</td>\n      <td>0.010117</td>\n      <td>0.174665</td>\n      <td>...</td>\n      <td>0.301537</td>\n      <td>0.552984</td>\n      <td>0.879647</td>\n      <td>0.380063</td>\n      <td>0.300516</td>\n      <td>0.398224</td>\n      <td>0.739755</td>\n      <td>0.462363</td>\n      <td>0.046500</td>\n      <td>0.510257</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.246476</td>\n      <td>0.425691</td>\n      <td>0.344394</td>\n      <td>0.302752</td>\n      <td>0.592942</td>\n      <td>0.350859</td>\n      <td>0.620692</td>\n      <td>0.516486</td>\n      <td>0.033306</td>\n      <td>0.304789</td>\n      <td>...</td>\n      <td>0.131419</td>\n      <td>0.591061</td>\n      <td>0.840573</td>\n      <td>0.664932</td>\n      <td>0.472257</td>\n      <td>0.998087</td>\n      <td>0.073214</td>\n      <td>0.984443</td>\n      <td>0.957306</td>\n      <td>0.745294</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.303911</td>\n      <td>0.326860</td>\n      <td>0.147415</td>\n      <td>0.208054</td>\n      <td>0.371225</td>\n      <td>0.910451</td>\n      <td>0.544563</td>\n      <td>0.394877</td>\n      <td>0.424432</td>\n      <td>0.418725</td>\n      <td>...</td>\n      <td>0.556135</td>\n      <td>0.013987</td>\n      <td>0.149688</td>\n      <td>0.604044</td>\n      <td>0.574204</td>\n      <td>0.930515</td>\n      <td>0.797487</td>\n      <td>0.785470</td>\n      <td>0.447085</td>\n      <td>0.587239</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.885194</td>\n      <td>0.156615</td>\n      <td>0.795850</td>\n      <td>0.208292</td>\n      <td>0.658539</td>\n      <td>0.738021</td>\n      <td>0.750869</td>\n      <td>0.233611</td>\n      <td>0.890301</td>\n      <td>0.501873</td>\n      <td>...</td>\n      <td>0.149136</td>\n      <td>0.617435</td>\n      <td>0.659965</td>\n      <td>0.096339</td>\n      <td>0.922107</td>\n      <td>0.220996</td>\n      <td>0.989596</td>\n      <td>0.604612</td>\n      <td>0.461251</td>\n      <td>0.658624</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows \u00d7 100 columns</p>\n</div>", 
                        "text/plain": "         0         1         2         3         4         5         6   \\\n0  0.218387  0.681619  0.127457  0.176184  0.208149  0.431590  0.436492   \n1  0.438189  0.477055  0.292263  0.737119  0.986888  0.571864  0.952132   \n2  0.246476  0.425691  0.344394  0.302752  0.592942  0.350859  0.620692   \n3  0.303911  0.326860  0.147415  0.208054  0.371225  0.910451  0.544563   \n4  0.885194  0.156615  0.795850  0.208292  0.658539  0.738021  0.750869   \n\n         7         8         9     ...           90        91        92  \\\n0  0.346246  0.632588  0.829314    ...     0.432520  0.974076  0.894935   \n1  0.589837  0.010117  0.174665    ...     0.301537  0.552984  0.879647   \n2  0.516486  0.033306  0.304789    ...     0.131419  0.591061  0.840573   \n3  0.394877  0.424432  0.418725    ...     0.556135  0.013987  0.149688   \n4  0.233611  0.890301  0.501873    ...     0.149136  0.617435  0.659965   \n\n         93        94        95        96        97        98        99  \n0  0.410939  0.487212  0.428583  0.598402  0.706644  0.385664  0.493396  \n1  0.380063  0.300516  0.398224  0.739755  0.462363  0.046500  0.510257  \n2  0.664932  0.472257  0.998087  0.073214  0.984443  0.957306  0.745294  \n3  0.604044  0.574204  0.930515  0.797487  0.785470  0.447085  0.587239  \n4  0.096339  0.922107  0.220996  0.989596  0.604612  0.461251  0.658624  \n\n[5 rows x 100 columns]"
                    }, 
                    "metadata": {}, 
                    "output_type": "execute_result"
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 9, 
            "source": "#usecase you have a file in Python (for example from a database) and you want to create a file in local object storage\n\ndef put_file(credentials, local_file_name):  \n    \"\"\"This functions returns a StringIO object containing\n    the file content from Bluemix Object Storage V3.\"\"\"\n    f = open(local_file_name,'r')\n    my_data = f.read()\n    url1 = ''.join(['https://identity.open.softlayer.com', '/v3/auth/tokens'])\n    data = {'auth': {'identity': {'methods': ['password'],\n            'password': {'user': {'name': credentials['username'],'domain': {'id': credentials['domainId']},\n            'password': credentials['password']}}}}}\n    headers1 = {'Content-Type': 'application/json'}\n    resp1 = requests.post(url=url1, data=json.dumps(data), headers=headers1)\n    resp1_body = resp1.json()\n    for e1 in resp1_body['token']['catalog']:\n        if(e1['type']=='object-store'):\n            for e2 in e1['endpoints']:\n                        if(e2['interface']=='public'and e2['region']=='dallas'):\n                            url2 = ''.join([e2['url'],'/', credentials['container'], '/', local_file_name])\n    s_subject_token = resp1.headers['x-subject-token']\n    headers2 = {'X-Auth-Token': s_subject_token, 'accept': 'application/json'}\n    resp2 = requests.put(url=url2, headers=headers2, data = my_data )\n    print resp2\n    \n    \n# step 1: store object in local file.\n# data_train is an earlier defined pandas dataframe containing data\nrandom_data.to_csv('random_data.csv',index=False)\n#step 2: move to object storage    \nput_file(credentials_1,\"random_data.csv\")  ", 
            "metadata": {
                "collapsed": false
            }, 
            "outputs": [
                {
                    "name": "stdout", 
                    "output_type": "stream", 
                    "text": "<Response [201]>\n"
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 10, 
            "source": "#use case: you upload a file in the object storage and want to access it in Python.\n\ndef get_object_storage_file_with_credentials(credentials,container, filename):\n    \"\"\"This functions returns a StringIO object containing\n    the file content from Bluemix Object Storage.\"\"\"\n\n    url1 = ''.join(['https://identity.open.softlayer.com', '/v3/auth/tokens'])\n    data = {'auth': {'identity': {'methods': ['password'],\n            'password': {'user': {'name': credentials['username'],'domain': {'id': credentials['domainId']},\n            'password': credentials['password']}}}}}\n    headers1 = {'Content-Type': 'application/json'}\n    resp1 = requests.post(url=url1, data=json.dumps(data), headers=headers1)\n    resp1_body = resp1.json()\n    for e1 in resp1_body['token']['catalog']:\n        if(e1['type']=='object-store'):\n            for e2 in e1['endpoints']:\n                        if(e2['interface']=='public'and e2['region']=='dallas'):\n                            url2 = ''.join([e2['url'],'/', container, '/', filename])\n    s_subject_token = resp1.headers['x-subject-token']\n    headers2 = {'X-Auth-Token': s_subject_token, 'accept': 'application/json'}\n    resp2 = requests.get(url=url2, headers=headers2)\n    return StringIO(resp2.text)\n\n# step 1: get file from object storage\n#data_train.csv is an existing object in the Object Storage \nrandom_data = pd.read_csv(get_object_storage_file_with_credentials(credentials_1,'DSETraining101ObjectStorage', 'random_data.csv'))\nrandom_data.head()\n\n\n", 
            "metadata": {
                "collapsed": false
            }, 
            "outputs": [
                {
                    "execution_count": 10, 
                    "data": {
                        "text/html": "<div>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n      <th>4</th>\n      <th>5</th>\n      <th>6</th>\n      <th>7</th>\n      <th>8</th>\n      <th>9</th>\n      <th>...</th>\n      <th>90</th>\n      <th>91</th>\n      <th>92</th>\n      <th>93</th>\n      <th>94</th>\n      <th>95</th>\n      <th>96</th>\n      <th>97</th>\n      <th>98</th>\n      <th>99</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.218387</td>\n      <td>0.681619</td>\n      <td>0.127457</td>\n      <td>0.176184</td>\n      <td>0.208149</td>\n      <td>0.431590</td>\n      <td>0.436492</td>\n      <td>0.346246</td>\n      <td>0.632588</td>\n      <td>0.829314</td>\n      <td>...</td>\n      <td>0.432520</td>\n      <td>0.974076</td>\n      <td>0.894935</td>\n      <td>0.410939</td>\n      <td>0.487212</td>\n      <td>0.428583</td>\n      <td>0.598402</td>\n      <td>0.706644</td>\n      <td>0.385664</td>\n      <td>0.493396</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0.438189</td>\n      <td>0.477055</td>\n      <td>0.292263</td>\n      <td>0.737119</td>\n      <td>0.986888</td>\n      <td>0.571864</td>\n      <td>0.952132</td>\n      <td>0.589837</td>\n      <td>0.010117</td>\n      <td>0.174665</td>\n      <td>...</td>\n      <td>0.301537</td>\n      <td>0.552984</td>\n      <td>0.879647</td>\n      <td>0.380063</td>\n      <td>0.300516</td>\n      <td>0.398224</td>\n      <td>0.739755</td>\n      <td>0.462363</td>\n      <td>0.046500</td>\n      <td>0.510257</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.246476</td>\n      <td>0.425691</td>\n      <td>0.344394</td>\n      <td>0.302752</td>\n      <td>0.592942</td>\n      <td>0.350859</td>\n      <td>0.620692</td>\n      <td>0.516486</td>\n      <td>0.033306</td>\n      <td>0.304789</td>\n      <td>...</td>\n      <td>0.131419</td>\n      <td>0.591061</td>\n      <td>0.840573</td>\n      <td>0.664932</td>\n      <td>0.472257</td>\n      <td>0.998087</td>\n      <td>0.073214</td>\n      <td>0.984443</td>\n      <td>0.957306</td>\n      <td>0.745294</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.303911</td>\n      <td>0.326860</td>\n      <td>0.147415</td>\n      <td>0.208054</td>\n      <td>0.371225</td>\n      <td>0.910451</td>\n      <td>0.544563</td>\n      <td>0.394877</td>\n      <td>0.424432</td>\n      <td>0.418725</td>\n      <td>...</td>\n      <td>0.556135</td>\n      <td>0.013987</td>\n      <td>0.149688</td>\n      <td>0.604044</td>\n      <td>0.574204</td>\n      <td>0.930515</td>\n      <td>0.797487</td>\n      <td>0.785470</td>\n      <td>0.447085</td>\n      <td>0.587239</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.885194</td>\n      <td>0.156615</td>\n      <td>0.795850</td>\n      <td>0.208292</td>\n      <td>0.658539</td>\n      <td>0.738021</td>\n      <td>0.750869</td>\n      <td>0.233611</td>\n      <td>0.890301</td>\n      <td>0.501873</td>\n      <td>...</td>\n      <td>0.149136</td>\n      <td>0.617435</td>\n      <td>0.659965</td>\n      <td>0.096339</td>\n      <td>0.922107</td>\n      <td>0.220996</td>\n      <td>0.989596</td>\n      <td>0.604612</td>\n      <td>0.461251</td>\n      <td>0.658624</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows \u00d7 100 columns</p>\n</div>", 
                        "text/plain": "          0         1         2         3         4         5         6  \\\n0  0.218387  0.681619  0.127457  0.176184  0.208149  0.431590  0.436492   \n1  0.438189  0.477055  0.292263  0.737119  0.986888  0.571864  0.952132   \n2  0.246476  0.425691  0.344394  0.302752  0.592942  0.350859  0.620692   \n3  0.303911  0.326860  0.147415  0.208054  0.371225  0.910451  0.544563   \n4  0.885194  0.156615  0.795850  0.208292  0.658539  0.738021  0.750869   \n\n          7         8         9    ...           90        91        92  \\\n0  0.346246  0.632588  0.829314    ...     0.432520  0.974076  0.894935   \n1  0.589837  0.010117  0.174665    ...     0.301537  0.552984  0.879647   \n2  0.516486  0.033306  0.304789    ...     0.131419  0.591061  0.840573   \n3  0.394877  0.424432  0.418725    ...     0.556135  0.013987  0.149688   \n4  0.233611  0.890301  0.501873    ...     0.149136  0.617435  0.659965   \n\n         93        94        95        96        97        98        99  \n0  0.410939  0.487212  0.428583  0.598402  0.706644  0.385664  0.493396  \n1  0.380063  0.300516  0.398224  0.739755  0.462363  0.046500  0.510257  \n2  0.664932  0.472257  0.998087  0.073214  0.984443  0.957306  0.745294  \n3  0.604044  0.574204  0.930515  0.797487  0.785470  0.447085  0.587239  \n4  0.096339  0.922107  0.220996  0.989596  0.604612  0.461251  0.658624  \n\n[5 rows x 100 columns]"
                    }, 
                    "metadata": {}, 
                    "output_type": "execute_result"
                }
            ]
        }
    ], 
    "metadata": {
        "kernelspec": {
            "display_name": "Python 2 with Spark 1.6", 
            "language": "python", 
            "name": "python2"
        }, 
        "language_info": {
            "codemirror_mode": {
                "version": 2, 
                "name": "ipython"
            }, 
            "version": "2.7.11", 
            "name": "python", 
            "pygments_lexer": "ipython2", 
            "nbconvert_exporter": "python", 
            "mimetype": "text/x-python", 
            "file_extension": ".py"
        }
    }, 
    "nbformat_minor": 0
}