Python/json_parser.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def set_hadoop_config(credentials):\n",
    "    prefix = \"fs.swift.service.\" + credentials['name'] \n",
    "    hconf = sc._jsc.hadoopConfiguration()\n",
    "    hconf.set(prefix + \".auth.url\", credentials['auth_url']+'/v3/auth/tokens')\n",
    "    hconf.set(prefix + \".auth.endpoint.prefix\", \"endpoints\")\n",
    "    hconf.set(prefix + \".tenant\", credentials['project_id'])\n",
    "    hconf.set(prefix + \".username\", credentials['user_id'])\n",
    "    hconf.set(prefix + \".password\", credentials['password'])\n",
    "    hconf.setInt(prefix + \".http.port\", 8080)\n",
    "    hconf.set(prefix + \".region\", credentials['region'])\n",
    "    hconf.setBoolean(prefix + \".public\", True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "objectStorageCreds = {\n",
    "  'auth_uri':'',\n",
    "  'global_account_auth_uri':'',\n",
    "  'username':'Admin_5b8faff871d87700d342130e7cc52e06a7019feb',\n",
    "  'password':\"\"\"XXXXXXXXXXXX\"\"\",\n",
    "  'auth_url':'https://identity.open.softlayer.com',\n",
    "  'project':'object_storage_216c032f_3f57_4763_ae97_5c6a83a0d523',\n",
    "  'project_id':'e097bbd898534ed1ad0e45c82baedb2d',\n",
    "  'region':'dallas',\n",
    "  'user_id':'a493676500794827b020874099c5ee1c',\n",
    "  'domain_id':'da5b6dd1c8374f67b1050172badbef8c',\n",
    "  'domain_name':'837523',\n",
    "  'filename':'meta_Grocery_and_Gourmet_Food.json',\n",
    "  'container':'1464138585443',\n",
    "  'tenantId':'sf58-a0340cf1eaefe8-2c631c8ff999'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "objectStorageCreds['name'] = 'testcredname'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "set_hadoop_config(objectStorageCreds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.sql import SQLContext\n",
    "sqlContext = SQLContext(sc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = sqlContext.jsonFile(\"swift://\" + objectStorageCreds['container'] + \".\" + objectStorageCreds['name'] + \"/\" + objectStorageCreds['filename'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- asin: string (nullable = true)\n",
      " |-- brand: string (nullable = true)\n",
      " |-- categories: array (nullable = true)\n",
      " |    |-- element: array (containsNull = true)\n",
      " |    |    |-- element: string (containsNull = true)\n",
      " |-- description: string (nullable = true)\n",
      " |-- imUrl: string (nullable = true)\n",
      " |-- price: double (nullable = true)\n",
      " |-- related: struct (nullable = true)\n",
      " |    |-- also_bought: array (nullable = true)\n",
      " |    |    |-- element: string (containsNull = true)\n",
      " |    |-- also_viewed: array (nullable = true)\n",
      " |    |    |-- element: string (containsNull = true)\n",
      " |    |-- bought_together: array (nullable = true)\n",
      " |    |    |-- element: string (containsNull = true)\n",
      " |    |-- buy_after_viewing: array (nullable = true)\n",
      " |    |    |-- element: string (containsNull = true)\n",
      " |-- salesRank: struct (nullable = true)\n",
      " |    |-- Arts, Crafts & Sewing: long (nullable = true)\n",
      " |    |-- Automotive: long (nullable = true)\n",
      " |    |-- Beauty: long (nullable = true)\n",
      " |    |-- Camera &amp; Photo: long (nullable = true)\n",
      " |    |-- Cell Phones & Accessories: long (nullable = true)\n",
      " |    |-- Clothing: long (nullable = true)\n",
      " |    |-- Computers & Accessories: long (nullable = true)\n",
      " |    |-- Electronics: long (nullable = true)\n",
      " |    |-- Grocery & Gourmet Food: long (nullable = true)\n",
      " |    |-- Health & Personal Care: long (nullable = true)\n",
      " |    |-- Home &amp; Kitchen: long (nullable = true)\n",
      " |    |-- Home Improvement: long (nullable = true)\n",
      " |    |-- Industrial & Scientific: long (nullable = true)\n",
      " |    |-- Jewelry: long (nullable = true)\n",
      " |    |-- Kitchen & Dining: long (nullable = true)\n",
      " |    |-- Movies & TV: long (nullable = true)\n",
      " |    |-- Music: long (nullable = true)\n",
      " |    |-- Office Products: long (nullable = true)\n",
      " |    |-- Patio, Lawn & Garden: long (nullable = true)\n",
      " |    |-- Pet Supplies: long (nullable = true)\n",
      " |    |-- Shoes: long (nullable = true)\n",
      " |    |-- Software: long (nullable = true)\n",
      " |    |-- Sports &amp; Outdoors: long (nullable = true)\n",
      " |    |-- Toys & Games: long (nullable = true)\n",
      " |    |-- Video Games: long (nullable = true)\n",
      " |-- title: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(asin=u'0657745316', brand=None, categories=[[u'Grocery & Gourmet Food']], description=u'This is real vanilla extract made with only 3 premium ingredients. GMO free, no fillers you find in store bought \"vanilla extract.\" \\n\\nThe taste will knock your socks off. Everyone will notice a difference in your baking and cooking and they\\'ll want to know your secret. I also use this for a special homemade coffee creamer that\\'s out of this world and I use it for tea and black coffee as well as espresso drinks. You can add this to make a vanilla latte and skip the sugary syrup for a healthier latte with more flavor! \\n\\nWhen this item arrives, there will also be instructions to refill the product as its used so that you won\\'t have to age it or repurchase it. I\\'ve been using the same vanilla for 2 years now and have friends who\\'ve had theirs for 5 years. It\\'s just as tasteful, just as sweet, strong, and yummy. \\n\\nI use only top shelf liquor and the product is aged a minimum of 4 months. \\n\\nThese also make great gifts. I currently have plenty on the shelf but they won\\'t be ready to use until February (just in time to bake your honey some sweets). Contact me for details. I also make designer jars, and can add any decorative details such as ribbon upon request. Please note, custom orders will require longer shipping and prices will vary depending on how detailed you\\'d like the jar or ribbons etc. \\n\\nMost of these will come in a small mason jar which contains approx 12-16 ounces of extract. The beans are right in the jar and I always throw in extra beans so your product will stay much longer for use in refilling. This product gets sweeter with time.. Like a fine wine it gets better with age.. So the longer you have it on the shelf the better it continues to get.', imUrl=u'http://ecx.images-amazon.com/images/I/41gFi5h0jYL._SY300_.jpg', price=None, related=Row(also_bought=None, also_viewed=[u'B001GE8N4Y'], bought_together=None, buy_after_viewing=None), salesRank=Row(Arts, Crafts & Sewing=None, Automotive=None, Beauty=None, Camera &amp; Photo=None, Cell Phones & Accessories=None, Clothing=None, Computers & Accessories=None, Electronics=None, Grocery & Gourmet Food=374004, Health & Personal Care=None, Home &amp; Kitchen=None, Home Improvement=None, Industrial & Scientific=None, Jewelry=None, Kitchen & Dining=None, Movies & TV=None, Music=None, Office Products=None, Patio, Lawn & Garden=None, Pet Supplies=None, Shoes=None, Software=None, Sports &amp; Outdoors=None, Toys & Games=None, Video Games=None), title=u'100 Percent All Natural Vanilla Extract'),\n",
       " Row(asin=u'0700026444', brand=None, categories=[[u'Grocery & Gourmet Food']], description=u'Silverpot Tea, Pure Darjeeling, is an exquisite tea enjoyed at leisure, when you want to relax or celebrate.   Its mellow yet layered taste will constantly surprise and delight.  This subtle and refined tea is of incomparable taste and flavor.  \\n\\nPackaged in an exotic handmade pinewood chestlet, this magnificent tea was rolled from tender leaves grown and hand plucked in the legendary mist covered fields of Darjeeling.\\n\\nFor authentic experience of this luxury tea, it is presented as 100 gms  loose leaf, and is a great way to show your appreciation, celebrate an occasion or send your best wishes.\\n\\nSilverpot is different - it embraces the pioneering spirit of the Ghose family, personifies the highest standards of excellence, and is proof that this is only possible from the love and care of a family tea company located at the country of origin, with a tea tradition of five generations.The Silverpot philosophy is, first and foremost, about a passionatecommitment to quality and a series of uncompromising choices. These, taken together, create a taste anda style that is as legendary as it is unique. Silverpot Tea balances richness, freshness andfinesse, and is nurtured from the leaf to the cup with painstaking care and attention to detail.HERITAGE.Silverpot is born of a cherished heritage dating back to 1879, when tea was still a monopoly of the colonial rulers, and the Ghose family established Indias first native owned tea plantation. The legacy of Debes Chandra Ghose and his family continues till this day. Each expression of Silverpot draws on the companys long history, rich traditions and legendary dedication to quality teas.', imUrl=u'http://ecx.images-amazon.com/images/I/51hs8sox%2BJL._SY300_.jpg', price=None, related=None, salesRank=Row(Arts, Crafts & Sewing=None, Automotive=None, Beauty=None, Camera &amp; Photo=None, Cell Phones & Accessories=None, Clothing=None, Computers & Accessories=None, Electronics=None, Grocery & Gourmet Food=620307, Health & Personal Care=None, Home &amp; Kitchen=None, Home Improvement=None, Industrial & Scientific=None, Jewelry=None, Kitchen & Dining=None, Movies & TV=None, Music=None, Office Products=None, Patio, Lawn & Garden=None, Pet Supplies=None, Shoes=None, Software=None, Sports &amp; Outdoors=None, Toys & Games=None, Video Games=None), title=u'Pure Darjeeling Tea: Loose Leaf'),\n",
       " Row(asin=u'1403796890', brand=None, categories=[[u'Grocery & Gourmet Food']], description=u'Must have for any WWE Fan\\n \\n \\n \\nFeaturing John Cena slippers \\n\\nmade of velvet\\n \\ncolor Black\\n \\nsize 9/10 kids aprox 6 inches long\\n \\nGreat for gifts\\n \\nparty favors', imUrl=u'http://ecx.images-amazon.com/images/I/518SEST5DqL._SY300_.jpg', price=3.99, related=None, salesRank=None, title=u'WWE Kids Todler Velvet Slippers featuring John Cena size 7/8 Infants COLOR Black -'),\n",
       " Row(asin=u'141278509X', brand=None, categories=[[u'Grocery & Gourmet Food']], description=u'Infused with Vitamins and Electrolytes Good Source of Calcium', imUrl=u'http://ecx.images-amazon.com/images/I/51CFQIisucL._SY300_.jpg', price=None, related=Row(also_bought=None, also_viewed=[u'B0051IETTY'], bought_together=None, buy_after_viewing=None), salesRank=Row(Arts, Crafts & Sewing=None, Automotive=None, Beauty=None, Camera &amp; Photo=None, Cell Phones & Accessories=None, Clothing=None, Computers & Accessories=None, Electronics=None, Grocery & Gourmet Food=620322, Health & Personal Care=None, Home &amp; Kitchen=None, Home Improvement=None, Industrial & Scientific=None, Jewelry=None, Kitchen & Dining=None, Movies & TV=None, Music=None, Office Products=None, Patio, Lawn & Garden=None, Pet Supplies=None, Shoes=None, Software=None, Sports &amp; Outdoors=None, Toys & Games=None, Video Games=None), title=u'Archer Farms Strawberry Dragonfruit Drink Mix 8-0.16 Oz Packets'),\n",
       " Row(asin=u'1453060375', brand=u'Mio', categories=[[u'Grocery & Gourmet Food']], description=u'MiO Energy is your portable energy source giving you energy whenever and wherever you need it!\\nWhen one squirt (about half a teaspoon) is mixed into 8 ounces of water, it creates a delicious drink that contains 60 mg of caffeine - about the same as a 6-ounce cup of coffee, as well as B vitamins that help release energy from the food you eat. \\n*BPA Free - Contains 60mg/ caffeine per serving - Contains 0% Juice - Not for children - Do not add to caffeinated or alcoholic drinks', imUrl=u'http://ecx.images-amazon.com/images/I/51EUsMcnVVL._SX300_.jpg', price=11.99, related=Row(also_bought=None, also_viewed=[u'B006MSEOJ2', u'B005VOOQLO', u'B005VOOQKA', u'B00C121U9A', u'B00CEPYH6W', u'B009T6U1WS', u'B007SVF26G', u'B007EP4EA6', u'B007XFFKTQ', u'B00933QC7A', u'B00933R96I'], bought_together=None, buy_after_viewing=None), salesRank=Row(Arts, Crafts & Sewing=None, Automotive=None, Beauty=None, Camera &amp; Photo=None, Cell Phones & Accessories=None, Clothing=None, Computers & Accessories=None, Electronics=None, Grocery & Gourmet Food=268754, Health & Personal Care=None, Home &amp; Kitchen=None, Home Improvement=None, Industrial & Scientific=None, Jewelry=None, Kitchen & Dining=None, Movies & TV=None, Music=None, Office Products=None, Patio, Lawn & Garden=None, Pet Supplies=None, Shoes=None, Software=None, Sports &amp; Outdoors=None, Toys & Games=None, Video Games=None), title=u'Mio Energy Liquid Water Enhancer Black Cherry &amp; Green Thunder 1.62oz')]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.take(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}