Project-boto/AWS-SDK-for-python-boto3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using AWS SDK for python the boto libraty"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Boto is a python library that enables usage of AWS serivces programatically. \n",
    "It is one of many SDK that AWS offers - [AWS_SDK-s](https://aws.amazon.com/developer/tools/?nc1=f_dr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "import pandas as pa"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Keep your keys in a safe place. Load them into a variable. Never expose the keys in code. Regenerate your keys regularely and delete all unneeded ones. Here I am loading the keys from the csv file, which I downloaded on creation of a user in IAM."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load keys from vault\n",
    "AWS_KEY_ID = pa.read_csv('../../AWS_vault/credentials_boto_test_2.csv')['Access key ID'][0]\n",
    "AWS_SECRET = pa.read_csv('../../AWS_vault/credentials_boto_test_2.csv')['Secret access key'][0]\n",
    "\n",
    "# define your preferred region\n",
    "my_region = 'eu-central-1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate the boto3 client for interacting with S3 and SNS\n",
    "s3 = boto3.client('s3',  region_name= my_region,\n",
    "                         aws_access_key_id=AWS_KEY_ID, \n",
    "                         aws_secret_access_key=AWS_SECRET)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Creating an S3 bucket, uploading objects, listing objects, deleting objects, deleting bucket"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Buckets': [{'CreationDate': datetime.datetime(2020, 1, 29, 12, 49, 47, tzinfo=tzutc()),\n",
       "   'Name': 'test-bokeh'}],\n",
       " 'Owner': {'ID': 'd14ca27c07aac3d80654a5143be2aaa7c7e40b70f5c2c5564807e54b96cbe49d'},\n",
       " 'ResponseMetadata': {'HTTPHeaders': {'content-type': 'application/xml',\n",
       "   'date': 'Mon, 03 Feb 2020 09:15:29 GMT',\n",
       "   'server': 'AmazonS3',\n",
       "   'transfer-encoding': 'chunked',\n",
       "   'x-amz-id-2': 'Xu0sOSiLfjttQirepMMNLyjP4ouJuCtyuMn9rGbdu1CPhDMBSXSDQGfHE+6o5jqjNOupdJV3CUI=',\n",
       "   'x-amz-request-id': '61361378548CF413'},\n",
       "  'HTTPStatusCode': 200,\n",
       "  'HostId': 'Xu0sOSiLfjttQirepMMNLyjP4ouJuCtyuMn9rGbdu1CPhDMBSXSDQGfHE+6o5jqjNOupdJV3CUI=',\n",
       "  'RequestId': '61361378548CF413',\n",
       "  'RetryAttempts': 0}}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# List S3 buckets \n",
    "s3.list_buckets()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Location': 'http://test-bokeh.s3.amazonaws.com/', 'ResponseMetadata': {'HTTPHeaders': {'x-amz-id-2': 'QcpMcRJprshLUHc7d7leaJO/gZdlDvIlONNaklvkC30Xxy9DAtYi9jsszqr+UpF21Vg1YBwrHp8=', 'content-length': '0', 'x-amz-request-id': '851F18682F2A3C1D', 'server': 'AmazonS3', 'date': 'Wed, 29 Jan 2020 12:49:46 GMT', 'location': 'http://test-bokeh.s3.amazonaws.com/'}, 'RequestId': '851F18682F2A3C1D', 'HostId': 'QcpMcRJprshLUHc7d7leaJO/gZdlDvIlONNaklvkC30Xxy9DAtYi9jsszqr+UpF21Vg1YBwrHp8=', 'RetryAttempts': 0, 'HTTPStatusCode': 200}}\n"
     ]
    }
   ],
   "source": [
    "# Create the buckets\n",
    "response_staging = s3.create_bucket(Bucket='test-bokeh',\n",
    "                                    CreateBucketConfiguration={'LocationConstraint': my_region})\n",
    "\n",
    "# Print out the response\n",
    "print(response_staging)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Upload a file\n",
    "s3.upload_file(Bucket='test-bokeh',\n",
    "              # Set filename and key\n",
    "               Filename='final_report.csv', \n",
    "               Key='final_report.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test-bokeh\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'Contents': [{'ETag': '\"2143b2e98c7759312732afd7dadb8fda\"',\n",
       "   'Key': 'final_report.csv',\n",
       "   'LastModified': datetime.datetime(2020, 1, 29, 12, 50, 42, tzinfo=tzutc()),\n",
       "   'Owner': {'ID': 'd14ca27c07aac3d80654a5143be2aaa7c7e40b70f5c2c5564807e54b96cbe49d'},\n",
       "   'Size': 199,\n",
       "   'StorageClass': 'STANDARD'},\n",
       "  {'ETag': '\"e8a346159ac66da8f9c25217543225f6\"',\n",
       "   'Key': 'final_report.html',\n",
       "   'LastModified': datetime.datetime(2020, 1, 29, 13, 58, tzinfo=tzutc()),\n",
       "   'Owner': {'ID': 'd14ca27c07aac3d80654a5143be2aaa7c7e40b70f5c2c5564807e54b96cbe49d'},\n",
       "   'Size': 1606,\n",
       "   'StorageClass': 'STANDARD'},\n",
       "  {'ETag': '\"2143b2e98c7759312732afd7dadb8fda\"',\n",
       "   'Key': 'final_report_.csv',\n",
       "   'LastModified': datetime.datetime(2020, 2, 3, 9, 36, 23, tzinfo=tzutc()),\n",
       "   'Owner': {'ID': 'd14ca27c07aac3d80654a5143be2aaa7c7e40b70f5c2c5564807e54b96cbe49d'},\n",
       "   'Size': 199,\n",
       "   'StorageClass': 'STANDARD'}],\n",
       " 'EncodingType': 'url',\n",
       " 'IsTruncated': False,\n",
       " 'Marker': '',\n",
       " 'MaxKeys': 1000,\n",
       " 'Name': 'test-bokeh',\n",
       " 'Prefix': 'final',\n",
       " 'ResponseMetadata': {'HTTPHeaders': {'content-type': 'application/xml',\n",
       "   'date': 'Mon, 03 Feb 2020 09:36:57 GMT',\n",
       "   'server': 'AmazonS3',\n",
       "   'transfer-encoding': 'chunked',\n",
       "   'x-amz-bucket-region': 'eu-central-1',\n",
       "   'x-amz-id-2': 'BkhDNL9k1n5Oi/YU16XHyNrJnbnU47bpEN3cDFaNqZM66JnRyejtwPaKl1uHZq2aUFie0sTv7kE=',\n",
       "   'x-amz-request-id': 'EA939425D61967F9'},\n",
       "  'HTTPStatusCode': 200,\n",
       "  'HostId': 'BkhDNL9k1n5Oi/YU16XHyNrJnbnU47bpEN3cDFaNqZM66JnRyejtwPaKl1uHZq2aUFie0sTv7kE=',\n",
       "  'RequestId': 'EA939425D61967F9',\n",
       "  'RetryAttempts': 0}}"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get the list_buckets response\n",
    "response = s3.list_buckets()\n",
    "\n",
    "# Iterate over Buckets from .list_buckets() response\n",
    "for bucket in response['Buckets']:\n",
    "    print(bucket['Name'])\n",
    "\n",
    "\n",
    "# Get object metadata\n",
    "response = s3.list_objects(Bucket='test-bokeh', \n",
    "                           Prefix='final')\n",
    "response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'final_report.csv'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get files in the bucket using keys\n",
    "response.get('Contents')[0].get('Key')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['final_report.csv',\n",
       " 'final_report.html',\n",
       " 'final_report_.csv',\n",
       " 'indes.html',\n",
       " 'index.html',\n",
       " 'slider-test.html',\n",
       " 'st',\n",
       " 'st-boxes.png',\n",
       " 'statistics_vs_machine_learning.png',\n",
       " 'test-range-tool.html']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# List all objects in the bucket'               \n",
    "[obj['Key'] for obj in s3.list_objects(Bucket='test-bokeh')['Contents']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['final_report.csv', 'final_report_.csv']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# list out specified files \n",
    "[obj['Key'] for obj in s3.list_objects(Bucket='test-bokeh')['Contents'] if obj['Key'].endswith('csv')]    "
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# delete all objects in the bucket\n",
    "for obj in response['Contents']:\n",
    "    s3.delete_object(Bucket='test-bokeh', Key=obj['Key'])\n",
    "\n",
    "# or \n",
    "\n",
    "# delete a bucket with all of its content\n",
    "s3.delete_bucket(Bucket='somebucket')\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Security and permissions\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://test-bokeh.s3.amazonaws.com/final_report.csv\n",
      "https://test-bokeh.s3.amazonaws.com/final_report.html\n",
      "https://test-bokeh.s3.amazonaws.com/final_report_.csv\n"
     ]
    }
   ],
   "source": [
    "# Upload a file specifying its ACL rule \n",
    "\n",
    "s3.upload_file(\n",
    "    Bucket='test-bokeh',\n",
    "    Filename='final_report.csv', \n",
    "    Key='final_report_.csv',\n",
    "    ExtraArgs = {'ACL': 'public-read'})\n",
    "\n",
    "# Iterate over the objects\n",
    "for obj in response['Contents']:\n",
    "    s3.put_object_acl(Bucket='test-bokeh', \n",
    "                      Key=obj['Key'], \n",
    "                      ACL='public-read')\n",
    "    \n",
    "    #print out clickable link to the files \n",
    "    print(\"https://{}.s3.amazonaws.com/{}\".format( 'test-bokeh', obj['Key']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://test-bokeh.s3.amazonaws.com/final_report.csv?X-Amz-Expires=3600&X-Amz-SignedHeaders=host&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20200130T170329Z&X-Amz-Credential=AKIAZLPDXHVMJ4IZU7GO%2F20200130%2Feu-central-1%2Fs3%2Faws4_request&X-Amz-Signature=51c49344afab9245a1f9236ce17a83290b9f952d9cfc1fe787753d5a629f72ab\n"
     ]
    }
   ],
   "source": [
    "# Generate presigned_url for the uploaded object\n",
    "share_url = s3.generate_presigned_url(\n",
    "    # Specify allowable operations\n",
    "    ClientMethod='get_object',\n",
    "    # Set the expiration time\n",
    "    ExpiresIn=3600,\n",
    "    # Set bucket and shareable object's name\n",
    "    Params={'Bucket': 'test-bokeh',\n",
    "            'Key': 'final_report.csv'})\n",
    "\n",
    "# Print out the presigned URL\n",
    "print(share_url)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Retrieveing files from S3 with get_object - Use cloud, but work locally \n",
    "In this case the csv files will be downloaded and stored in a dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'ETag': '\"2143b2e98c7759312732afd7dadb8fda\"',\n",
       "  'Key': 'final_report.csv',\n",
       "  'LastModified': datetime.datetime(2020, 1, 29, 12, 50, 42, tzinfo=tzutc()),\n",
       "  'Owner': {'ID': 'd14ca27c07aac3d80654a5143be2aaa7c7e40b70f5c2c5564807e54b96cbe49d'},\n",
       "  'Size': 199,\n",
       "  'StorageClass': 'STANDARD'},\n",
       " {'ETag': '\"e8a346159ac66da8f9c25217543225f6\"',\n",
       "  'Key': 'final_report.html',\n",
       "  'LastModified': datetime.datetime(2020, 1, 29, 13, 58, tzinfo=tzutc()),\n",
       "  'Owner': {'ID': 'd14ca27c07aac3d80654a5143be2aaa7c7e40b70f5c2c5564807e54b96cbe49d'},\n",
       "  'Size': 1606,\n",
       "  'StorageClass': 'STANDARD'},\n",
       " {'ETag': '\"2143b2e98c7759312732afd7dadb8fda\"',\n",
       "  'Key': 'final_report_.csv',\n",
       "  'LastModified': datetime.datetime(2020, 2, 3, 9, 21, 3, tzinfo=tzutc()),\n",
       "  'Owner': {'ID': 'd14ca27c07aac3d80654a5143be2aaa7c7e40b70f5c2c5564807e54b96cbe49d'},\n",
       "  'Size': 199,\n",
       "  'StorageClass': 'STANDARD'},\n",
       " {'ETag': '\"f71a01ee9dbb21fa37dcacfd2986d981\"',\n",
       "  'Key': 'indes.html',\n",
       "  'LastModified': datetime.datetime(2020, 1, 31, 17, 38, 54, tzinfo=tzutc()),\n",
       "  'Owner': {'ID': 'd14ca27c07aac3d80654a5143be2aaa7c7e40b70f5c2c5564807e54b96cbe49d'},\n",
       "  'Size': 1220,\n",
       "  'StorageClass': 'STANDARD'},\n",
       " {'ETag': '\"e2faf239a98fc5e5809bcae0e55ebb26\"',\n",
       "  'Key': 'index.html',\n",
       "  'LastModified': datetime.datetime(2020, 1, 31, 17, 50, 27, tzinfo=tzutc()),\n",
       "  'Owner': {'ID': 'd14ca27c07aac3d80654a5143be2aaa7c7e40b70f5c2c5564807e54b96cbe49d'},\n",
       "  'Size': 1481,\n",
       "  'StorageClass': 'STANDARD'},\n",
       " {'ETag': '\"70afce88968ae6dbedb785b4d70cc5d0\"',\n",
       "  'Key': 'slider-test.html',\n",
       "  'LastModified': datetime.datetime(2020, 1, 30, 17, 17, 20, tzinfo=tzutc()),\n",
       "  'Owner': {'ID': 'd14ca27c07aac3d80654a5143be2aaa7c7e40b70f5c2c5564807e54b96cbe49d'},\n",
       "  'Size': 18551,\n",
       "  'StorageClass': 'STANDARD'},\n",
       " {'ETag': '\"0fe4fb22f73681d9adaa10719a879552\"',\n",
       "  'Key': 'st',\n",
       "  'LastModified': datetime.datetime(2020, 1, 31, 18, 54, 10, tzinfo=tzutc()),\n",
       "  'Owner': {'ID': 'd14ca27c07aac3d80654a5143be2aaa7c7e40b70f5c2c5564807e54b96cbe49d'},\n",
       "  'Size': 802875,\n",
       "  'StorageClass': 'STANDARD'},\n",
       " {'ETag': '\"7d510c081d16a655c9a11bc08459e36a\"',\n",
       "  'Key': 'st-boxes.png',\n",
       "  'LastModified': datetime.datetime(2020, 1, 31, 19, 35, 49, tzinfo=tzutc()),\n",
       "  'Owner': {'ID': 'd14ca27c07aac3d80654a5143be2aaa7c7e40b70f5c2c5564807e54b96cbe49d'},\n",
       "  'Size': 4610059,\n",
       "  'StorageClass': 'STANDARD'},\n",
       " {'ETag': '\"a887257a11fe0f85040612a7ef4dadf6\"',\n",
       "  'Key': 'statistics_vs_machine_learning.png',\n",
       "  'LastModified': datetime.datetime(2020, 1, 31, 18, 43, 59, tzinfo=tzutc()),\n",
       "  'Owner': {'ID': 'd14ca27c07aac3d80654a5143be2aaa7c7e40b70f5c2c5564807e54b96cbe49d'},\n",
       "  'Size': 109776,\n",
       "  'StorageClass': 'STANDARD'},\n",
       " {'ETag': '\"6e4250026e0c66962fa2061205be5a3b\"',\n",
       "  'Key': 'test-range-tool.html',\n",
       "  'LastModified': datetime.datetime(2020, 1, 29, 14, 32, 24, tzinfo=tzutc()),\n",
       "  'Owner': {'ID': 'd14ca27c07aac3d80654a5143be2aaa7c7e40b70f5c2c5564807e54b96cbe49d'},\n",
       "  'Size': 66165,\n",
       "  'StorageClass': 'STANDARD'}]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "request_files = s3.list_objects(Bucket='test-bokeh')['Contents']\n",
    "request_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'AcceptRanges': 'bytes',\n",
       " 'Body': <botocore.response.StreamingBody at 0x7f6bb1b83828>,\n",
       " 'ContentLength': 199,\n",
       " 'ContentType': 'binary/octet-stream',\n",
       " 'ETag': '\"2143b2e98c7759312732afd7dadb8fda\"',\n",
       " 'LastModified': datetime.datetime(2020, 1, 29, 12, 50, 42, tzinfo=tzutc()),\n",
       " 'Metadata': {},\n",
       " 'ResponseMetadata': {'HTTPHeaders': {'accept-ranges': 'bytes',\n",
       "   'content-length': '199',\n",
       "   'content-type': 'binary/octet-stream',\n",
       "   'date': 'Mon, 03 Feb 2020 09:23:38 GMT',\n",
       "   'etag': '\"2143b2e98c7759312732afd7dadb8fda\"',\n",
       "   'last-modified': 'Wed, 29 Jan 2020 12:50:42 GMT',\n",
       "   'server': 'AmazonS3',\n",
       "   'x-amz-id-2': 'BP7pDAyWGvpopSYZMIc3v4CZPirekhfhjRva9F00y4/N4Ri8gHqcNgU/R5+Vw0gIy69uupaDRGg=',\n",
       "   'x-amz-request-id': 'AE187EFAC3A026DA'},\n",
       "  'HTTPStatusCode': 200,\n",
       "  'HostId': 'BP7pDAyWGvpopSYZMIc3v4CZPirekhfhjRva9F00y4/N4Ri8gHqcNgU/R5+Vw0gIy69uupaDRGg=',\n",
       "  'RequestId': 'AE187EFAC3A026DA',\n",
       "  'RetryAttempts': 0}}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s3.get_object(Bucket='test-bokeh', Key=request_files[0]['Key'])#['Body']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Service_name</th>\n",
       "      <th>request_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>72 Hour Violation</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Graffiti Removal</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Missed Collection</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Street Light Out</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Pothole</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Service_name  request_count\n",
       "0  72 Hour Violation              8\n",
       "1   Graffiti Removal              2\n",
       "2  Missed Collection             12\n",
       "3   Street Light Out             21\n",
       "4            Pothole             33"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# obtain a file from S3 and load it into a dataframe\n",
    "\n",
    "df_list = [] \n",
    "# Load each object from s3\n",
    "for file in request_files:\n",
    "    s3_day_reqs = s3.get_object(Bucket='test-bokeh', \n",
    "                                Key=file['Key'])\n",
    "    if file['Key'].endswith('csv'):\n",
    "    # Read the DataFrame into pandas, append it to the list\n",
    "        day_reqs = pa.read_csv(s3_day_reqs['Body'])\n",
    "        df_list.append(day_reqs)\n",
    "\n",
    "# Concatenate all the DataFrames in the list\n",
    "all_reqs = pa.concat(df_list)\n",
    "\n",
    "# Preview the DataFrame\n",
    "all_reqs.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Sharing file through a website - S3 serving static html web pages\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generating links to S3 object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate an html table from the df using pandas \n",
    "df.to_html('final_report.html', \n",
    "           columns=['Service_name', 'request_count'],\n",
    "           border=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "http://test-bokeh.s3.amazonaws.com/final_report.html\n"
     ]
    }
   ],
   "source": [
    "# Upload html file to S3\n",
    "s3.upload_file(Filename='final_report.html', \n",
    "               Bucket='test-bokeh', Key='final_report.html',\n",
    "               ExtraArgs = {\n",
    "                 'ContentType':'text/html',\n",
    "                 'ACL': 'public-read'})\n",
    "\n",
    "# Print the S3 Public Object URL for the new file.\n",
    "print(\"http://{}.s3.amazonaws.com/{}\".format('test-bokeh', 'final_report.html'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plotting in Bokeh and exporting to html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import bokeh\n",
    "\n",
    "# to get all bokeh example datasets uncomment the following line:\n",
    "\n",
    "#bokeh.sampledata.download() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#example from https://docs.bokeh.org/en/latest/docs/gallery.html\n",
    "\n",
    "from bokeh.io import show\n",
    "from bokeh.models import LogColorMapper\n",
    "from bokeh.palettes import Viridis6 as palette\n",
    "from bokeh.plotting import figure\n",
    "from bokeh.sampledata.us_counties import data as counties\n",
    "from bokeh.sampledata.unemployment import data as unemployment\n",
    "\n",
    "palette.reverse()\n",
    "\n",
    "counties = {code: county for code, county in counties.items() if county[\"state\"] == \"tx\"}\n",
    "\n",
    "county_xs = [county[\"lons\"] for county in counties.values()]\n",
    "county_ys = [county[\"lats\"] for county in counties.values()]\n",
    "\n",
    "county_names = [county['name'] for county in counties.values()]\n",
    "county_rates = [unemployment[county_id] for county_id in counties]\n",
    "color_mapper = LogColorMapper(palette=palette)\n",
    "\n",
    "data=dict(\n",
    "    x=county_xs,\n",
    "    y=county_ys,\n",
    "    name=county_names,\n",
    "    rate=county_rates,\n",
    ")\n",
    "\n",
    "TOOLS = \"pan,wheel_zoom,reset,hover,save\"\n",
    "\n",
    "p = figure(\n",
    "    title=\"Texas Unemployment, 2009\", tools=TOOLS,\n",
    "    x_axis_location=None, y_axis_location=None,\n",
    "    tooltips=[\n",
    "        (\"Name\", \"@name\"), (\"Unemployment rate)\", \"@rate%\"), (\"(Long, Lat)\", \"($x, $y)\")\n",
    "    ])\n",
    "p.grid.grid_line_color = None\n",
    "p.hover.point_policy = \"follow_mouse\"\n",
    "\n",
    "p.patches('x', 'y', source=data,\n",
    "          fill_color={'field': 'rate', 'transform': color_mapper},\n",
    "          fill_alpha=0.7, line_color=\"white\", line_width=0.5)\n",
    "\n",
    "show(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dalabsba/v_env_python3/aws_cli/lib/python3.5/site-packages/bokeh/io/saving.py:126: UserWarning: save() called but no resources were supplied and output_file(...) was never called, defaulting to resources.CDN\n",
      "  warn(\"save() called but no resources were supplied and output_file(...) was never called, defaulting to resources.CDN\")\n",
      "/home/dalabsba/v_env_python3/aws_cli/lib/python3.5/site-packages/bokeh/io/saving.py:139: UserWarning: save() called but no title was supplied and output_file(...) was never called, using default title 'Bokeh Plot'\n",
      "  warn(\"save() called but no title was supplied and output_file(...) was never called, using default title 'Bokeh Plot'\")\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'/home/dalabsba/Documents/Projects/30_Udacity/tech-scholar-Bertelsmann/bertelsmann-dsml-group/Project_boto/test.html'"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# export the plot to html\n",
    "bokeh.io.saving.save(p,'test.html', data=data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/dalabsba/Documents/Projects/30_Udacity/tech-scholar-Bertelsmann/bertelsmann-dsml-group/Project_boto/test-range-tool.html'"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# example https://docs.bokeh.org/en/latest/docs/gallery/range_tool.html\n",
    "\n",
    "import numpy as np\n",
    "from bokeh.io import show\n",
    "from bokeh.layouts import column\n",
    "from bokeh.models import ColumnDataSource, RangeTool\n",
    "from bokeh.plotting import figure\n",
    "from bokeh.sampledata.stocks import AAPL\n",
    "\n",
    "dates = np.array(AAPL['date'], dtype=np.datetime64)\n",
    "source = ColumnDataSource(data=dict(date=dates, close=AAPL['adj_close']))\n",
    "\n",
    "p = figure(plot_height=300, plot_width=800, tools=\"xpan\", toolbar_location=None,\n",
    "           x_axis_type=\"datetime\", x_axis_location=\"above\",\n",
    "           background_fill_color=\"#efefef\", x_range=(dates[1500], dates[2500]))\n",
    "\n",
    "p.line('date', 'close', source=source)\n",
    "p.yaxis.axis_label = 'Price'\n",
    "\n",
    "select = figure(title=\"Drag the middle and edges of the selection box to change the range above\",\n",
    "                plot_height=130, plot_width=800, y_range=p.y_range,\n",
    "                x_axis_type=\"datetime\", y_axis_type=None,\n",
    "                tools=\"\", toolbar_location=None, background_fill_color=\"#efefef\")\n",
    "\n",
    "range_tool = RangeTool(x_range=p.x_range)\n",
    "range_tool.overlay.fill_color = \"navy\"\n",
    "range_tool.overlay.fill_alpha = 0.2\n",
    "\n",
    "select.line('date', 'close', source=source)\n",
    "select.ygrid.grid_line_color = None\n",
    "select.add_tools(range_tool)\n",
    "select.toolbar.active_multi = range_tool\n",
    "\n",
    "#show(column(p, select))\n",
    "pp=column(p, select)\n",
    "\n",
    "# convert bokeh visualization to html\n",
    "bokeh.io.saving.save(pp,'test-range-tool.html')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "http://test-bokeh.s3.amazonaws.com/test-range-tool.html\n"
     ]
    }
   ],
   "source": [
    "# Upload bokeh html plots to S3\n",
    "filename=\"test-range-tool.html\"\n",
    "\n",
    "s3.upload_file(Filename=filename, \n",
    "               Bucket='test-bokeh', Key='test-range-tool.html',\n",
    "               ExtraArgs = {\n",
    "                 'ContentType':'text/html',\n",
    "                 'ACL': 'public-read'})\n",
    "\n",
    "# Print the S3 Public Object URL for the new file.\n",
    "print(\"http://{}.s3.amazonaws.com/{}\".format('test-bokeh', filename))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dalabsba/v_env_python3/aws_cli/lib/python3.5/site-packages/bokeh/io/saving.py:126: UserWarning: save() called but no resources were supplied and output_file(...) was never called, defaulting to resources.CDN\n",
      "  warn(\"save() called but no resources were supplied and output_file(...) was never called, defaulting to resources.CDN\")\n",
      "/home/dalabsba/v_env_python3/aws_cli/lib/python3.5/site-packages/bokeh/io/saving.py:139: UserWarning: save() called but no title was supplied and output_file(...) was never called, using default title 'Bokeh Plot'\n",
      "  warn(\"save() called but no title was supplied and output_file(...) was never called, using default title 'Bokeh Plot'\")\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'/home/dalabsba/Documents/Projects/30_Udacity/tech-scholar-Bertelsmann/bertelsmann-dsml-group-projects/Project-boto/slider-test.html'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# example code from https://docs.bokeh.org/en/latest/docs/gallery.html\n",
    "\n",
    "import numpy as np\n",
    "from bokeh.layouts import row, column\n",
    "from bokeh.models import CustomJS, Slider\n",
    "from bokeh.plotting import figure, output_file, show, ColumnDataSource\n",
    "\n",
    "x = np.linspace(0, 10, 500)\n",
    "y = np.sin(x)\n",
    "\n",
    "source = ColumnDataSource(data=dict(x=x, y=y))\n",
    "\n",
    "plot = figure(y_range=(-10, 10), plot_width=400, plot_height=400)\n",
    "\n",
    "plot.line('x', 'y', source=source, line_width=3, line_alpha=0.6)\n",
    "\n",
    "amp_slider = Slider(start=0.1, end=10, value=1, step=.1, title=\"Amplitude\")\n",
    "freq_slider = Slider(start=0.1, end=10, value=1, step=.1, title=\"Frequency\")\n",
    "phase_slider = Slider(start=0, end=6.4, value=0, step=.1, title=\"Phase\")\n",
    "offset_slider = Slider(start=-5, end=5, value=0, step=.1, title=\"Offset\")\n",
    "\n",
    "callback = CustomJS(args=dict(source=source, amp=amp_slider, freq=freq_slider, phase=phase_slider, offset=offset_slider),\n",
    "                    code=\"\"\"\n",
    "    const data = source.data;\n",
    "    const A = amp.value;\n",
    "    const k = freq.value;\n",
    "    const phi = phase.value;\n",
    "    const B = offset.value;\n",
    "    const x = data['x']\n",
    "    const y = data['y']\n",
    "    for (var i = 0; i < x.length; i++) {\n",
    "        y[i] = B + A*Math.sin(k*x[i]+phi);\n",
    "    }\n",
    "    source.change.emit();\n",
    "\"\"\")\n",
    "\n",
    "amp_slider.js_on_change('value', callback)\n",
    "freq_slider.js_on_change('value', callback)\n",
    "phase_slider.js_on_change('value', callback)\n",
    "offset_slider.js_on_change('value', callback)\n",
    "\n",
    "layout = row(\n",
    "    plot,\n",
    "    column(amp_slider, freq_slider, phase_slider, offset_slider),\n",
    ")\n",
    "\n",
    "#output_file(\"slider.html\", title=\"slider.py example\")\n",
    "\n",
    "#show(layout)\n",
    "\n",
    "bokeh.io.saving.save(layout,'slider-test.html')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "http://test-bokeh.s3.amazonaws.com/slider-test.html\n"
     ]
    }
   ],
   "source": [
    "# Upload bokeh html plots to S3\n",
    "filename=\"slider-test.html\"\n",
    "\n",
    "s3.upload_file(Filename=filename, \n",
    "               Bucket='test-bokeh', Key=filename,\n",
    "               ExtraArgs = {\n",
    "                 'ContentType':'text/html',\n",
    "                 'ACL': 'public-read'})\n",
    "\n",
    "# Print the S3 Public Object URL for the new file.\n",
    "print(\"http://{}.s3.amazonaws.com/{}\".format('test-bokeh', filename))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Making an index page with links to all plots "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ResponseMetadata': {'HTTPHeaders': {'date': 'Mon, 03 Feb 2020 09:38:52 GMT',\n",
       "   'server': 'AmazonS3',\n",
       "   'x-amz-id-2': 'wUPEsk1+NKPNE+fUMS4EnWUR+aWOgtQEC8jUrCkRjJ10X8wM7aERcvfET61z+Ec96lMuLvSWrD8=',\n",
       "   'x-amz-request-id': 'E8B80310A556675B'},\n",
       "  'HTTPStatusCode': 204,\n",
       "  'HostId': 'wUPEsk1+NKPNE+fUMS4EnWUR+aWOgtQEC8jUrCkRjJ10X8wM7aERcvfET61z+Ec96lMuLvSWrD8=',\n",
       "  'RequestId': 'E8B80310A556675B',\n",
       "  'RetryAttempts': 0}}"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s3.delete_object(Bucket='test-bokeh', Key='index.html')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 560,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Key</th>\n",
       "      <th>Link</th>\n",
       "      <th>Last modified</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>final_report.html</td>\n",
       "      <td>http://test-bokeh.s3.amazonaws.com/final_report.html</td>\n",
       "      <td>Wed Jan 29 13:58:00 2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>index.html</td>\n",
       "      <td>http://test-bokeh.s3.amazonaws.com/index.html</td>\n",
       "      <td>Mon Feb  3 09:39:26 2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>slider-test.html</td>\n",
       "      <td>http://test-bokeh.s3.amazonaws.com/slider-test.html</td>\n",
       "      <td>Thu Jan 30 17:17:20 2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>test-range-tool.html</td>\n",
       "      <td>http://test-bokeh.s3.amazonaws.com/test-range-tool.html</td>\n",
       "      <td>Wed Jan 29 14:32:24 2020</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    Key  \\\n",
       "0  final_report.html      \n",
       "1  index.html             \n",
       "2  slider-test.html       \n",
       "3  test-range-tool.html   \n",
       "\n",
       "                                                      Link  \\\n",
       "0  http://test-bokeh.s3.amazonaws.com/final_report.html      \n",
       "1  http://test-bokeh.s3.amazonaws.com/index.html             \n",
       "2  http://test-bokeh.s3.amazonaws.com/slider-test.html       \n",
       "3  http://test-bokeh.s3.amazonaws.com/test-range-tool.html   \n",
       "\n",
       "              Last modified  \n",
       "0  Wed Jan 29 13:58:00 2020  \n",
       "1  Mon Feb  3 09:39:26 2020  \n",
       "2  Thu Jan 30 17:17:20 2020  \n",
       "3  Wed Jan 29 14:32:24 2020  "
      ]
     },
     "execution_count": 560,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# why this: \n",
    "pa.set_option('display.max_colwidth', -1)\n",
    "# because https://stackoverflow.com/questions/26277757/pandas-to-html-truncates-string-contents\n",
    "\n",
    "\n",
    "base_url = \"http://test-bokeh.s3.amazonaws.com/\"\n",
    "\n",
    "# List the bucket objects\n",
    "objects_list = s3.list_objects(Bucket='test-bokeh')\n",
    "objects_df=pa.DataFrame(columns=['Key','Link','Last modified'])\n",
    "\n",
    "for obj in s3.list_objects(Bucket='test-bokeh')['Contents']:\n",
    "    if obj['Key'].endswith('html'):\n",
    "        # Convert the response contents to DataFrame\n",
    "        objects_df = objects_df.append({'Link':base_url + obj['Key'],\n",
    "                                        'Key':obj['Key'],\n",
    "                                        'Last modified':obj['LastModified'].ctime()}, ignore_index=True)\n",
    "# Preview the resulting DataFrame\n",
    "objects_df.head()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write objects_df to an HTML file\n",
    "objects_df.to_html('list_of_plots.html',render_links=True, columns=list(objects_df.columns))\n",
    "\n",
    "# Overwrite index.html key by uploading the new file\n",
    "s3.upload_file( Filename='list_of_plots.html', Key='index.html', \n",
    "                Bucket='test-bokeh',\n",
    "                ExtraArgs = {'ContentType': 'text/html', \n",
    "                             'ACL': 'public-read'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 559,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "http://test-bokeh.s3.amazonaws.com/index.html\n"
     ]
    }
   ],
   "source": [
    "for obj in s3.list_objects(Bucket='test-bokeh')['Contents']:\n",
    "    if obj['Key']=='index.html':\n",
    "        # Create a column \"Link\" that contains Public Object URL\n",
    "        print(base_url + obj['Key'])\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}