Permalink
Browse files

updated monitor

  • Loading branch information...
AKSHAYUBHAT committed Jun 24, 2014
1 parent 8bd90f9 commit 2aaa42d857a49aa04d250e0208b25dd6419af358
Showing with 115 additions and 3 deletions.
  1. +115 −3 Monitor.ipynb
View
@@ -1,7 +1,7 @@
{
"metadata": {
"name": "",
"signature": "sha256:4afd7eee052f6857515297c7493a888a2894d9cb8a4827bbe3eeae9ef57edce2"
"signature": "sha256:4d28f1686e8299d8a0ba7db1fd003ffd93eb3bcd3d13a27f35f8ce87780818c0"
},
"nbformat": 3,
"nbformat_minor": 0,
@@ -20,12 +20,124 @@
"cell_type": "code",
"collapsed": false,
"input": [
"import config"
"import random,logging,config,spotinstance\n",
"from boto.s3.connection import S3Connection\n",
"from boto.s3 import key\n",
"from boto.sqs.connection import SQSConnection\n",
"logging.getLogger('boto').setLevel(logging.CRITICAL)"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 1
"prompt_number": 9
},
{
"cell_type": "heading",
"level": 4,
"metadata": {},
"source": [
"List of currently running instances"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"for instance in spotinstance.SpotInstance.get_spot_instances(config.EC2_Tag):\n",
" print instance.status()"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 10
},
{
"cell_type": "heading",
"level": 4,
"metadata": {},
"source": [
"Contents of the SQS queue"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 10
},
{
"cell_type": "heading",
"level": 4,
"metadata": {},
"source": [
"Contents of the output bucket"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"S3 = S3Connection()\n",
"bucket = S3.get_bucket(config.OUTPUT_S3_BUCKET)\n",
"keys = [example_key for example_key in bucket.list()]\n",
"with open(\"output_keys.json\",'w') as fh:\n",
" fh.write(json.dumps(keys))\n",
"print \"list of keys are stored in output_keys.json\"\n",
"print \"Number of keys in the output bucket \",len(keys)"
],
"language": "python",
"metadata": {},
"outputs": [
{
"ename": "S3ResponseError",
"evalue": "S3ResponseError: 403 Forbidden\n",
"output_type": "pyerr",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mS3ResponseError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-11-02a82a969bca>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mS3\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mS3Connection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mbucket\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mS3\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_bucket\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOUTPUT_S3_BUCKET\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 3\u001b[0m \u001b[0mkeys\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mexample_key\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mexample_key\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mbucket\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"output_keys.json\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'w'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mfh\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mfh\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdumps\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/boto/s3/connection.pyc\u001b[0m in \u001b[0;36mget_bucket\u001b[0;34m(self, bucket_name, validate, headers)\u001b[0m\n\u001b[1;32m 469\u001b[0m \"\"\"\n\u001b[1;32m 470\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mvalidate\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 471\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhead_bucket\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbucket_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mheaders\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 472\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 473\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbucket_class\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbucket_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/boto/s3/connection.pyc\u001b[0m in \u001b[0;36mhead_bucket\u001b[0;34m(self, bucket_name, headers)\u001b[0m\n\u001b[1;32m 502\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merror_code\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'AccessDenied'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 503\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merror_message\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'Access Denied'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 504\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 505\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m404\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 506\u001b[0m \u001b[0;31m# For backward-compatibility, we'll populate part of the exception\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mS3ResponseError\u001b[0m: S3ResponseError: 403 Forbidden\n"
]
}
],
"prompt_number": 11
},
{
"cell_type": "heading",
"level": 4,
"metadata": {},
"source": [
"Randomly selected key from the bucket"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"if keys:\n",
" example = key.Key(bucket)\n",
" example.key = random.sample(keys,1)[0]\n",
" data = example.get_contents_to_string()"
],
"language": "python",
"metadata": {},
"outputs": [
{
"ename": "NameError",
"evalue": "name 'keys' is not defined",
"output_type": "pyerr",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-7-c8a0894d8d04>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0mkeys\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0mexample\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mKey\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbucket\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mexample\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkey\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mexample\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_contents_to_string\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mNameError\u001b[0m: name 'keys' is not defined"
]
}
],
"prompt_number": 7
},
{
"cell_type": "code",

0 comments on commit 2aaa42d

Please sign in to comment.