Permalink
Browse files

Initial commit

  • Loading branch information...
grosskur committed Feb 24, 2013
0 parents commit 8e4b1d0e6bce2cf51ba34ee0be75ceb29a549dd5
Showing with 39,563 additions and 0 deletions.
  1. +6 −0 .gitignore
  2. +2 −0 Procfile
  3. +1 −0 README.md
  4. 0 ec2price/__init__.py
  5. +160 −0 ec2price/app.py
  6. +81 −0 ec2price/collector.py
  7. +45 −0 ec2price/sql/initial.sql
  8. +27 −0 ec2price/sql/schema.sql
  9. +2 −0 ec2price/static/css/base.css
  10. +41 −0 ec2price/static/js/iso8601.js
  11. +24 −0 ec2price/static/js/main.js
  12. +1,109 −0 ec2price/static/vendor/bootstrap-2.3.0/css/bootstrap-responsive.css
  13. +9 −0 ec2price/static/vendor/bootstrap-2.3.0/css/bootstrap-responsive.min.css
  14. +6,158 −0 ec2price/static/vendor/bootstrap-2.3.0/css/bootstrap.css
  15. +9 −0 ec2price/static/vendor/bootstrap-2.3.0/css/bootstrap.min.css
  16. BIN ec2price/static/vendor/bootstrap-2.3.0/img/glyphicons-halflings-white.png
  17. BIN ec2price/static/vendor/bootstrap-2.3.0/img/glyphicons-halflings.png
  18. +2,268 −0 ec2price/static/vendor/bootstrap-2.3.0/js/bootstrap.js
  19. +6 −0 ec2price/static/vendor/bootstrap-2.3.0/js/bootstrap.min.js
  20. +7,790 −0 ec2price/static/vendor/d3-3.0.6/js/d3.js
  21. +4 −0 ec2price/static/vendor/d3-3.0.6/js/d3.min.js
  22. +9,597 −0 ec2price/static/vendor/jquery-1.9.1/js/jquery.js
  23. +5 −0 ec2price/static/vendor/jquery-1.9.1/js/jquery.min.js
  24. +656 −0 ec2price/static/vendor/nvd3-0.0.1a/css/nv.d3.css
  25. +11,341 −0 ec2price/static/vendor/nvd3-0.0.1a/js/nv.d3.js
  26. +5 −0 ec2price/static/vendor/nvd3-0.0.1a/js/nv.d3.min.js
  27. +24 −0 ec2price/templates/base.html
  28. +75 −0 ec2price/templates/main.html
  29. +78 −0 ec2price/web.py
  30. +4 −0 requirements.txt
  31. +11 −0 scripts/ec2price
  32. +25 −0 setup.py
@@ -0,0 +1,6 @@
*.pyc
.env
.envdir
.venv
.vagrant
*.bak*
@@ -0,0 +1,2 @@
web: scripts/ec2price web
collector: scripts/ec2price collector
@@ -0,0 +1 @@
# EC2 Price Service
No changes.
@@ -0,0 +1,160 @@
"""
App
"""
import argparse
import logging
import os
import re
import psycopg2
import psycopg2.extras
import tornado.ioloop
import tornado.web
from .web import MainHandler
from .collector import collect
PROG = 'ec2price'
TEMPLATE_PATH = os.path.join(os.path.dirname(__file__), 'templates')
STATIC_PATH = os.path.join(os.path.dirname(__file__), 'static')
DATABASE_URL_REGEX = re.compile(
'postgres://'
'(?P<user>.*)'
':(?P<password>.*)'
'@(?P<host>[-a-z\d.]+)'
':(?P<port>\d+)'
'/(?P<dbname>.+)'
)
DATABASE_URL_EXAMPLE = 'postgres://username:password@host:port/dbname'
class ArgumentParser(argparse.ArgumentParser):
def error(self, message):
self.exit(2, '%s: error: %s\n' % (self.prog, message))
def main(args):
# setup logging
fmt = PROG + ': %(levelname)s%(message)s'
logging.basicConfig(level=logging.DEBUG, format=fmt)
logging.addLevelName(logging.DEBUG, 'debug: ')
logging.addLevelName(logging.INFO, '')
logging.addLevelName(logging.WARNING, 'warning: ')
logging.addLevelName(logging.ERROR, 'error: ')
logging.addLevelName(logging.CRITICAL, 'critical: ')
# parse command line
parser = ArgumentParser(
prog=PROG,
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
subparsers = parser.add_subparsers(help='commands')
p_web = subparsers.add_parser('web', help='web app')
p_web.set_defaults(cmd='web')
p_api = subparsers.add_parser('api', help='web API')
p_api.set_defaults(cmd='api')
p_collector = subparsers.add_parser('collector', help='collector')
p_collector.set_defaults(cmd='collector')
opts = parser.parse_args(args)
if opts.cmd == 'web':
debug = bool(os.getenv('DEBUG', False))
port = int(os.getenv('PORT', 8080))
address = os.getenv('ADDRESS', '')
cookie_secret = os.getenv('COOKIE_SECRET')
database_url = os.getenv('DATABASE_URL')
database_dsn = None
if database_url:
m = DATABASE_URL_REGEX.match(database_url)
if not m:
parser.error('must be of form %s' % DATABASE_URL_EXAMPLE)
database_dsn = ' '.join('%s=%s' % (k, v)
for k, v in m.groupdict().items())
if not database_url:
parser.error('DATABASE_URL is required')
if not cookie_secret:
parser.error('COOKIE_SECRET is required')
db_conn = _get_db_conn(database_dsn)
params = {'db_conn': db_conn}
handlers = [
(r'/', MainHandler, params),
]
_start_tornado_app(debug, cookie_secret, port, address, handlers)
elif opts.cmd == 'api':
debug = bool(os.getenv('DEBUG', False))
port = int(os.getenv('PORT', 8080))
address = os.getenv('ADDRESS', '')
cookie_secret = os.getenv('COOKIE_SECRET')
database_url = os.getenv('DATABASE_URL')
database_dsn = None
if database_url:
m = DATABASE_URL_REGEX.match(database_url)
if not m:
parser.error('must be of form %s' % DATABASE_URL_EXAMPLE)
database_dsn = ' '.join('%s=%s' % (k, v)
for k, v in m.groupdict().items())
if not database_url:
parser.error('DATABASE_URL is required')
if not cookie_secret:
parser.error('COOKIE_SECRET is required')
db_conn = _get_db_conn(database_dsn)
params = {'db_conn': db_conn}
handlers = [
(r'/', MainHandler, params),
]
_start_tornado_app(debug, cookie_secret, port, address, handlers)
elif opts.cmd == 'collector':
database_url = os.getenv('DATABASE_URL')
database_dsn = None
if database_url:
m = DATABASE_URL_REGEX.match(database_url)
if not m:
parser.error('must be of form %s' % DATABASE_URL_EXAMPLE)
database_dsn = ' '.join('%s=%s' % (k, v)
for k, v in m.groupdict().items())
if not database_url:
parser.error('DATABASE_URL is required')
db_conn = _get_db_conn(database_dsn)
collect(db_conn)
return 0
def _start_tornado_app(debug, cookie_secret, port, address, handlers):
settings = dict(
cookie_secret=cookie_secret,
template_path=TEMPLATE_PATH,
static_path=STATIC_PATH,
xsrf_cookies=False,
autoescape='xhtml_escape',
debug=debug,
)
app = tornado.web.Application(handlers, **settings)
logging.info('listening on port: %d', port)
app.listen(port, address)
tornado.ioloop.IOLoop.instance().start()
def _get_db_conn(database_dsn):
psycopg2.extras.register_uuid()
return psycopg2.connect(
database_dsn,
connection_factory=psycopg2.extras.RealDictConnection,
)
@@ -0,0 +1,81 @@
"""
Data collector
"""
import botocore.session
import contextlib
import datetime
import logging
import uuid
_HOURS = 8
_FMT = '%Y-%m-%dT%H:%M:%S.000Z'
_SELECT_SPOT_PRICE = """
select price
from spot_prices, availability_zones, instance_types
where spot_prices.availability_zone_id = availability_zones.id
and availability_zones.api_name = %s
and spot_prices.instance_type_id = instance_types.id
and instance_types.api_name = %s
and spot_prices.ts = %s
limit 1
"""
_INSERT_SPOT_PRICE = """
with a as (select id from availability_zones where api_name = %s),
i as (select id from instance_types where api_name = %s)
insert into spot_prices (id, availability_zone_id, instance_type_id, ts, price)
select %s, a.id, i.id, %s, %s
from a, i
"""
_SELECT_INSTANCE_TYPES = """
select api_name
from instance_types
order by api_name
"""
logging.getLogger('botocore').setLevel(logging.WARN)
logging.getLogger('requests.packages.urllib3').setLevel(logging.WARN)
def collect(db_conn):
session = botocore.session.get_session()
ec2 = session.get_service('ec2')
operation = ec2.get_operation('DescribeSpotPriceHistory')
d = datetime.datetime.utcnow() - datetime.timedelta(hours=_HOURS)
start_time = d.strftime(_FMT)
with contextlib.closing(db_conn.cursor()) as cursor:
cursor.execute(_SELECT_INSTANCE_TYPES)
rows = cursor.fetchall()
instance_types = [r['api_name'] for r in rows]
for region in ec2.region_names:
logging.debug('collecting spot prices from region: %s', region)
endpoint = ec2.get_endpoint(region)
response, data = operation.call(
endpoint,
instance_types=instance_types,
product_descriptions=['Linux/UNIX'],
start_time=start_time,
)
for i in data.get('spotPriceHistorySet', []):
with contextlib.closing(db_conn.cursor()) as cursor:
cursor.execute(_SELECT_SPOT_PRICE, [
i['availabilityZone'],
i['instanceType'],
i['timestamp'],
])
row = cursor.fetchone()
if not row:
logging.debug('inserting spot price: %s', i)
cursor.execute(_INSERT_SPOT_PRICE, [
i['availabilityZone'],
i['instanceType'],
uuid.uuid4(),
i['timestamp'],
i['spotPrice'],
])
db_conn.commit()
@@ -0,0 +1,45 @@
insert into availability_zones (id, api_name) values
('3e64bccf-1624-4426-af3d-1d945c883f7c', 'us-east-1a'),
('2855e655-e06e-443c-98f8-3c24cf2e6349', 'us-east-1b'),
('b67a629c-c4f0-4574-934c-76983647f179', 'us-east-1c'),
('8f120c9f-d9c2-4a68-bb84-89d6166630ca', 'us-east-1d'),
('73b52c28-d6e1-4c1a-b56f-c8a8d9bdc7ea', 'us-east-1e'),
('0e260d6d-9437-4cb2-8bd2-50e581118557', 'us-west-1a'),
('a2201edf-64d1-4985-8ca7-1cd5fd97a270', 'us-west-1b'),
('ef66215f-ec2c-4781-a2f2-780922a92665', 'us-west-1c'),
('5a29ef9e-3c89-47fd-bc4a-f90518150adb', 'us-west-2a'),
('35108234-36c4-4893-a92e-10aac99f385b', 'us-west-2b'),
('dbfbb97d-5fe9-4529-a456-11a4b6fd82b0', 'us-west-2c'),
('8b969ede-84fb-47aa-af42-640a35ef9c6b', 'eu-west-1a'),
('c43ce7b1-27d5-46bd-81db-5a0befaad41f', 'eu-west-1b'),
('46d66afd-42da-4982-a8bc-a0958878bff1', 'eu-west-1c'),
('5a911212-84a5-447c-be56-fff24e341879', 'ap-southeast-1a'),
('33fe6805-37b4-4cb6-ba31-cea03235d28d', 'ap-southeast-1b'),
('6563eda2-2437-4580-9fd8-5696251f59b9', 'ap-northeast-1a'),
('352afd4c-508a-440b-83b5-b4211737d5c3', 'ap-northeast-1b'),
('43866c7c-2e57-4a79-a51a-6615e6e5e2ca', 'ap-northeast-1c'),
('f0af3c64-5cf1-429b-bf73-168b553310db', 'ap-southeast-2a'),
('54ad4a70-2c80-4066-ab12-ad789339ef8f', 'ap-southeast-2b'),
('68696c72-3211-40c2-9e3d-3013614ce294', 'sa-east-1a'),
('50932294-9c06-4a9e-980b-ba2198aa49a0', 'sa-east-1b');
insert into instance_types (id, api_name, display_name) values
('fa217149-584e-416f-9023-b99b373366b9', 'm1.small', 'M1 Small'),
('f5ee48b8-ec2c-4b76-8f5e-9c400a6c0145', 'm1.medium', 'M1 Medium'),
('bf4b073f-571f-421e-9759-1d54cf57b580', 'm1.large', 'M1 Large'),
('36ea806a-ff2a-437c-a36b-b0b894b5ebe3', 'm1.xlarge', 'M1 Extra Large'),
('d9dd3b2e-7d02-4aae-ac67-cd592104d158', 'm3.xlarge', 'M3 Extra Large'),
('32f4f19b-cc46-4759-ad7e-0c9f5f2f54f0', 'm3.2xlarge', 'M3 Double Extra Large'),
('22d11989-da8d-4bc0-adca-d483b8a5afde', 't1.micro', 'Micro'),
('2801a60e-4a37-42f9-8a57-35f4dd97c41a', 'm2.xlarge', 'High-Memory Extra Large'),
('e4aa2c02-c4c7-495f-940d-42fa71d69dae', 'm2.2xlarge', 'High-Memory Double Extra Large'),
('62646de8-d7c1-43dc-a320-2b4edf367733', 'm2.4xlarge', 'High-Memory Quadruple Extra Large'),
('5b06e645-8f4f-4346-80f9-66b4b2fcd0c1', 'c1.medium', 'High-CPU Medium'),
('d70c2875-c1f6-4794-b8b6-186f25db6ecb', 'c1.xlarge', 'High-CPU Extra Large'),
('d5ff55de-d360-4ea0-9b13-ac560c56f821', 'cc1.4xlarge', 'Cluster Compute Quadruple Extra Large'),
('69918c5c-bcf5-4f39-ab94-32c27fb67846', 'cc2.8xlarge', 'Cluster Compute Eight Extra Large'),
('d42f738f-bd21-4a31-9cc2-ad1493b243c2', 'cr1.8xlarge', 'High Memory Cluster Eight Extra Large'),
('e798b87d-60df-4cbe-abcf-e92b36dcd525', 'cg1.4xlarge', 'Cluster GPU Quadruple Extra Large'),
('890dca68-a23f-4d3d-83e7-3fafe7336932', 'hi1.4xlarge', 'High I/O Quadruple Extra Large'),
('8f2860c2-0175-4e0a-b675-67c75b676e4d', 'hs1.8xlarge', 'High Storage Eight Extra Large');
@@ -0,0 +1,27 @@
create table availability_zones (
id uuid not null,
api_name text not null,
primary key (id),
unique (api_name)
);
create table instance_types (
id uuid not null,
api_name text not null,
display_name text not null,
primary key (id),
unique (api_name),
unique (display_name)
);
create table spot_prices (
id uuid not null,
availability_zone_id uuid not null,
instance_type_id uuid not null,
ts timestamptz not null,
price decimal not null,
primary key (id),
unique (ts, availability_zone_id, instance_type_id),
foreign key (availability_zone_id) references availability_zones (id),
foreign key (instance_type_id) references instance_types (id)
);
@@ -0,0 +1,2 @@
body {
}
@@ -0,0 +1,41 @@
/**
* Date.parse with progressive enhancement for ISO 8601 <https://github.com/csnover/js-iso8601>
* © 2011 Colin Snover <http://zetafleet.com>
* Released under MIT license.
*/
(function (Date, undefined) {
var origParse = Date.parse, numericKeys = [ 1, 4, 5, 6, 7, 10, 11 ];
Date.parse = function (date) {
var timestamp, struct, minutesOffset = 0;
// ES5 §15.9.4.2 states that the string should attempt to be parsed as a Date Time String Format string
// before falling back to any implementation-specific date parsing, so that’s what we do, even if native
// implementations could be faster
// 1 YYYY 2 MM 3 DD 4 HH 5 mm 6 ss 7 msec 8 Z 9 ± 10 tzHH 11 tzmm
if ((struct = /^(\d{4}|[+\-]\d{6})(?:-(\d{2})(?:-(\d{2}))?)?(?:T(\d{2}):(\d{2})(?::(\d{2})(?:\.(\d{3}))?)?(?:(Z)|([+\-])(\d{2})(?::(\d{2}))?)?)?$/.exec(date))) {
// avoid NaN timestamps caused by “undefined” values being passed to Date.UTC
for (var i = 0, k; (k = numericKeys[i]); ++i) {
struct[k] = +struct[k] || 0;
}
// allow undefined days and months
struct[2] = (+struct[2] || 1) - 1;
struct[3] = +struct[3] || 1;
if (struct[8] !== 'Z' && struct[9] !== undefined) {
minutesOffset = struct[10] * 60 + struct[11];
if (struct[9] === '+') {
minutesOffset = 0 - minutesOffset;
}
}
timestamp = Date.UTC(struct[1], struct[2], struct[3], struct[4], struct[5] + minutesOffset, struct[6], struct[7]);
}
else {
timestamp = origParse ? origParse(date) : NaN;
}
return timestamp;
};
}(Date));
@@ -0,0 +1,24 @@
nv.addGraph(function() {
var chart = nv.models.scatterChart();
chart.xAxis
.axisLabel('Date')
.tickFormat(function (d) {
return d3.time.format('%b %d')(new Date(d));
});
chart.yAxis
.axisLabel('Price ($)')
.tickFormat(d3.format('.03f'));
d3.select('#chart svg')
.datum(prices())
.transition().duration(500)
.call(chart);
nv.utils.windowResize(function () {
d3.select('#chart svg').call(chart);
});
return chart;
});
Oops, something went wrong.

0 comments on commit 8e4b1d0

Please sign in to comment.