Refactored pricecalculator to awspricecalculator in preparation for p…

…ackaging this code. Also, added setup.py for the same reason.

.gitignore

@@ -0,0 +1,25 @@
+#files that will not be committed
+*.pyc
+*.csv
+*.zip
+bin/*
+lib/*
+.idea/*
+include/*
+.serverless/*
+.Python
+pip-selfcheck.json
+
+
+#temporarily out
+awspricecalculator/s3*
+awspricecalculator/common/utils.py
+scripts/ec2-pricing.py
+scripts/rds-pricing.py
+scripts/s3-pricing.py
+scripts/lambda-pricing.py
+scripts/context.py
+
+
+test/events/constant-lambda-function-local.json
+test/events/constant-tag-local.json

MANIFEST.in

@@ -0,0 +1,2 @@
+recursive-include awspricecalculator/data *.csv *.json
+

README.md

@@ -216,7 +216,7 @@ export AWS_DEFAULT_REGION=<us-east-1|us-west-2|etc.>
 
 The code needs a local copy of the the AWS Price List API index file. 
 The GitHub repo doesn't come with the index file, therefore you have to
-download it the first time you run a test and every time AWS publishes a new
+download it the first time you run your code and every time AWS publishes a new
 Price List API index.
 
 Also, this index file is constantly updated by AWS. I recommend subscribing to the AWS Price List API
@@ -230,7 +230,9 @@ python get-latest-index.py --service=all
 
 The script takes a few seconds to execute since some index files are a little heavy (like the EC2 one).
 
-Once you have all dependencies installed, virtualenv activated, environment
+**Run a test**
+
+Once you have the virtualenv activated, all dependencies installed, environment
 variables set and the latest AWS Price List index file, it's time to run a test.
 
 Update ```test/events/constant-tag.json``` with a tag key/value pair that exists in your AWS account.

pricecalculator/awslambda/pricing.py → awspricecalculator/awslambda/pricing.py

@@ -2,7 +2,7 @@
 import json
 import logging
 from ..common import consts, phelper
-from ..common.data import PricingResult
+from ..common.models import PricingResult
 import tinydb
 
 log = logging.getLogger()

pricecalculator/common/consts.py → awspricecalculator/common/consts.py

@@ -3,6 +3,8 @@
 # COMMON
 #_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/
 DEFAULT_CURRENCY = "USD"
+FORECAST_PERIOD_MONTHLY = "monthly"
+FORECAST_PERIOD_YEARLY = "yearly"
 
 SERVICE_CODE_AWS_DATA_TRANSFER = 'AWSDataTransfer'
 
@@ -22,6 +24,42 @@
               'ap-south-1':'Asia Pacific (Mumbai)'
               }
 
+
+REGION_PREFIX_MAP = {'us-east-1':'',
+              'us-east-2':'USE2-',
+              'us-west-1':'USW1-',
+              'us-west-2':'USW2-',
+              'ca-central-1':'CAN1-',
+              'eu-west-1':'EU-',
+              'eu-west-2':'EUW2-',
+              'eu-central-1':'EUC1-',
+              'ap-northeast-1':'APN1-',
+              'ap-northeast-2':'APN2-',
+              'ap-southeast-1':'APS1-',
+              'ap-southeast-2':'APS2-',
+              'sa-east-1':'SAE1-',
+              'ap-south-1':'APS3-',
+              'US East (N. Virginia)':'',
+              'US East (Ohio)':'USE2-',
+              'US West (N. California)':'USW1-',
+              'US West (Oregon)':'USW2-',
+              'Canada (Central)':'CAN1-',
+              'EU (Ireland)':'EU-',
+              'EU (London)':'EUW2-',
+              'EU (Frankfurt)':'EUC1-',
+              'Asia Pacific (Tokyo)':'APN1-',
+              'Asia Pacific (Seoul)':'APN2-',
+              'Asia Pacific (Singapore)':'APS1-',
+              'Asia Pacific (Sydney)':'APS2-',
+              'South America (Sao Paulo)':'SAE1-',
+              'Asia Pacific (Mumbai)':'APS3-',
+              'AWS GovCloud (US)':'UGW1-',
+              'External':'',
+              'Any': ''
+              }
+
+
+
 REGION_REPORT_MAP = {'us-east-1':'N. Virginia',
               'us-east-2':'Ohio',
               'us-west-1':'N. California',
@@ -38,21 +76,8 @@
               'ap-south-1':'Mumbai'
               }
 
-REGION_PREFIX_MAP = {'us-east-1':'',
-              'us-east-2':'USE2-',
-              'us-west-1':'USW1-',
-              'us-west-2':'USW2-',
-              'ca-central-1':'CAN1-',
-              'eu-west-1':'EU-',
-              'eu-west-2':'EUW2-',
-              'eu-central-1':'EUC1-',
-              'ap-northeast-1':'APN1-',
-              'ap-northeast-2':'APN2-',
-              'ap-southeast-1':'APS1-',
-              'ap-southeast-2':'APS2-',
-              'sa-east-1':'SAE1-',
-              'ap-south-1':'APS3-'
-              }
+
+
 
 
 

pricecalculator/common/data.py → awspricecalculator/common/models.py

@@ -75,7 +75,8 @@ def __init__(self, **kargs):
       self.instanceHours = 0
       if 'instanceHours' in kargs: self.instanceHours = kargs['instanceHours']
       self.operatingSystem = consts.SCRIPT_OPERATING_SYSTEM_LINUX
-      if 'operatingSystem' in kargs: self.operatingSystem = kargs['operatingSystem']
+      if 'operatingSystem' in kargs:
+          if kargs['operatingSystem']: self.operatingSystem = kargs['operatingSystem']
 
       #TODO: Add support for pre-installed software (i.e. SQL Web in Windows instances)
       self.preInstalledSoftware = 'NA'
@@ -122,40 +123,38 @@ def __init__(self, **kargs):
       self.validate()
 
     def validate(self):
-      validation_ok = True
       validation_message = ""
 
       if self.instanceType and self.instanceType not in consts.SUPPORTED_INSTANCE_TYPES:
-        validation_message = "instance-type must be one of the following values:"+str(consts.SUPPORTED_INSTANCE_TYPES)
-        validation_ok = False
+        validation_message += "instance-type must be one of the following values:"+str(consts.SUPPORTED_INSTANCE_TYPES)+"\n"
       if self.region not in consts.SUPPORTED_REGIONS:
-        validation_message = "region must be one of the following values:"+str(consts.SUPPORTED_REGIONS)
-        validation_ok = False
+        validation_message += "region must be one of the following values:"+str(consts.SUPPORTED_REGIONS)+"\n"
+      if not self.operatingSystem:
+        validation_message += "operating-system cannot be empty\n"
       if self.operatingSystem and self.operatingSystem not in consts.SUPPORTED_EC2_OPERATING_SYSTEMS:
-        validation_message = "operating-system must be one of the following values:"+str(consts.SUPPORTED_EC2_OPERATING_SYSTEMS)
-        validation_ok = False
+        validation_message += "operating-system must be one of the following values:"+str(consts.SUPPORTED_EC2_OPERATING_SYSTEMS)+"\n"
       if self.ebsVolumeType and self.ebsVolumeType not in consts.SUPPORTED_EBS_VOLUME_TYPES:
-        validation_message = "ebs-volume-type must be one of the following values:"+str(consts.SUPPORTED_EBS_VOLUME_TYPES)
-        validation_ok = False
+        validation_message += "ebs-volume-type must be one of the following values:"+str(consts.SUPPORTED_EBS_VOLUME_TYPES)+"\n"
+      if self.dataTransferOutInterRegionGb > 0 and not self.toRegion:
+        validation_message += "Must specify a to-region if you specify data-transfer-out-interregion-gb\n"
       if self.dataTransferOutInterRegionGb and self.toRegion not in consts.SUPPORTED_REGIONS:
-        validation_message = "to-region must be one of the following values:"+str(consts.SUPPORTED_REGIONS)
+        validation_message += "to-region must be one of the following values:"+str(consts.SUPPORTED_REGIONS)+"\n"
       if self.dataTransferOutInterRegionGb and self.region == self.toRegion:
-        validation_message = "source and destination regions must be different for inter-regional data transfers"
-
+        validation_message += "source and destination regions must be different for inter-regional data transfers\n"
       if self.termType not in consts.SUPPORTED_TERM_TYPES:
-          validation_message = "term-type must be one of the following values:[{}]".format(consts.SUPPORTED_TERM_TYPES)
-
+          validation_message += "term-type must be one of the following values:[{}]".format(consts.SUPPORTED_TERM_TYPES)+"\n"
       if self.termType == consts.SCRIPT_TERM_TYPE_RESERVED:
           if not self.offeringClass:
-              validation_message = "offering-class must be specified for Reserved instances"
+              validation_message += "offering-class must be specified for Reserved instances\n"
           if not self.purchaseOption:
-              validation_message = "purchase-option must be specified for Reserved instances"
+              validation_message += "purchase-option must be specified for Reserved instances\n"
 
 
       #TODO: add validation for max number of IOPS
       #TODO: add validation for negative numbers
 
-      if not validation_ok:
+      validation_ok = True
+      if validation_message:
           raise ValidationError(validation_message)
 
       return validation_ok

pricecalculator/common/phelper.py → awspricecalculator/common/phelper.py

@@ -4,7 +4,7 @@
 import datetime
 import logging
 import csv, json
-from data import PricingRecord, PricingResult
+from models import PricingRecord, PricingResult
 
 log = logging.getLogger()
 

awspricecalculator/common/utils.py

@@ -0,0 +1,143 @@
+import json
+import consts, models
+import datetime
+
+from ..ec2 import pricing as ec2pricing
+from ..s3 import pricing as s3pricing
+from ..rds import pricing as rdspricing
+from ..awslambda import pricing as lambdapricing
+
+
+
+#Creates a table with all the SKUs that are part of the total price
+def buildSkuTable(evaluated_sku_desc):
+  result = {}
+  sorted_descriptions = sorted(evaluated_sku_desc)
+  result_table_header = "Price | Description | Price Per Unit | Usage | Rate Code"
+  result_records = ""
+  total = 0
+  for s in sorted_descriptions:
+    result_records = result_records + "$" + str(s[0]) + "|" + str(s[1]) + "|" + str(s[2]) + "|" + str(s[3]) + "|" + s[4]+"\n"
+    total = total + s[0]
+
+  result['header']=result_table_header
+  result['records']=result_records
+  result['total']=total
+  return result
+
+
+
+#It calculates price based on a variable price dimension. For example: by region, os, instance type, etc.
+def compare(**kwargs):
+  service = kwargs['service']
+  sortCriteria = kwargs['sortCriteria']
+  result = []
+  cheapest_price = 0
+  criteria_array = ()
+  kwargs_key = ""
+
+  #Sort by AWS Region - Total Cost and To-region (for sorting by destination - find which regions are cheaper for backups)
+  if sortCriteria in [consts.SORT_CRITERIA_REGION, consts.SORT_CRITERIA_TO_REGION]:
+    tableCriteriaHeader = "Sorted by total cost by region\nRegion code\tRegion name\t"
+    if sortCriteria == consts.SORT_CRITERIA_TO_REGION:
+      tableCriteriaHeader = "Sorted by data transfer destination from region ["+kwargs['region']+"] to other regions\nTo-Region code\tTo-Region name\t"
+
+    for r in consts.SUPPORTED_REGIONS:
+      if sortCriteria == consts.SORT_CRITERIA_TO_REGION:
+        kwargs['toRegion']=r
+      else:
+        kwargs['region']=r
+      if service == consts.SERVICE_EC2:
+        p = ec2pricing.calculate(models.Ec2PriceDimension(**kwargs))
+      if service == consts.SERVICE_S3:
+        p = s3pricing.calculate(models.S3PriceDimension(**kwargs))
+      if service == consts.SERVICE_RDS:
+        p = rdspricing.calculate(models.RdsPriceDimension(**kwargs))
+      if service == consts.SERVICE_LAMBDA:
+        p = lambdapricing.calculate(models.LambdaPriceDimension(**kwargs))
+
+      print (json.dumps(p, indent=True))
+      #Only append records for those combinations that exist in the PriceList API
+      if p['pricingRecords']: result.append((p['totalCost'],r))
+
+
+ #Sort by EC2 Operating System
+  if sortCriteria == consts.SORT_CRITERIA_OS:
+    tableCriteriaHeader = "Sorted by total cost by Operating System in region ["+kwargs['region']+"]\nOS\t"
+    for o in consts.SUPPORTED_EC2_OPERATING_SYSTEMS:
+      kwargs['operatingSystem']=o
+      if service == consts.SERVICE_EC2:
+        p = ec2pricing.calculate(models.Ec2PriceDimension(**kwargs))
+
+      result.append((p['totalCost'],o))
+
+
+  #Sort by Lambda memory
+  if sortCriteria == consts.SORT_CRITERIA_LAMBDA_MEMORY:
+    tableCriteriaHeader = "Sorted by total Allocated Memory in region ["+kwargs['region']+"]\nMemory\t"
+    for m in consts.LAMBDA_MEM_SIZES:
+      kwargs['memoryMb']=m
+      p = lambdapricing.calculate(models.LambdaPriceDimension(**kwargs))
+      if p['pricingRecords']: result.append((p['totalCost'],m))
+
+
+  #Sort by S3 Storage Class
+  if sortCriteria == consts.SORT_CRITERIA_S3_STORAGE_CLASS:
+    #TODO: Use criteria_array for all sort calculations
+    tableCriteriaHeader = "Sorted by S3 Storage Class in region ["+kwargs['region']+"]\nStorage Class\t"
+    criteria_array = consts.SUPPORTED_S3_STORAGE_CLASSES
+    for c in criteria_array:
+      kwargs['storageClass']=c
+      p = s3pricing.calculate(models.S3PriceDimension(**kwargs))
+      if p['pricingRecords']: result.append((p['totalCost'],c))
+
+
+  sorted_result = sorted(result)
+  print ("sorted_result: {}".format(sorted_result))
+  if sorted_result:
+    cheapest_price = sorted_result[0][0]
+  result = []
+  i = 0
+  #TODO: use a structured object (Class or dict) instead of using indexes for each field in the table
+  for r in sorted_result:
+    #Calculate the current record relative to the last record
+    delta_last = 0
+    pct_to_last = 0
+    pct_to_cheapest = 0
+    #TODO:handle cases where cheapest is 0
+    if i >= 1:
+      delta_last = sorted_result[i][0]-sorted_result[i-1][0]
+      if sorted_result[i-1][0] > 0:
+        pct_to_last = ((sorted_result[i][0]-sorted_result[i-1][0])/sorted_result[i-1][0])*100
+    if cheapest_price > 0:
+      pct_to_cheapest = ((r[0]-cheapest_price)/cheapest_price)*100
+
+    result.append((r[0], r[1],pct_to_cheapest, pct_to_last,(r[0]-cheapest_price),delta_last))
+
+    i = i+1
+
+  print("Sorted cost table:")
+  print(tableCriteriaHeader+"Cost(USD)\t% compared to cheapest\t% compared to previous\tdelta cheapest\tdelta previous")
+  for r in result:
+    rowCriteriaValues = ""
+    if sortCriteria in [consts.SORT_CRITERIA_REGION, consts.SORT_CRITERIA_TO_REGION]:
+      rowCriteriaValues = r[1]+"\t"+consts.REGION_MAP[r[1]]+"\t"
+    if sortCriteria in [consts.SORT_CRITERIA_OS, consts.SORT_CRITERIA_LAMBDA_MEMORY, consts.SORT_CRITERIA_S3_STORAGE_CLASS]:
+      rowCriteriaValues = str(r[1])+"\t"
+    print(rowCriteriaValues+str(r[0])+"\t"+str(r[2])+"\t"+str(r[3])+"\t"+str(r[4])+"\t"+str(r[5]))
+
+
+
+  return result
+
+
+def get_index_file_name(service, name, format):
+  result = '../awspricecalculator/'+service+'/data/'+name+'.'+format
+  return result
+
+
+
+
+
+
+

pricecalculator/data/ec2/index_metadata.json → awspricecalculator/data/ec2/index_metadata.json

@@ -1,7 +1,7 @@
 {
     "OfferCode": "AmazonEC2", 
     "FormatVersion": "v1.0", 
-    "Version": "20170324211955", 
-    "Publication Date": "2017-03-24T21:19:55Z", 
+    "Version": "20170526181956", 
+    "Publication Date": "2017-05-26T18:19:56Z", 
     "Disclaimer": "This pricing list is for informational purposes only. All prices are subject to the additional terms included in the pricing pages on http://aws.amazon.com. All Free Tier prices are also subject to the terms included at https://aws.amazon.com/free/"
 }

pricecalculator/data/lambda/index_metadata.json → awspricecalculator/data/lambda/index_metadata.json

@@ -1,7 +1,7 @@
 {
     "OfferCode": "AWSLambda", 
     "FormatVersion": "v1.0", 
-    "Version": "20161207235208", 
-    "Publication Date": "2016-12-07T23:52:08Z", 
+    "Version": "20170519011124", 
+    "Publication Date": "2017-05-19T01:11:24Z", 
     "Disclaimer": "This pricing list is for informational purposes only. All prices are subject to the additional terms included in the pricing pages on http://aws.amazon.com. All Free Tier prices are also subject to the terms included at https://aws.amazon.com/free/"
 }

pricecalculator/data/rds/index_metadata.json → awspricecalculator/data/rds/index_metadata.json

@@ -1,7 +1,7 @@
 {
     "OfferCode": "AmazonRDS", 
     "FormatVersion": "v1.0", 
-    "Version": "20170327042929", 
-    "Publication Date": "2017-03-27T04:29:29Z", 
+    "Version": "20170419200300", 
+    "Publication Date": "2017-04-19T20:03:00Z", 
     "Disclaimer": "This pricing list is for informational purposes only. All prices are subject to the additional terms included in the pricing pages on http://aws.amazon.com. All Free Tier prices are also subject to the terms included at https://aws.amazon.com/free/"
 }

pricecalculator/data/s3/index_metadata.json → awspricecalculator/data/s3/index_metadata.json

@@ -1,7 +1,7 @@
 {
     "OfferCode": "AmazonS3", 
     "FormatVersion": "v1.0", 
-    "Version": "20170329010048", 
-    "Publication Date": "2017-03-29T01:00:48Z", 
+    "Version": "20170424230114", 
+    "Publication Date": "2017-04-24T23:01:14Z", 
     "Disclaimer": "This pricing list is for informational purposes only. All prices are subject to the additional terms included in the pricing pages on http://aws.amazon.com. All Free Tier prices are also subject to the terms included at https://aws.amazon.com/free/"
 }