Skip to content

Issue #5 - Replace JSON with SQLite #7

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bencehezso merged 1 commit into from
Dec 9, 2024
Merged

Issue #5 - Replace JSON with SQLite #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@

**Release Date: November 20th, 2024**

cloudexit is an open-source tool that empowers cloud engineers to conduct comprehensive cloud exit assessments. It helps identify and evaluate the risks associated with their cloud environment while providing actionable insights into the challenges and constraints of transitioning away from their current cloud provider. By leveraging EscapeCloud OSS, organizations can better prepare for a potential cloud exit, ensuring a smoother and more informed decision-making process.
cloudexit is an open-source tool that empowers cloud engineers to conduct comprehensive cloud exit assessments. It helps identify and evaluate the risks associated with their cloud environment while providing actionable insights into the challenges and constraints of transitioning away from their current cloud provider. By leveraging EscapeCloud Community Edition, organizations can better prepare for a potential cloud exit, ensuring a smoother and more informed decision-making process.


## Required Packages

Expand Down
4 changes: 2 additions & 2 deletions assets/template/index.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1" />
<!-- Favicon & Title -->
<link rel="icon" type="image/png" sizes="16x16" href="assets/img/logo/favicon.png" />
<title>EscapeCloud OSS - Cloud Exit Assessment Report</title>
<title>EscapeCloud Community Edition - Cloud Exit Assessment Report</title>
<!-- CSS only -->
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.11.0/font/bootstrap-icons.css">
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-T3c6CoIi6uLrA9TneNEoa7RxnatzjcDSCmG1MXxSR1GAsXEV/Dwwykc2MPK8M2HN" crossorigin="anonymous">
Expand All @@ -24,7 +24,7 @@
<div class="top-head">
<h4>
<img src="assets/img/logo/logo.png" width="30" alt="EscapeCloud" />
EscapeCloud OSS - Cloud Exit Assessment Report
EscapeCloud Community Edition - Cloud Exit Assessment Report
</h4>
</div>
</header>
Expand Down
246 changes: 93 additions & 153 deletions core/engine.py
View file
Open in desktop

Large diffs are not rendered by default.

97 changes: 58 additions & 39 deletions core/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
import shutil
import logging
from datetime import datetime
from collections import defaultdict

logger = logging.getLogger("core.engine.utils")

def copy_assets(report_path):
assets_folders = ["css", "img", "icons"]
Expand All @@ -19,6 +22,18 @@ def copy_assets(report_path):
if not os.path.exists(dest_path):
shutil.copytree(src_path, dest_path, dirs_exist_ok=True)

# Copy datasets/data.db to data/assessment.db
db_src_path = "datasets/data.db"
db_dest_dir = os.path.join(report_path, "data")
db_dest_path = os.path.join(db_dest_dir, "assessment.db")

# Create the 'data' directory if it doesn't exist
os.makedirs(db_dest_dir, exist_ok=True)

# Only copy if the destination doesn't already exist
if not os.path.exists(db_dest_path):
shutil.copyfile(db_src_path, db_dest_path)


def get_cost_summary(cost_data):
months = []
Expand All @@ -32,6 +47,13 @@ def get_cost_summary(cost_data):
"EUR": "€"
}

# Convert list to dictionary if necessary
if isinstance(cost_data, list):
cost_data = {
item["month"]: {"cost": item["cost"], "currency": item["currency"]}
for item in cost_data
}

# Extract currency from the first entry, assuming all costs use the same currency
first_entry = next(iter(cost_data.values()), None)
currency_code = first_entry.get("currency", "USD") if first_entry else "USD"
Expand All @@ -47,75 +69,72 @@ def get_cost_summary(cost_data):
return months, cost_values, total_cost, currency_symbol

def get_risk_summary(risk_data, risk_definitions, resource_inventory):
logger = logging.getLogger(__name__)

severity_order = {'high': 1, 'medium': 2, 'low': 3}
severity_counts = {'high': 0, 'medium': 0, 'low': 0}
sorted_risks = []

# Map resource IDs to resource names for quick lookup
resource_name_map = {str(item['resource_type']): item['resource_name'] for item in resource_inventory.values()}

# Log the resource_name_map to verify it has been built correctly
#logger.info(f"Resource Name Map: {resource_name_map}")
resource_name_map = {str(key): value['name'] for key, value in resource_inventory.items()}

# Log the risk_data to verify its structure
#logger.info(f"Risk Data: {risk_data}")

# Group risks by their risk code and track impacted resources
risk_map = {}
# Group risks by their risk code and impacted resources
risk_map = defaultdict(lambda: {"impacted_resources": set(), "count": 0})
for risk_entry in risk_data:
risk_code = risk_entry['risk']
resource_type = str(risk_entry['resource_type']) # Convert to string to match the map keys

# Initialize risk entry in the map if it doesn't exist
if risk_code not in risk_map:
risk_map[risk_code] = {
"impacted_resources": set(), # To store unique resources
"count": 0
}
resource_type = str(risk_entry['resource_type']) if risk_entry['resource_type'] != "null" else None

# If resource_type is not "null", add it to impacted resources
if resource_type != "null":
if resource_type:
# Handle risks with associated resource types
resource_name = resource_name_map.get(resource_type, "Unknown Resource")
risk_map[risk_code]["impacted_resources"].add(resource_name)
risk_map[risk_code]["count"] += 1
else:
# Mark this entry as a general risk without specific resources
# Handle overall risks with no specific resource type
risk_map[risk_code]["impacted_resources"] = []
risk_map[risk_code]["count"] = None

# Log the intermediate risk_map to verify resource processing
#logger.info(f"Risk Map After Processing: {risk_map}")

# Process each risk code in the map to populate sorted_risks
# Process risk definitions
for risk_code, risk_info in risk_map.items():
# Look up the risk definition from risk_definitions
risk_definition = next((rd for rd in risk_definitions if rd["id"] == risk_code), None)
if not risk_definition:
continue

severity = risk_definition['severity']
severity_counts[severity] += 1

# Format the impacted resources and count
impacted_resources = list(risk_info["impacted_resources"]) if risk_info["impacted_resources"] else []
impacted_resources_count = risk_info["count"]

# Append detailed risk information
sorted_risks.append({
'name': risk_definition['name'],
'description': risk_definition['description'],
'impacted_resources': impacted_resources,
'impacted_resources_count': impacted_resources_count,
'impacted_resources': list(risk_info["impacted_resources"]),
'impacted_resources_count': risk_info["count"],
'severity': severity
})

# Sort risks by severity level
# Sort risks by severity
sorted_risks.sort(key=lambda x: severity_order.get(x['severity'], 4))

# Log the final sorted risks for verification
#logger.info(f"Sorted Risks: {sorted_risks}")
#logger.info(f"Severity Counts: {severity_counts}")

return sorted_risks, severity_counts

def prepare_alternative_technologies(resource_inventory, alternatives, alternative_technologies, exit_strategy):
alt_tech_data = []
for resource in resource_inventory:
resource_type = resource.get("resource_type")
relevant_alternatives = [
alt for alt in alternatives
if str(alt["resource_type"]) == str(resource_type) and str(alt["strategy_type"]) == str(exit_strategy)
]
for alt in relevant_alternatives:
tech = next(
(t for t in alternative_technologies if t["id"] == alt["alternative_technology"] and t["status"] == "t"),
None
)
if tech:
alt_tech_data.append({
"resource_type_id": resource_type,
"product_name": tech.get("product_name"),
"product_description": tech.get("product_description"),
"product_url": tech.get("product_url"),
"open_source": tech.get("open_source") == "t",
"support_plan": tech.get("support_plan") == "t",
"status": tech.get("status") == "t",
})
return alt_tech_data
137 changes: 92 additions & 45 deletions core/utils_aws.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,12 @@
import time
import logging
from datetime import date, datetime, timedelta
from collections import defaultdict
from dateutil.relativedelta import relativedelta
from botocore.exceptions import NoCredentialsError, ClientError

from .utils_db import connect, load_data

logger = logging.getLogger("core.engine.aws")

def aws_api_call_with_retry(client, function_name, parameters, max_retries, retry_delay):
Expand Down Expand Up @@ -59,19 +62,20 @@ def build_aws_resource_inventory(cloud_service_provider, provider_details, repor
region_name=region
)

# Load the ResourceType mapping to include both `id` and `name`
with open("datasets/resourcetype.json", "r", encoding="utf-8") as f:
resource_type_mapping = {
item["code"]: {"id": item["id"], "name": item["name"]}
for item in json.load(f)
if item["csp"] == "2" and item["status"] == "t"
}
db_path = os.path.join(report_path, "data", "assessment.db")

# Load the ResourceType mapping
resource_type_mapping = {
item["code"]: {"id": item["id"], "name": item["name"]}
for item in load_data("resourcetype")
if item["csp"] == 2 and item["status"] == "t"
}

resource_summary = {}
# Save raw data for debugging and auditing purposes
raw_data = []

# Initialize a custom counter
resource_inventory_id_counter = 1
# Aggregate resources by type and location
aggregated_resources = defaultdict(int)

# Iterate through each resource type in the JSON
for idx, (resource_type_code, resource_info) in enumerate(resource_type_mapping.items(), start=1):
Expand Down Expand Up @@ -108,17 +112,9 @@ def build_aws_resource_inventory(cloud_service_provider, provider_details, repor
#logger.warning(f"No valid response found for {service_name} operation {operation_name}. Skipping.")
continue

# Count resources and add to summary if count > 0
resource_count = len(resources)
if resource_count > 0:
resource_inventory_id = str(resource_inventory_id_counter)
resource_summary[resource_inventory_id] = {
"resource_name": resource_info["name"],
"resource_type": resource_info["id"],
"location": region,
"count": resource_count
}
resource_inventory_id_counter += 1
# Aggregate the resources
for resource in resources:
aggregated_resources[(resource_type_code, region)] += 1

# Store raw data
raw_data.append({
Expand All @@ -127,9 +123,6 @@ def build_aws_resource_inventory(cloud_service_provider, provider_details, repor
"resources": resources
})


#logger.info(f"Processed {resource_count} resources for service {service_name} with operation {operation_name}")

except (NoCredentialsError, ClientError, Exception) as e:
#logger.error(f"Error while processing {service_name}: {str(e)}", exc_info=True)
continue
Expand All @@ -140,13 +133,35 @@ def build_aws_resource_inventory(cloud_service_provider, provider_details, repor
raw_file_path = os.path.join(raw_data_path, "resource_inventory_raw_data.json")
with open(raw_file_path, "w", encoding="utf-8") as raw_file:
json.dump(raw_data, raw_file, indent=4)
#logger.info(f"AWS raw resource inventory saved to {raw_file_path}")

# Save structured data to a JSON file
structured_file_path = os.path.join(report_path, "resource_inventory_standard_data.json")
with open(structured_file_path, "w", encoding="utf-8") as structured_file:
json.dump(resource_summary, structured_file, indent=4)
#logger.info(f"AWS structured resource inventory saved to {structured_file_path}")
# Insert aggregated data into SQLite
with connect(db_path=db_path) as conn:
cursor = conn.cursor()

for (resource_type_code, resource_location), resource_count in aggregated_resources.items():
try:
# Map resource type code to resource_type_id
resource_info = resource_type_mapping.get(resource_type_code)
if not resource_info:
#logger.warning(f"Resource type {resource_type_code} not found in resourcetype mapping. Skipping.")
continue

resource_type_id = resource_info["id"]

cursor.execute(
"""
INSERT INTO resource_inventory (resource_type, location, count)
VALUES (?, ?, ?)
ON CONFLICT(resource_type, location) DO UPDATE SET count = excluded.count
""",
(resource_type_id, resource_location, resource_count)
)
except sqlite3.Error as e:
logger.error(f"SQLite error while processing aggregated resource: {e}", exc_info=True)
except Exception as e:
logger.error(f"Unexpected error while processing aggregated resource: {e}", exc_info=True)

conn.commit()

except Exception as e:
logger.error(f"Error creating AWS resource inventory: {str(e)}", exc_info=True)
Expand All @@ -172,6 +187,8 @@ def build_aws_cost_inventory(cloud_service_provider, provider_details, report_pa
)
cost_explorer = session.client('ce', region_name='us-east-1')

db_path = os.path.join(report_path, "data", "assessment.db")

end_time = date.today()
start_time = end_time.replace(day=1) - timedelta(days=180)
start_time = start_time.replace(day=1)
Expand All @@ -189,24 +206,54 @@ def build_aws_cost_inventory(cloud_service_provider, provider_details, report_pa
}
)

cost_inventory_raw_path = os.path.join(report_path, "cost_inventory_raw_data.json")
cost_inventory_raw_path = os.path.join(raw_data_path, "cost_inventory_raw_data.json")
with open(cost_inventory_raw_path, "w", encoding="utf-8") as raw_file:
json.dump(cost_and_usage, raw_file, indent=4)

structured_costs = {}
for result in cost_and_usage['ResultsByTime']:
month_str = result['TimePeriod']['Start']
total_cost = sum(float(group['Metrics']['UnblendedCost']['Amount']) for group in result['Groups'])
currency = result['Groups'][0]['Metrics']['UnblendedCost']['Unit'] if result['Groups'] else 'USD'
structured_costs[month_str] = {"cost": total_cost, "currency": currency}

missing_months = get_missing_months_aws(structured_costs.keys(), 6)
for missing_month in missing_months:
structured_costs[missing_month.isoformat()] = {"cost": 0.00, "currency": currency}

cost_inventory_standard_path = os.path.join(report_path, "cost_inventory_standard_data.json")
with open(cost_inventory_standard_path, "w", encoding="utf-8") as structured_file:
json.dump(structured_costs, structured_file, indent=4)
# Insert structured data into SQLite
with connect(db_path=db_path) as conn:
cursor = conn.cursor()

for result in cost_and_usage['ResultsByTime']:
month_str = result['TimePeriod']['Start']
total_cost = sum(float(group['Metrics']['UnblendedCost']['Amount']) for group in result['Groups'])
currency = result['Groups'][0]['Metrics']['UnblendedCost']['Unit'] if result['Groups'] else 'USD'
month_date = datetime.strptime(month_str, '%Y-%m-%d').date().replace(day=1).isoformat()

# Insert or update the cost data for the month
cursor.execute(
"""
INSERT INTO cost_inventory (month, cost, currency)
VALUES (?, ?, ?)
ON CONFLICT(month) DO UPDATE SET
cost = excluded.cost,
currency = excluded.currency
""",
(month_date, total_cost, currency)
)

# Handle missing months
structured_months = {datetime.strptime(result['TimePeriod']['Start'], '%Y-%m-%d').date() for result in cost_and_usage['ResultsByTime']}
missing_months = get_missing_months_aws({month.isoformat() for month in structured_months}, 6)

for missing_month in missing_months:
cursor.execute(
"""
INSERT INTO cost_inventory (month, cost, currency)
VALUES (?, 0.00, ?)
ON CONFLICT(month) DO UPDATE SET
currency = excluded.currency
""",
(missing_month.isoformat(), currency)
)

conn.commit()

except sqlite3.Error as e:
logger.error(f"SQLite error: {str(e)}", exc_info=True)
except Exception as e:
logger.error(f"Error creating AWS cost inventory: {str(e)}", exc_info=True)
raise

except Exception as e:
logger.error(f"Error creating AWS cost inventory: {str(e)}", exc_info=True)
Expand Down
Loading