Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Sumo 233810 fix unit testing automation #69

Merged
merged 15 commits into from
Jan 19, 2024
Merged

Sumo 233810 fix unit testing automation #69

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
c599abf
arm template update
priyansh-sumo Jan 11, 2024
3f4ca38
Modification to unittest base on latest version
priyansh-sumo Jan 11, 2024
acbb828
Resolved PR comments
priyansh-sumo Jan 12, 2024
03fd691
PR modification in ARM Template
priyansh-sumo Jan 12, 2024
c9864fb
set os.environ and set timespan
priyansh-sumo Jan 15, 2024
fd68790
change debug to verbose
priyansh-sumo Jan 15, 2024
17cc24a
revert to log check
priyansh-sumo Jan 17, 2024
4ac3cce
Unit testing modifications
priyansh-sumo Jan 17, 2024
3c33228
revert to earlier
priyansh-sumo Jan 17, 2024
66e6ff3
Added sleep of 5 min for log check
priyansh-sumo Jan 17, 2024
fcc8a29
Modification log to verbose
priyansh-sumo Jan 17, 2024
aa7d7ff
app_insights in check functions as parameter
priyansh-sumo Jan 18, 2024
d3f280d
app_insights check function updates
priyansh-sumo Jan 18, 2024
3fe6fce
Modification in unit test check and query
priyansh-sumo Jan 18, 2024
c64b52f
removed service bus and modified other changes.
priyansh-sumo Jan 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions EventHubs/requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
requests==2.20.0
azure-mgmt-resource==1.2.2
azure-mgmt-eventhub==1.2.0
requests==2.31.0
azure-monitor-query==1.2.0
azure-identity==1.15.0
azure-core==1.29.6
azure-mgmt-resource==23.0.1
azure-mgmt-eventhub==11.0.0
azure-servicebus==0.21.1
himanshu219 marked this conversation as resolved.
Show resolved Hide resolved
azure-cosmosdb-table==1.0.2
azure-mgmt-storage==1.0.0

azure-mgmt-storage==21.1.0
72 changes: 37 additions & 35 deletions EventHubs/src/azuredeploy_metrics.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,30 +3,30 @@
"contentVersion": "1.0.0.0",
"parameters": {
"sites_SumoMetricsFunctionApp_name": {
"defaultValue": "[concat('SumoMetricsFuncApp', uniqueString(resourceGroup().id))]",
"defaultValue": "SumoMetricsFuncApp",
himanshu219 marked this conversation as resolved.
Show resolved Hide resolved
"type": "String"
},
"namespaces_SumoMetNamespace_name": {
"defaultValue": "[concat('SumoMetricsNamespace', uniqueString(resourceGroup().id))]",
"namespaces_SumoMetricsNamespace_name": {
himanshu219 marked this conversation as resolved.
Show resolved Hide resolved
"defaultValue": "SumoMetricsNamespace",
"type": "String"
},
"storageAccounts_sumometapplogs_name": {
"defaultValue": "[concat('sumometlogs', uniqueString(resourceGroup().id))]",
"storageAccounts_SumoMetricsAppLogs_name": {
"defaultValue": "sumometricsapplogs",
"type": "String"
},
"serverfarms_SumoMetricsAppServicePlan_name": {
"defaultValue": "[concat('SumoMetricsAppServicePlan', uniqueString(resourceGroup().id))]",
"storageAccounts_SumoMetricsFailedLogs_name": {
"defaultValue": "sumometricsfailedlogs",
"type": "String"
},
"storageAccounts_sumometricsfailedmsg": {
"defaultValue": "[concat('sumometfail', uniqueString(resourceGroup().id))]",
"serverfarms_SumoMetricsAppServicePlan_name": {
"defaultValue": "SumoMetricsAppServicePlan",
"type": "String"
},
"eventhubs_insights_metrics_pt1m_name": {
"eventhubs_InsightsMetrics_name": {
"defaultValue": "insights-metrics-pt1m",
"type": "String"
},
"AuthorizationRules_RootManageSharedAccessKey_name": {
"authorizationRules_RootManageSharedAccessKey_name": {
"defaultValue": "RootManageSharedAccessKey",
"type": "String"
},
Expand All @@ -46,21 +46,21 @@
"defaultValue": "true",
"type": "String"
},
"SumoEndpointURL": {
"sumoEndpointURL": {
"defaultValue": "Enter HTTP Source URL",
"type": "String"
},
"appInsightsName": {
"defaultValue": "[concat('SumoMetricsAppInsights', uniqueString(resourceGroup().id))]",
"defaultValue": "SumoMetricsAppInsights",
"type": "String"
},
"logAnalyticsWorkspaceName": {
"defaultValue": "[concat('SumoMetricsWorkspace', uniqueString(resourceGroup().id))]",
"defaultValue": "SumoMetricsWorkspace",
"type": "String"
},
"location": {
"type": "string",
"defaultValue": "[resourceGroup().location]",
"defaultValue": "eastus",
"metadata": {
"description": "Location for the resources."
}
Expand All @@ -73,7 +73,9 @@
}
}
},
"variables": {},
"variables": {
"appInsightsResourceType": "[concat('microsoft.insights/components/', parameters('appInsightsName'))]"
},
"resources": [
{
"type": "Microsoft.EventHub/namespaces",
Expand All @@ -82,7 +84,7 @@
"tier": "Standard",
"capacity": 1
},
"name": "[parameters('namespaces_SumoMetNamespace_name')]",
"name": "[parameters('namespaces_SumoMetricsNamespace_name')]",
"apiVersion": "2022-10-01-preview",
"location": "[parameters('location')]",
"tags": {},
Expand All @@ -100,7 +102,7 @@
"tier": "Standard"
},
"kind": "Storage",
"name": "[parameters('storageAccounts_sumometapplogs_name')]",
"name": "[parameters('storageAccounts_SumoMetricsAppLogs_name')]",
"apiVersion": "2022-09-01",
"location": "[parameters('location')]",
"tags": {},
Expand Down Expand Up @@ -132,7 +134,7 @@
"tier": "Standard"
},
"kind": "Storage",
"name": "[parameters('storageAccounts_sumometricsfailedmsg')]",
"name": "[parameters('storageAccounts_SumoMetricsFailedLogs_name')]",
"apiVersion": "2022-09-01",
"location": "[parameters('location')]",
"tags": {},
Expand Down Expand Up @@ -201,14 +203,14 @@
{ "name": "FUNCTIONS_WORKER_RUNTIME", "value": "node"},
{ "name": "WEBSITE_NODE_DEFAULT_VERSION", "value": "~18"},
{ "name": "Project", "value": "EventHubs/target/metrics_build/" },
{ "name": "AzureWebJobsStorage", "value": "[concat('DefaultEndpointsProtocol=https;AccountName=', parameters('storageAccounts_sumometapplogs_name'), ';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccounts_sumometapplogs_name')),'2022-09-01').keys[0].value,';EndpointSuffix=', parameters('storageEndpointSuffix'))]" },
{ "name": "APPLICATIONINSIGHTS_CONNECTION_STRING", "value": "[reference(concat('microsoft.insights/components/', parameters('appInsightsName')), '2015-05-01').ConnectionString]" },
{ "name": "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING", "value" : "[concat('DefaultEndpointsProtocol=https;AccountName=', parameters('storageAccounts_sumometapplogs_name'), ';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccounts_sumometapplogs_name')),'2022-09-01').keys[0].value,';EndpointSuffix=', parameters('storageEndpointSuffix'))]"},
{ "name": "AzureWebJobsStorage", "value": "[concat('DefaultEndpointsProtocol=https;AccountName=', parameters('storageAccounts_SumoMetricsAppLogs_name'), ';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccounts_SumoMetricsAppLogs_name')),'2022-09-01').keys[0].value,';EndpointSuffix=', parameters('storageEndpointSuffix'))]" },
{ "name": "APPLICATIONINSIGHTS_CONNECTION_STRING", "value": "[reference(variables('appInsightsResourceType'), '2015-05-01').ConnectionString]" },
{ "name": "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING", "value" : "[concat('DefaultEndpointsProtocol=https;AccountName=', parameters('storageAccounts_SumoMetricsAppLogs_name'), ';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccounts_SumoMetricsAppLogs_name')),'2022-09-01').keys[0].value,';EndpointSuffix=', parameters('storageEndpointSuffix'))]"},
{ "name": "WEBSITE_CONTENTSHARE", "value": "[toLower(parameters('sites_SumoMetricsFunctionApp_name'))]" },
{ "name": "FUNCTION_APP_EDIT_MODE", "value": "readwrite"},
{ "name": "SumoLabsMetricEndpoint", "value": "[parameters('SumoEndpointURL')]" },
{ "name": "AzureEventHubConnectionString", "value": "[concat(listkeys(resourceId('Microsoft.EventHub/namespaces/authorizationRules', parameters('namespaces_SumoMetNamespace_name'),parameters('AuthorizationRules_RootManageSharedAccessKey_name')), '2022-10-01-preview').primaryConnectionString,';EntityPath=',parameters('eventhubs_insights_metrics_pt1m_name'))]" },
{ "name": "StorageConnectionString", "value": "[concat('DefaultEndpointsProtocol=https;AccountName=', parameters('storageAccounts_sumometricsfailedmsg'), ';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccounts_sumometricsfailedmsg')),'2022-09-01').keys[0].value,';EndpointSuffix=', parameters('storageEndpointSuffix'))]" }
{ "name": "SumoLabsMetricEndpoint", "value": "[parameters('sumoEndpointURL')]" },
{ "name": "AzureEventHubConnectionString", "value": "[concat(listkeys(resourceId('Microsoft.EventHub/namespaces/authorizationRules', parameters('namespaces_SumoMetricsNamespace_name'),parameters('authorizationRules_RootManageSharedAccessKey_name')), '2022-10-01-preview').primaryConnectionString,';EntityPath=',parameters('eventhubs_InsightsMetrics_name'))]" },
{ "name": "StorageConnectionString", "value": "[concat('DefaultEndpointsProtocol=https;AccountName=', parameters('storageAccounts_SumoMetricsFailedLogs_name'), ';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccounts_SumoMetricsFailedLogs_name')),'2022-09-01').keys[0].value,';EndpointSuffix=', parameters('storageEndpointSuffix'))]" }
]
}
},
Expand All @@ -229,14 +231,14 @@
],
"dependsOn": [
"[resourceId('Microsoft.Web/serverfarms', parameters('serverfarms_SumoMetricsAppServicePlan_name'))]",
"[resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccounts_sumometapplogs_name'))]",
"[resourceId('Microsoft.EventHub/namespaces/', parameters('namespaces_SumoMetNamespace_name'))]",
"[resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccounts_SumoMetricsAppLogs_name'))]",
"[resourceId('Microsoft.EventHub/namespaces/', parameters('namespaces_SumoMetricsNamespace_name'))]",
"[resourceId('Microsoft.Insights/components', parameters('appInsightsName'))]"
]
},
{
"type": "Microsoft.EventHub/namespaces/AuthorizationRules",
"name": "[concat(parameters('namespaces_SumoMetNamespace_name'), '/', parameters('AuthorizationRules_RootManageSharedAccessKey_name'))]",
"name": "[concat(parameters('namespaces_SumoMetricsNamespace_name'), '/', parameters('authorizationRules_RootManageSharedAccessKey_name'))]",
"apiVersion": "2022-10-01-preview",
"location": "[parameters('location')]",
"scale": null,
Expand All @@ -248,12 +250,12 @@
]
},
"dependsOn": [
"[resourceId('Microsoft.EventHub/namespaces', parameters('namespaces_SumoMetNamespace_name'))]"
"[resourceId('Microsoft.EventHub/namespaces', parameters('namespaces_SumoMetricsNamespace_name'))]"
]
},
{
"type": "Microsoft.EventHub/namespaces/eventhubs",
"name": "[concat(parameters('namespaces_SumoMetNamespace_name'), '/', parameters('eventhubs_insights_metrics_pt1m_name'))]",
"name": "[concat(parameters('namespaces_SumoMetricsNamespace_name'), '/', parameters('eventhubs_InsightsMetrics_name'))]",
"apiVersion": "2022-10-01-preview",
"location": "[parameters('location')]",
"scale": null,
Expand All @@ -263,19 +265,19 @@
"status": "Active"
},
"dependsOn": [
"[resourceId('Microsoft.EventHub/namespaces', parameters('namespaces_SumoMetNamespace_name'))]"
"[resourceId('Microsoft.EventHub/namespaces', parameters('namespaces_SumoMetricsNamespace_name'))]"
]
},
{
"type": "Microsoft.EventHub/namespaces/eventhubs/consumergroups",
"name": "[concat(parameters('namespaces_SumoMetNamespace_name'), '/', parameters('eventhubs_insights_metrics_pt1m_name'), '/', parameters('consumerGroupName'))]",
"name": "[concat(parameters('namespaces_SumoMetricsNamespace_name'), '/', parameters('eventhubs_InsightsMetrics_name'), '/', parameters('consumerGroupName'))]",
"apiVersion": "2022-10-01-preview",
"location": "[parameters('location')]",
"scale": null,
"properties": {},
"dependsOn": [
"[resourceId('Microsoft.EventHub/namespaces', parameters('namespaces_SumoMetNamespace_name'))]",
"[resourceId('Microsoft.EventHub/namespaces/eventhubs', parameters('namespaces_SumoMetNamespace_name'), parameters('eventhubs_insights_metrics_pt1m_name'))]"
"[resourceId('Microsoft.EventHub/namespaces', parameters('namespaces_SumoMetricsNamespace_name'))]",
"[resourceId('Microsoft.EventHub/namespaces/eventhubs', parameters('namespaces_SumoMetricsNamespace_name'), parameters('eventhubs_InsightsMetrics_name'))]"
]
},
{
Expand Down Expand Up @@ -311,4 +313,4 @@
}
}
]
}
}
54 changes: 54 additions & 0 deletions EventHubs/src/azuredeploy_metrics.parameters.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
{
himanshu219 marked this conversation as resolved.
Show resolved Hide resolved
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"sites_SumoMetricsFunctionApp_name": {
"value": "SumoMetricsFuncApp"
},
"namespaces_SumoMetricsNamespace_name": {
"value": "SumoMetricsNamespace"
},
"storageAccounts_SumoMetricsAppLogs_name": {
"value": "sumometricsapplogs"
},
"storageAccounts_SumoMetricsFailedLogs_name": {
"value": "sumometricsfailedlogs"
},
"serverfarms_SumoMetricsAppServicePlan_name": {
"value": "SumoMetricsAppServicePlan"
},
"eventhubs_InsightsMetrics_name": {
"value": "insights-metrics-pt1m"
},
"authorizationRules_RootManageSharedAccessKey_name": {
"value": "RootManageSharedAccessKey"
},
"consumerGroupName": {
"value": "$Default"
},
"sourceCodeRepositoryURL": {
"value": "https://github.com/SumoLogic/sumologic-azure-function"
},
"sourceCodeBranch": {
"value": "SUMO-233810-fix-unit-testing-automation"
},
"sourceCodeManualIntegration": {
"value": "true"
},
"sumoEndpointURL": {
"value": "Enter HTTP Source URL"
},
"appInsightsName": {
"value": "SumoMetricsAppInsights"
},
"logAnalyticsWorkspaceName": {
"value": "SumoMetricsWorkspace"
},
"location": {
"value": "eastus"
},
"storageEndpointSuffix": {
"value": "core.windows.net"
}
}
}
69 changes: 39 additions & 30 deletions EventHubs/tests/baseeventhubtest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,32 +1,32 @@
import os
import sys
sys.path.insert(0, '../../test_utils')
from basetest import BaseTest
import json
import time
from time import sleep
from requests import Session
from datetime import datetime
from azure.servicebus import ServiceBusService
from azure.mgmt.resource import ResourceManagementClient
from azure.mgmt.eventhub import EventHubManagementClient
from azure.mgmt.storage import StorageManagementClient
from azure.cosmosdb.table.tableservice import TableService
from azure.servicebus import ServiceBusService

sys.path.insert(0, '../../test_utils')
from basetest import BaseTest

class BaseEventHubTest(BaseTest):

def setUp(self):
self.create_credentials()
self.resource_client = ResourceManagementClient(self.credentials,
self.subscription_id)
self.resource_client = ResourceManagementClient(self.azure_credential, self.subscription_id)
himanshu219 marked this conversation as resolved.
Show resolved Hide resolved
try:
self.sumo_endpoint_url = os.environ["SumoEndpointURL"]
except KeyError:
raise Exception("SumoEndpointURL environment variables are not set")

self.repo_name, self.branch_name = self.get_git_info()

def tearDown(self):
if self.resource_group_exists(self.RESOURCE_GROUP_NAME):
print("RESOURCE_GROUP_NAME...:",self.RESOURCE_GROUP_NAME)
himanshu219 marked this conversation as resolved.
Show resolved Hide resolved
self.delete_resource_group()

def get_resource_name(self, resprefix, restype):
Expand All @@ -49,39 +49,48 @@ def wait_for_table_results(self, query):
sleep(15)
max_retries -= 1

def get_table_service(self):
storage_client = StorageManagementClient(self.credentials,
self.subscription_id)
STORAGE_ACCOUNT_NAME = self.get_resource_name(self.STORAGE_ACCOUNT_NAME, "Microsoft.Storage/storageAccounts")
storage_keys = storage_client.storage_accounts.list_keys(
self.RESOURCE_GROUP_NAME, STORAGE_ACCOUNT_NAME)
acckey = storage_keys.keys[0].value
table_service = TableService(account_name=STORAGE_ACCOUNT_NAME,
account_key=acckey)
return table_service

def insert_mock_logs_in_EventHub(self, filename):
print("Inserting fake logs in EventHub")

himanshu219 marked this conversation as resolved.
Show resolved Hide resolved
defaultauthorule_name = "RootManageSharedAccessKey"
namespace_name = self.get_resource_name(self.event_hub_namespace_prefix, "Microsoft.EventHub/namespaces")
eventhub_client = EventHubManagementClient(self.azure_credential, self.subscription_id)
eventhub_keys = eventhub_client.namespaces.list_keys(self.RESOURCE_GROUP_NAME, namespace_name, defaultauthorule_name)

sbs = ServiceBusService(
namespace_name,
shared_access_key_name=defaultauthorule_name,
shared_access_key_value=eventhub_keys.primary_key,
request_session=Session()
)

mock_logs = json.load(open(filename))
# print("inserting %s" % (mock_logs))
sbs.send_event(self.eventhub_name, json.dumps(mock_logs))
print("Event inserted")

def insert_mock_metrics_in_EventHub(self, filename):
print("Inserting fake logs in EventHub")
himanshu219 marked this conversation as resolved.
Show resolved Hide resolved

defaultauthorule_name = "RootManageSharedAccessKey"

eventhub_client = EventHubManagementClient(self.credentials,
self.subscription_id)

ehkeys = eventhub_client.namespaces.list_keys(
self.RESOURCE_GROUP_NAME, namespace_name, defaultauthorule_name)

namespace_name = self.get_resource_name(self.event_hub_namespace_prefix, "Microsoft.EventHub/namespaces")
eventhub_client = EventHubManagementClient(self.azure_credential, self.subscription_id)
eventhub_keys = eventhub_client.namespaces.list_keys(self.RESOURCE_GROUP_NAME, namespace_name, defaultauthorule_name)

sbs = ServiceBusService(
namespace_name,
shared_access_key_name=defaultauthorule_name,
shared_access_key_value=ehkeys.primary_key,
shared_access_key_value=eventhub_keys.primary_key,
request_session=Session()
)

mock_logs = json.load(open(filename))
himanshu219 marked this conversation as resolved.
Show resolved Hide resolved
print("inserting %s" % (mock_logs))
sbs.send_event(self.eventhub_name, json.dumps(mock_logs))
mock_logs = json.dumps(mock_logs)
mock_logs = mock_logs.replace("2018-03-07T14:23:51.991Z", datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ"))
mock_logs = mock_logs.replace("C088DC46", "%d-%s" % (1, str(int(time.time()))))

# print("inserting %s" % (mock_logs))
sbs.send_event(self.eventhub_name, mock_logs)
print("Event inserted")

def _parse_template(self):
Expand All @@ -92,7 +101,7 @@ def _parse_template(self):
with open(template_path, 'r') as template_file_fd:
template_data = json.load(template_file_fd)

template_data["parameters"]["SumoEndpointURL"]["defaultValue"] = self.sumo_endpoint_url
template_data["parameters"]["sumoEndpointURL"]["defaultValue"] = self.sumo_endpoint_url
template_data["parameters"]["sourceCodeBranch"]["defaultValue"] = self.branch_name
template_data["parameters"]["sourceCodeRepositoryURL"]["defaultValue"] = self.repo_name

Expand Down
Loading