Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Sumo 233810 fix unit testing automation #69

Merged
merged 15 commits into from
Jan 19, 2024
Merged

Sumo 233810 fix unit testing automation #69

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
c599abf
arm template update
priyansh-sumo Jan 11, 2024
3f4ca38
Modification to unittest base on latest version
priyansh-sumo Jan 11, 2024
acbb828
Resolved PR comments
priyansh-sumo Jan 12, 2024
03fd691
PR modification in ARM Template
priyansh-sumo Jan 12, 2024
c9864fb
set os.environ and set timespan
priyansh-sumo Jan 15, 2024
fd68790
change debug to verbose
priyansh-sumo Jan 15, 2024
17cc24a
revert to log check
priyansh-sumo Jan 17, 2024
4ac3cce
Unit testing modifications
priyansh-sumo Jan 17, 2024
3c33228
revert to earlier
priyansh-sumo Jan 17, 2024
66e6ff3
Added sleep of 5 min for log check
priyansh-sumo Jan 17, 2024
fcc8a29
Modification log to verbose
priyansh-sumo Jan 17, 2024
aa7d7ff
app_insights in check functions as parameter
priyansh-sumo Jan 18, 2024
d3f280d
app_insights check function updates
priyansh-sumo Jan 18, 2024
3fe6fce
Modification in unit test check and query
priyansh-sumo Jan 18, 2024
c64b52f
removed service bus and modified other changes.
priyansh-sumo Jan 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions EventHubs/requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
requests==2.20.0
azure-mgmt-resource==1.2.2
azure-mgmt-eventhub==1.2.0
azure-servicebus==0.21.1
azure-cosmosdb-table==1.0.2
azure-mgmt-storage==1.0.0

requests==2.31.0
azure-monitor-query==1.2.0
azure-identity==1.15.0
azure-core==1.29.6
azure-mgmt-resource==23.0.1
azure-mgmt-eventhub==11.0.0
azure-eventhub==5.11.5
azure-mgmt-storage==21.1.0
zure-mgmt-loganalytics==12.0.0
70 changes: 36 additions & 34 deletions EventHubs/src/azuredeploy_metrics.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,30 +3,30 @@
"contentVersion": "1.0.0.0",
"parameters": {
"sites_SumoMetricsFunctionApp_name": {
"defaultValue": "[concat('SumoMetricsFuncApp', uniqueString(resourceGroup().id))]",
"defaultValue": "[concat('SMFuncApp', uniqueString(resourceGroup().id))]",
"type": "String"
},
"namespaces_SumoMetNamespace_name": {
"defaultValue": "[concat('SumoMetricsNamespace', uniqueString(resourceGroup().id))]",
"namespaces_SumoMetricsNamespace_name": {
himanshu219 marked this conversation as resolved.
Show resolved Hide resolved
"defaultValue": "[concat('SMNamespace', uniqueString(resourceGroup().id))]",
"type": "String"
},
"storageAccounts_sumometapplogs_name": {
"defaultValue": "[concat('sumometlogs', uniqueString(resourceGroup().id))]",
"storageAccounts_SumoMetricsAppLogs_name": {
"defaultValue": "[concat('smapplogs', uniqueString(resourceGroup().id))]",
"type": "String"
},
"serverfarms_SumoMetricsAppServicePlan_name": {
"defaultValue": "[concat('SumoMetricsAppServicePlan', uniqueString(resourceGroup().id))]",
"storageAccounts_SumoMetricsFailedLogs_name": {
"defaultValue": "[concat('smfaillogs', uniqueString(resourceGroup().id))]",
"type": "String"
},
"storageAccounts_sumometricsfailedmsg": {
"defaultValue": "[concat('sumometfail', uniqueString(resourceGroup().id))]",
"serverfarms_SumoMetricsAppServicePlan_name": {
"defaultValue": "[concat('SMAppServicePlan', uniqueString(resourceGroup().id))]",
"type": "String"
},
"eventhubs_insights_metrics_pt1m_name": {
"eventhubs_InsightsMetrics_name": {
"defaultValue": "insights-metrics-pt1m",
"type": "String"
},
"AuthorizationRules_RootManageSharedAccessKey_name": {
"authorizationRules_RootManageSharedAccessKey_name": {
"defaultValue": "RootManageSharedAccessKey",
"type": "String"
},
Expand All @@ -46,16 +46,16 @@
"defaultValue": "true",
"type": "String"
},
"SumoEndpointURL": {
"sumoEndpointURL": {
"defaultValue": "Enter HTTP Source URL",
"type": "String"
},
"appInsightsName": {
"defaultValue": "[concat('SumoMetricsAppInsights', uniqueString(resourceGroup().id))]",
"defaultValue": "[concat('SMAppInsights', uniqueString(resourceGroup().id))]",
"type": "String"
},
"logAnalyticsWorkspaceName": {
"defaultValue": "[concat('SumoMetricsWorkspace', uniqueString(resourceGroup().id))]",
"defaultValue": "[concat('SMWorkspace', uniqueString(resourceGroup().id))]",
"type": "String"
},
"location": {
Expand All @@ -73,7 +73,9 @@
}
}
},
"variables": {},
"variables": {
"appInsightsResourceType": "[concat('microsoft.insights/components/', parameters('appInsightsName'))]"
},
"resources": [
{
"type": "Microsoft.EventHub/namespaces",
Expand All @@ -82,7 +84,7 @@
"tier": "Standard",
"capacity": 1
},
"name": "[parameters('namespaces_SumoMetNamespace_name')]",
"name": "[parameters('namespaces_SumoMetricsNamespace_name')]",
"apiVersion": "2022-10-01-preview",
"location": "[parameters('location')]",
"tags": {},
Expand All @@ -100,7 +102,7 @@
"tier": "Standard"
},
"kind": "Storage",
"name": "[parameters('storageAccounts_sumometapplogs_name')]",
"name": "[parameters('storageAccounts_SumoMetricsAppLogs_name')]",
"apiVersion": "2022-09-01",
"location": "[parameters('location')]",
"tags": {},
Expand Down Expand Up @@ -132,7 +134,7 @@
"tier": "Standard"
},
"kind": "Storage",
"name": "[parameters('storageAccounts_sumometricsfailedmsg')]",
"name": "[parameters('storageAccounts_SumoMetricsFailedLogs_name')]",
"apiVersion": "2022-09-01",
"location": "[parameters('location')]",
"tags": {},
Expand Down Expand Up @@ -201,14 +203,14 @@
{ "name": "FUNCTIONS_WORKER_RUNTIME", "value": "node"},
{ "name": "WEBSITE_NODE_DEFAULT_VERSION", "value": "~18"},
{ "name": "Project", "value": "EventHubs/target/metrics_build/" },
{ "name": "AzureWebJobsStorage", "value": "[concat('DefaultEndpointsProtocol=https;AccountName=', parameters('storageAccounts_sumometapplogs_name'), ';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccounts_sumometapplogs_name')),'2022-09-01').keys[0].value,';EndpointSuffix=', parameters('storageEndpointSuffix'))]" },
{ "name": "APPLICATIONINSIGHTS_CONNECTION_STRING", "value": "[reference(concat('microsoft.insights/components/', parameters('appInsightsName')), '2015-05-01').ConnectionString]" },
{ "name": "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING", "value" : "[concat('DefaultEndpointsProtocol=https;AccountName=', parameters('storageAccounts_sumometapplogs_name'), ';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccounts_sumometapplogs_name')),'2022-09-01').keys[0].value,';EndpointSuffix=', parameters('storageEndpointSuffix'))]"},
{ "name": "AzureWebJobsStorage", "value": "[concat('DefaultEndpointsProtocol=https;AccountName=', parameters('storageAccounts_SumoMetricsAppLogs_name'), ';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccounts_SumoMetricsAppLogs_name')),'2022-09-01').keys[0].value,';EndpointSuffix=', parameters('storageEndpointSuffix'))]" },
{ "name": "APPLICATIONINSIGHTS_CONNECTION_STRING", "value": "[reference(variables('appInsightsResourceType'), '2015-05-01').ConnectionString]" },
{ "name": "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING", "value" : "[concat('DefaultEndpointsProtocol=https;AccountName=', parameters('storageAccounts_SumoMetricsAppLogs_name'), ';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccounts_SumoMetricsAppLogs_name')),'2022-09-01').keys[0].value,';EndpointSuffix=', parameters('storageEndpointSuffix'))]"},
{ "name": "WEBSITE_CONTENTSHARE", "value": "[toLower(parameters('sites_SumoMetricsFunctionApp_name'))]" },
{ "name": "FUNCTION_APP_EDIT_MODE", "value": "readwrite"},
{ "name": "SumoLabsMetricEndpoint", "value": "[parameters('SumoEndpointURL')]" },
{ "name": "AzureEventHubConnectionString", "value": "[concat(listkeys(resourceId('Microsoft.EventHub/namespaces/authorizationRules', parameters('namespaces_SumoMetNamespace_name'),parameters('AuthorizationRules_RootManageSharedAccessKey_name')), '2022-10-01-preview').primaryConnectionString,';EntityPath=',parameters('eventhubs_insights_metrics_pt1m_name'))]" },
{ "name": "StorageConnectionString", "value": "[concat('DefaultEndpointsProtocol=https;AccountName=', parameters('storageAccounts_sumometricsfailedmsg'), ';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccounts_sumometricsfailedmsg')),'2022-09-01').keys[0].value,';EndpointSuffix=', parameters('storageEndpointSuffix'))]" }
{ "name": "SumoLabsMetricEndpoint", "value": "[parameters('sumoEndpointURL')]" },
{ "name": "AzureEventHubConnectionString", "value": "[concat(listkeys(resourceId('Microsoft.EventHub/namespaces/authorizationRules', parameters('namespaces_SumoMetricsNamespace_name'),parameters('authorizationRules_RootManageSharedAccessKey_name')), '2022-10-01-preview').primaryConnectionString,';EntityPath=',parameters('eventhubs_InsightsMetrics_name'))]" },
{ "name": "StorageConnectionString", "value": "[concat('DefaultEndpointsProtocol=https;AccountName=', parameters('storageAccounts_SumoMetricsFailedLogs_name'), ';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccounts_SumoMetricsFailedLogs_name')),'2022-09-01').keys[0].value,';EndpointSuffix=', parameters('storageEndpointSuffix'))]" }
]
}
},
Expand All @@ -229,14 +231,14 @@
],
"dependsOn": [
"[resourceId('Microsoft.Web/serverfarms', parameters('serverfarms_SumoMetricsAppServicePlan_name'))]",
"[resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccounts_sumometapplogs_name'))]",
"[resourceId('Microsoft.EventHub/namespaces/', parameters('namespaces_SumoMetNamespace_name'))]",
"[resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccounts_SumoMetricsAppLogs_name'))]",
"[resourceId('Microsoft.EventHub/namespaces/', parameters('namespaces_SumoMetricsNamespace_name'))]",
"[resourceId('Microsoft.Insights/components', parameters('appInsightsName'))]"
]
},
{
"type": "Microsoft.EventHub/namespaces/AuthorizationRules",
"name": "[concat(parameters('namespaces_SumoMetNamespace_name'), '/', parameters('AuthorizationRules_RootManageSharedAccessKey_name'))]",
"name": "[concat(parameters('namespaces_SumoMetricsNamespace_name'), '/', parameters('authorizationRules_RootManageSharedAccessKey_name'))]",
"apiVersion": "2022-10-01-preview",
"location": "[parameters('location')]",
"scale": null,
Expand All @@ -248,12 +250,12 @@
]
},
"dependsOn": [
"[resourceId('Microsoft.EventHub/namespaces', parameters('namespaces_SumoMetNamespace_name'))]"
"[resourceId('Microsoft.EventHub/namespaces', parameters('namespaces_SumoMetricsNamespace_name'))]"
]
},
{
"type": "Microsoft.EventHub/namespaces/eventhubs",
"name": "[concat(parameters('namespaces_SumoMetNamespace_name'), '/', parameters('eventhubs_insights_metrics_pt1m_name'))]",
"name": "[concat(parameters('namespaces_SumoMetricsNamespace_name'), '/', parameters('eventhubs_InsightsMetrics_name'))]",
"apiVersion": "2022-10-01-preview",
"location": "[parameters('location')]",
"scale": null,
Expand All @@ -263,19 +265,19 @@
"status": "Active"
},
"dependsOn": [
"[resourceId('Microsoft.EventHub/namespaces', parameters('namespaces_SumoMetNamespace_name'))]"
"[resourceId('Microsoft.EventHub/namespaces', parameters('namespaces_SumoMetricsNamespace_name'))]"
]
},
{
"type": "Microsoft.EventHub/namespaces/eventhubs/consumergroups",
"name": "[concat(parameters('namespaces_SumoMetNamespace_name'), '/', parameters('eventhubs_insights_metrics_pt1m_name'), '/', parameters('consumerGroupName'))]",
"name": "[concat(parameters('namespaces_SumoMetricsNamespace_name'), '/', parameters('eventhubs_InsightsMetrics_name'), '/', parameters('consumerGroupName'))]",
"apiVersion": "2022-10-01-preview",
"location": "[parameters('location')]",
"scale": null,
"properties": {},
"dependsOn": [
"[resourceId('Microsoft.EventHub/namespaces', parameters('namespaces_SumoMetNamespace_name'))]",
"[resourceId('Microsoft.EventHub/namespaces/eventhubs', parameters('namespaces_SumoMetNamespace_name'), parameters('eventhubs_insights_metrics_pt1m_name'))]"
"[resourceId('Microsoft.EventHub/namespaces', parameters('namespaces_SumoMetricsNamespace_name'))]",
"[resourceId('Microsoft.EventHub/namespaces/eventhubs', parameters('namespaces_SumoMetricsNamespace_name'), parameters('eventhubs_InsightsMetrics_name'))]"
]
},
{
Expand Down Expand Up @@ -311,4 +313,4 @@
}
}
]
}
}
6 changes: 3 additions & 3 deletions EventHubs/target/logs_build/EventHubs_Logs/sumoclient.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ SumoClient.prototype.flushBucketToSumo = function(metaKey) {
var self = this;
let curOptions = Object.assign({},this.options);

this.context.debug("Flush buffer for metaKey:"+metaKey);
this.context.log.verbose("Flush buffer for metaKey:"+metaKey);

function httpSend(messageArray,data) {
return new Promise( (resolve,reject) => {
Expand Down Expand Up @@ -175,10 +175,10 @@ SumoClient.prototype.flushBucketToSumo = function(metaKey) {

return zlib.gzip(msgArray.join('\n'),function(e,compressed_data){
if (!e) {
self.context.debug("gzip successful");
self.context.log.verbose("gzip successful");
return sumoutils.p_retryMax(httpSend,self.MaxAttempts,self.RetryInterval,[msgArray,compressed_data])
.then(()=> {
self.context.debug("Successfully sent to Sumo after "+self.MaxAttempts);
self.context.log.verbose("Successfully sent to Sumo after "+self.MaxAttempts);
self.success_callback(self.context);}
)
.catch((err) => {
Expand Down
8 changes: 3 additions & 5 deletions EventHubs/target/logs_build/EventHubs_Logs/sumometricclient.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@ var sumoclient = require('./sumoclient');
var bucket = require('./messagebucket');
var sumoutils = require('./sumoutils.js');



var metadataMap = {"category":"X-Sumo-Category","sourceName":"X-Sumo-Name","sourceHost":"X-Sumo-Host"};
/**
* Class to receive metrics to a designated Sumo endpoint. Similar to the Log client is best used independently with a batch of messages so one can track the number
Expand Down Expand Up @@ -119,7 +117,7 @@ SumoMetricClient.prototype.flushBucketToSumo = function(metaKey) {
var self = this;
let curOptions = Object.assign({},this.options);

this.context.debug("Flush METRIC buffer for metaKey:"+metaKey);
this.context.log.verbose("Flush METRIC buffer for metaKey:"+metaKey);

function httpSend(messageArray,data) {

Expand Down Expand Up @@ -172,10 +170,10 @@ SumoMetricClient.prototype.flushBucketToSumo = function(metaKey) {

return zlib.gzip(msgArray.join('\n'),function(e,compressed_data){
if (!e) {
self.context.debug("gzip successful");
self.context.log.verbose("gzip successful");
sumoutils.p_retryMax(httpSend,self.MaxAttempts,self.RetryInterval,[msgArray,compressed_data])
.then(()=> {
self.context.debug("Successfully sent to Sumo after "+self.MaxAttempts);
self.context.log.verbose("Successfully sent to Sumo after "+self.MaxAttempts);
self.success_callback(self.context);}
)
.catch((err) => {
Expand Down
6 changes: 3 additions & 3 deletions EventHubs/target/metrics_build/EventHubs_Metrics/sumoclient.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ SumoClient.prototype.flushBucketToSumo = function(metaKey) {
var self = this;
let curOptions = Object.assign({},this.options);

this.context.debug("Flush buffer for metaKey:"+metaKey);
this.context.log.verbose("Flush buffer for metaKey:"+metaKey);

function httpSend(messageArray,data) {
return new Promise( (resolve,reject) => {
Expand Down Expand Up @@ -175,10 +175,10 @@ SumoClient.prototype.flushBucketToSumo = function(metaKey) {

return zlib.gzip(msgArray.join('\n'),function(e,compressed_data){
if (!e) {
self.context.debug("gzip successful");
self.context.log.verbose("gzip successful");
return sumoutils.p_retryMax(httpSend,self.MaxAttempts,self.RetryInterval,[msgArray,compressed_data])
.then(()=> {
self.context.debug("Successfully sent to Sumo after "+self.MaxAttempts);
self.context.log.verbose("Successfully sent to Sumo after "+self.MaxAttempts);
self.success_callback(self.context);}
)
.catch((err) => {
Expand Down
8 changes: 3 additions & 5 deletions EventHubs/target/metrics_build/EventHubs_Metrics/sumometricclient.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@ var sumoclient = require('./sumoclient');
var bucket = require('./messagebucket');
var sumoutils = require('./sumoutils.js');



var metadataMap = {"category":"X-Sumo-Category","sourceName":"X-Sumo-Name","sourceHost":"X-Sumo-Host"};
/**
* Class to receive metrics to a designated Sumo endpoint. Similar to the Log client is best used independently with a batch of messages so one can track the number
Expand Down Expand Up @@ -119,7 +117,7 @@ SumoMetricClient.prototype.flushBucketToSumo = function(metaKey) {
var self = this;
let curOptions = Object.assign({},this.options);

this.context.debug("Flush METRIC buffer for metaKey:"+metaKey);
this.context.log.verbose("Flush METRIC buffer for metaKey:"+metaKey);

function httpSend(messageArray,data) {

Expand Down Expand Up @@ -172,10 +170,10 @@ SumoMetricClient.prototype.flushBucketToSumo = function(metaKey) {

return zlib.gzip(msgArray.join('\n'),function(e,compressed_data){
if (!e) {
self.context.debug("gzip successful");
self.context.log.verbose("gzip successful");
sumoutils.p_retryMax(httpSend,self.MaxAttempts,self.RetryInterval,[msgArray,compressed_data])
.then(()=> {
self.context.debug("Successfully sent to Sumo after "+self.MaxAttempts);
self.context.log.verbose("Successfully sent to Sumo after "+self.MaxAttempts);
self.success_callback(self.context);}
)
.catch((err) => {
Expand Down
2 changes: 1 addition & 1 deletion EventHubs/target/metrics_build/host.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
"default": "Warning",
"Host.Aggregator": "Trace",
"Host.Results": "Information",
"Function": "Trace"
"Function": "Information"
}
},
"functions": [
Expand Down
Loading