From f3feb10c4749c93f94e6a23b23ed83fbb7c12097 Mon Sep 17 00:00:00 2001 From: Douglas Laudenschlager Date: Mon, 25 Sep 2017 14:34:06 -0700 Subject: [PATCH] Added 2 new Azure SQL Data Sync sample scripts for use in content. --- .../sql-data-sync/DataSyncLogOmsView.omsview | 286 ++++++++++++++++++ .../DataSyncLogPowerShellRunbook.ps1 | 233 ++++++++++++++ 2 files changed, 519 insertions(+) create mode 100644 samples/features/sql-data-sync/DataSyncLogOmsView.omsview create mode 100644 samples/features/sql-data-sync/DataSyncLogPowerShellRunbook.ps1 diff --git a/samples/features/sql-data-sync/DataSyncLogOmsView.omsview b/samples/features/sql-data-sync/DataSyncLogOmsView.omsview new file mode 100644 index 0000000000..402b7737e3 --- /dev/null +++ b/samples/features/sql-data-sync/DataSyncLogOmsView.omsview @@ -0,0 +1,286 @@ +{ + "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "location": { + "type": "string", + "defaultValue": "" + }, + "resourcegroup": { + "type": "string", + "defaultValue": "" + }, + "subscriptionId": { + "type": "string", + "defaultValue": "" + }, + "workspace": { + "type": "string", + "defaultValue": "" + }, + "workspaceapiversion": { + "type": "string", + "defaultValue": "" + } + }, + "resources": [{ + "apiVersion": "[parameters('workspaceapiversion')]", + "name": "[parameters('workspace')]", + "type": "Microsoft.OperationalInsights/workspaces", + "location": "[parameters('location')]", + "id": "[Concat('/subscriptions/', parameters('subscriptionId'), '/resourceGroups/', parameters('resourcegroup'), '/providers/Microsoft.OperationalInsights/workspaces/', parameters('workspace'))]", + "resources": [{ + "apiVersion": "2015-11-01-preview", + "name": "Data Sync", + "type": "views", + "location": "[parameters('location')]", + "id": "[Concat('/subscriptions/', parameters('subscriptionId'), '/resourceGroups/', parameters('resourcegroup'), '/providers/Microsoft.OperationalInsights/workspaces/', parameters('workspace'),'/views/Data Sync')]", + "dependson": [ + "[Concat('/subscriptions/', parameters('subscriptionId'), '/resourceGroups/', parameters('resourcegroup'), '/providers/Microsoft.OperationalInsights/workspaces/', parameters('workspace'))]" + ], + "properties": { + "Id": "Data Sync", + "Name": "Data Sync", + "Author": "jognanay@microsoft.com", + "Source": "Local", + "Version": 1, + "Dashboard": [{ + "Id": "SingleQueryDonutBuilderBladeV1", + "Type": "Blade", + "Version": 0, + "Configuration": { + "General": { + "title": "All Sync Groups", + "newGroup": false, + "icon": "", + "useIcon": false + }, + "Header": { + "Title": "Error and Warnings", + "Subtitle": "" + }, + "Donut": { + "Query": " LogLevel_s !=Success Type=DataSyncLog_CL TimeStamp_t>NOW-1DAY |measure count() by SyncGroupName_s", + "CenterLegend": { + "Text": "Total", + "Operation": "Sum", + "ArcsToSelect": [] + }, + "Options": { + "colors": [ + "#00188f", + "#0072c6", + "#00bcf2" + ], + "valueColorMapping": [] + } + }, + "List": { + "Query": " LogLevel_s !=Success Type=DataSyncLog_CL TimeStamp_t>NOW-1DAY | measure count() by SyncGroupName_s", + "HideGraph": false, + "enableSparklines": false, + "operation": "Summary", + "ColumnsTitle": { + "Name": "Sync Group", + "Value": "Error and Warning Count" + }, + "Color": "#0072c6", + "thresholds": { + "isEnabled": true, + "values": [{ + "name": "Norefaultmal", + "threshold": "Default", + "color": "#009e49", + "isDefault": true + }, + { + "name": "Good", + "threshold": "0", + "color": "#fcd116", + "isDefault": false + }, + { + "name": "Error", + "threshold": "1", + "color": "#ba141a", + "isDefault": false + } + ] + }, + "NameDSVSeparator": "", + "NavigationQuery": "{selected item}" + } + } + }, + { + "Id": "SingleQueryDonutBuilderBladeV1", + "Type": "Blade", + "Version": 0, + "Configuration": { + "General": { + "title": "Sample Sync Group 1", + "newGroup": false, + "icon": "", + "useIcon": false + }, + "Header": { + "Title": "Contoso Sync", + "Subtitle": "" + }, + "Donut": { + "Query": "SyncGroupName_s=ContosoSync TimeStamp_t>NOW-1DAY| measure count() by LogLevel_s", + "CenterLegend": { + "Text": "", + "Operation": "Sum", + "ArcsToSelect": [] + }, + "Options": { + "colors": [ + "#00188f", + "#0072c6", + "#00bcf2" + ], + "valueColorMapping": [] + } + }, + "List": { + "Query": "LogLevel_s!=Success TimeStamp_t>NOW-1DAY SyncGroupName_s=ContosoSync | select Details_s", + "HideGraph": false, + "enableSparklines": false, + "ColumnsTitle": { + "Name": "Recent Errors and Warnings", + "Value": "" + }, + "Color": "#0072c6", + "operation": "Summary", + "thresholds": { + "isEnabled": false, + "values": [{ + "name": "Normal", + "threshold": "Default", + "color": "#009e49", + "isDefault": true + }, + { + "name": "Warning", + "threshold": "60", + "color": "#fcd116", + "isDefault": false + }, + { + "name": "Error", + "threshold": "90", + "color": "#ba141a", + "isDefault": false + } + ] + }, + "NameDSVSeparator": "", + "NavigationQuery": "{selected item}" + } + } + }, + { + "Id": "SingleQueryDonutBuilderBladeV1", + "Type": "Blade", + "Version": 0, + "Configuration": { + "General": { + "title": "Sample Sync Group 2", + "newGroup": false, + "icon": "", + "useIcon": false + }, + "Header": { + "Title": "Contsoso Sync 2", + "Subtitle": "" + }, + "Donut": { + "Query": "SyncGroupName_s=ContosoSync2 TimeStamp_t>NOW-1DAY | measure count() by LogLevel_s", + "CenterLegend": { + "Text": "Total", + "Operation": "Sum", + "ArcsToSelect": [] + }, + "Options": { + "colors": [ + "#00188f", + "#0072c6", + "#00bcf2" + ], + "valueColorMapping": [] + } + }, + "List": { + "Query": "LogLevel_s=Error TimeStamp_t>NOW-1DAY SyncGroupName_s=ContosoSync2 |select Details_s", + "HideGraph": false, + "enableSparklines": false, + "ColumnsTitle": { + "Name": "Recent Errors and Warnings", + "Value": "" + }, + "Color": "#0072c6", + "operation": "Summary", + "thresholds": { + "isEnabled": false, + "values": [{ + "name": "Normal", + "threshold": "Default", + "color": "#009e49", + "isDefault": true + }, + { + "name": "Warning", + "threshold": "60", + "color": "#fcd116", + "isDefault": false + }, + { + "name": "Error", + "threshold": "90", + "color": "#ba141a", + "isDefault": false + } + ] + }, + "NameDSVSeparator": "", + "NavigationQuery": "{selected item}" + } + } + } + ], + "Filters": [], + "OverviewTile": { + "Id": "SingleQueryDonutBuilderTileV1", + "Type": "OverviewTile", + "Version": 0, + "Configuration": { + "Donut": { + "Query": "Type=DataSyncLog_CL |measure count() by LogLevel_s", + "CenterLegend": { + "Text": "Total", + "Operation": "Sum", + "ArcsToSelect": [] + }, + "Options": { + "colors": [ + "#00188f", + "#0072c6", + "#00bcf2" + ], + "valueColorMapping": [] + } + }, + "Advanced": { + "DataFlowVerification": { + "Enabled": false, + "Query": "*", + "Message": "" + } + } + } + } + } + }] + }] +} \ No newline at end of file diff --git a/samples/features/sql-data-sync/DataSyncLogPowerShellRunbook.ps1 b/samples/features/sql-data-sync/DataSyncLogPowerShellRunbook.ps1 new file mode 100644 index 0000000000..69f2ac599b --- /dev/null +++ b/samples/features/sql-data-sync/DataSyncLogPowerShellRunbook.ps1 @@ -0,0 +1,233 @@ +#Data Sync OMS Integration Runbook + +#To use this script Change all the strings below to reflect your information. + +#Information for Sync Group 1 +#If you want to use all the sync groups in your subscription keep the $DS_xxxx fields empty. +#If you want to use all sync groups in a Resource Group define the $DS_ResourceGroupName. +#If you want to use all sync groups in a Server define $DS_ResourceGroupName and $DS_ServerName. +#If you want to use all sync groups in a Database define $DS_ResourceGroupName, $DS_ServerName and $DS_DatabaseName. +#If you want to use a specific sync group define $DS_ResourceGroupName, $DS_ServerName, $DS_DatabaseName and $DS_SyncGroupName. + +$SubscriptionId = "SubscriptionId" +$DS_ResourceGroupName = "" +$DS_ServerName = "" +$DS_DatabaseName = "" +$DS_SyncGroupName = "" + +$AC_ResourceGroupName = "ResourceGroupName" +$AC_AccountName = "AutomationAccountName" +$AC_LastUpdatedTimeVariableName = "DataSyncLogLastUpdatedTime" + +# Replace with your OMS Workspace ID +$CustomerId = "OMSCustomerID" + +# Replace with your OMS Primary Key +$SharedKey = "SharedKey" + +# Specify the name of the record type that you'll be creating +$LogType = "DataSyncLog" + +# Specify a field with the created time for the records +$TimeStampField = "DateValue" + +#Specify the interval of how often you want to send data to oms +#You can use -hours, -minutes or -days, use a negative number +#$interval = New-TimeSpan -hours -1 + + +#Connect to Azure +$connectionName = "AzureRunAsConnection" +try +{ + # Get the connection "AzureRunAsConnection " + $servicePrincipalConnection=Get-AutomationConnection -Name $connectionName + + "Logging in to Azure..." + Add-AzureRmAccount ` + -ServicePrincipal ` + -TenantId $servicePrincipalConnection.TenantId ` + -ApplicationId $servicePrincipalConnection.ApplicationId ` + -CertificateThumbprint $servicePrincipalConnection.CertificateThumbprint +} +catch { + if (!$servicePrincipalConnection) + { + $ErrorMessage = "Connection $connectionName not found." + throw $ErrorMessage + } else{ + Write-Error -Message $_.Exception + throw $_.Exception + } +} + +# Create the function to create the authorization signature +Function Build-Signature ($customerId, $sharedKey, $date, $contentLength, $method, $contentType, $resource) +{ + $xHeaders = "x-ms-date:" + $date + $stringToHash = $method + "`n" + $contentLength + "`n" + $contentType + "`n" + $xHeaders + "`n" + $resource + + $bytesToHash = [Text.Encoding]::UTF8.GetBytes($stringToHash) + $keyBytes = [Convert]::FromBase64String($sharedKey) + + $sha256 = New-Object System.Security.Cryptography.HMACSHA256 + $sha256.Key = $keyBytes + $calculatedHash = $sha256.ComputeHash($bytesToHash) + $encodedHash = [Convert]::ToBase64String($calculatedHash) + $authorization = 'SharedKey {0}:{1}' -f $customerId,$encodedHash + return $authorization +} + + +# Create the function to create and post the request +Function Post-OMSData($customerId, $sharedKey, $body, $logType) +{ + $method = "POST" + $contentType = "application/json" + $resource = "/api/logs" + $rfc1123date = [DateTime]::UtcNow.ToString("r") + $contentLength = $body.Length + $signature = Build-Signature ` + -customerId $customerId ` + -sharedKey $sharedKey ` + -date $rfc1123date ` + -contentLength $contentLength ` + -fileName $fileName ` + -method $method ` + -contentType $contentType ` + -resource $resource + $uri = "https://" + $customerId + ".ods.opinsights.azure.com" + $resource + "?api-version=2016-04-01" + + $headers = @{ + "Authorization" = $signature; + "Log-Type" = $logType; + "x-ms-date" = $rfc1123date; + "time-generated-field" = $TimeStampField; + } + + $response = Invoke-WebRequest -Uri $uri -Method $method -ContentType $contentType -Headers $headers -Body $body -UseBasicParsing + return $response.StatusCode + +} + + +#Get Log Data +select-azurermsubscription -SubscriptionId $SubscriptionId +$endtime =[System.DateTime]::UtcNow +$StartTime = Get-AzureRmAutomationVariable -ResourceGroupName $AC_ResourceGroupName ` + –AutomationAccountName $AC_AccountName ` + -Name $AC_LastUpdatedTimeVariableName | Select -ExpandProperty Value + + +#Get Log +Write-Output "Getting Data Sync Log from $StartTime to $EndTime" + +if ($DS_ResourceGroupName -eq "") +{ + $ResourceGroupName = Get-AzureRmResourceGroup | select -ExpandProperty ResourceGroupName +} +else +{ + $ResourceGroupName = $DS_ResourceGroupName +} + + foreach ($ResourceGroup in $ResourceGroupName) + { + if ($DS_ServerName -eq "") + { + $ServerName = Get-AzureRmSqlServer -ResourceGroupName $ResourceGroup | select -ExpandProperty ServerName + } + else + { + $ServerName = $DS_ServerName + } + + foreach ($Server in $ServerName) + { + if ($DS_DatabaseName -eq "") + { + $DatabaseName = Get-AzureRmSqlDatabase -ResourceGroupName $ResourceGroup -ServerName $Server | select -ExpandProperty DatabaseName + } + else + { + $DatabaseName = $DS_DatabaseName + } + + foreach ($Database in $DatabaseName) + { + if ($Database -eq "master") + { + continue; + } + + if ($DS_SyncGroupName -eq "") + { + $SyncGroupName = Get-AzureRmSqlSyncGroup -ResourceGroupName $ResourceGroup -ServerName $Server -DatabaseName $Database | select -ExpandProperty SyncGroupName + } + else + { + $SyncGroupName = $DS_SyncGroupName + } + + foreach ($SyncGroup in $SyncGroupName) + { + $Logs = Get-AzureRmSqlSyncGroupLog -ResourceGroupName $ResourceGroup ` + -ServerName $Server ` + -DatabaseName $Database ` + -SyncGroupName $SyncGroup ` + -starttime $StartTime ` + -endtime $EndTime; + + if ($Logs.Length -gt 0) + { + foreach ($Log in $Logs) + { + $Log | Add-Member -Name "SubscriptionId" -Value $SubscriptionId -MemberType NoteProperty + $Log | Add-Member -Name "ResourceGroupName" -Value $ResourceGroup -MemberType NoteProperty + $Log | Add-Member -Name "ServerName" -Value $Server -MemberType NoteProperty + $Log | Add-Member -Name "HubDatabaseName" -Value $Database -MemberType NoteProperty + $Log | Add-Member -Name "SyncGroupName" -Value $SyncGroup -MemberType NoteProperty + + #Filter out Successes to Reduce Data Volume to OMS + #Include the 5 commented out line below to enable the filter + #For($i=0; $i -lt $Log.Length; $i++ ) { + # if($Log[$i].LogLevel -eq "Success") { + # $Log[$i] ="" + # } + # } + + + + } + + + $json = ConvertTo-JSON $logs + + + + Post-OMSData -customerId $customerId -sharedKey $sharedKey -body ([System.Text.Encoding]::UTF8.GetBytes($json)) -logType $logType + if ($result -eq 200) + { + Write-Host "Success" + } + if ($result -ne 200) + { + throw +@" + Posting to OMS Failed   + Runbook Name: DataSyncOMSIntegration   +"@ + } + } + } + } + } + } + + + + Set-AzureRmAutomationVariable -ResourceGroupName $AC_ResourceGroupName ` + –AutomationAccountName $AC_AccountName ` + -Name $AC_LastUpdatedTimeVariableName ` + -Value $EndTime ` + -Encrypted $False