Skip to content

Commit

Permalink
Merge 69e6d6c into 09d0c2a
Browse files Browse the repository at this point in the history
  • Loading branch information
direnakkoc committed Jul 12, 2022
2 parents 09d0c2a + 69e6d6c commit 61a373c
Show file tree
Hide file tree
Showing 16 changed files with 603 additions and 10 deletions.
28 changes: 28 additions & 0 deletions README.md
Expand Up @@ -158,6 +158,18 @@ SNS Topic dashboard widgets show:
|--|--|
|![Invalid Attributes](https://raw.githubusercontent.com/fourtheorem/slic-watch/main/docs/snsInvalidAttributes.png)|![Notifications Failed](https://raw.githubusercontent.com/fourtheorem/slic-watch/main/docs/snsNotificationsFailed.png) |

### EventBridge

EventBridge alarms are created for:
1. Failed Invocations
2. ThrottledRules

EventBridge Rule dashboard widgets show:

|FailedInvocations|Invocations|ThrottledRules|
|--|--|
|![FailedInvocations](https://raw.githubusercontent.com/fourtheorem/slic-watch/main/docs/eventBridgeFailedInvocations.png)|![Invocations](https://raw.githubusercontent.com/fourtheorem/slic-watch/main/docs/eventBridgeInvocations.png)|

## Configuration

Configuration is entirely optional - SLIC Watch provides defaults that work out of the box.
Expand Down Expand Up @@ -286,6 +298,14 @@ custom:
NumberOfNotificationsFailed:
Statistic: Sum
Threshold: 1
Events:
#EventBridge
FailedInvocations:
Statistic: Sum
Threshold: 1
ThrottledRules:
Statistic: Sum
Threshold: 1

dashboard:
enabled: true
Expand Down Expand Up @@ -371,6 +391,14 @@ custom:
Statistic: ["Sum"]
NumberOfNotificationsFailed:
Statistic: ["Sum"]
Events:
#EventBridge
FailedInvocations:
Statistic: ["Sum"]
ThrottledRules:
Statistic: ["Sum"]
Invocations:
Statistic: ["Sum"]
```

An example project is provided for reference: [serverless-test-project](./serverless-test-project)
Expand Down
Binary file added docs/eventBridgeFailedInvocations.png
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added docs/eventBridgeInvocations.png
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
119 changes: 119 additions & 0 deletions serverless-plugin/alarms-eventbridge.js
@@ -0,0 +1,119 @@
'use strict'

/**
* @param {object} eventsAlarmsConfig The fully resolved alarm configuration
*/
module.exports = function eventsAlarms (eventsAlarmsConfig, context) {
return {
createRuleAlarms
}

/**
* Add all required Events alarms to the provided CloudFormation template
* based on the EventBridge Rule found within
*
* @param {CloudFormationTemplate} cfTemplate A CloudFormation template object
*/
function createRuleAlarms (cfTemplate) {
const ruleResources = cfTemplate.getResourcesByType(
'AWS::Events::Rule'
)

for (const [ruleResourceName, ruleResource] of Object.entries(ruleResources)) {
if (eventsAlarmsConfig.FailedInvocations.enabled) {
const failedInvocations = createFailedInvocationsAlarm(
ruleResourceName,
ruleResource,
eventsAlarmsConfig.FailedInvocations
)
cfTemplate.addResource(failedInvocations.resourceName, failedInvocations.resource)
}

if (eventsAlarmsConfig.ThrottledRules.enabled) {
const throttledRules = createThrottledRulesAlarm(
ruleResourceName,
ruleResource,
eventsAlarmsConfig.ThrottledRules
)
cfTemplate.addResource(throttledRules.resourceName, throttledRules.resource)
}
}
}

function createRuleAlarm (
alarmName,
alarmDescription,
ruleName,
comparisonOperator,
threshold,
metricName,
statistic,
period,
evaluationPeriods,
treatMissingData
) {
const metricProperties = {
Dimensions: [{ Name: 'RuleName', Value: ruleName }],
MetricName: metricName,
Namespace: 'AWS/Events',
Period: period,
Statistic: statistic
}

return {
Type: 'AWS::CloudWatch::Alarm',
Properties: {
ActionsEnabled: true,
AlarmActions: context.alarmActions,
AlarmName: alarmName,
AlarmDescription: alarmDescription,
EvaluationPeriods: evaluationPeriods,
ComparisonOperator: comparisonOperator,
Threshold: threshold,
TreatMissingData: treatMissingData,
...metricProperties
}
}
}

function createFailedInvocationsAlarm (ruleResourceName, ruleResource, config) {
const ruleName = ruleResource.Properties.Name
const threshold = config.Threshold

return {
resourceName: `slicWatchEventsFailedInvocationsAlarm${ruleResourceName}`,
resource: createRuleAlarm(
`EventsFailedInvocationsAlarm_${ruleName}`, // alarmName
`Failed Invocations for ${ruleName} breaches ${threshold}`, // alarmDescription
ruleName,
config.ComparisonOperator,
threshold,
'FailedInvocations', // metricName
config.Statistic,
config.Period,
config.EvaluationPeriods,
config.TreatMissingData
)
}
}

function createThrottledRulesAlarm (ruleResourceName, ruleResource, config) {
const ruleName = ruleResource.Properties.Name
const threshold = config.Threshold
return {
resourceName: `slicWatchEventsThrottledRulesAlarm${ruleResourceName}`,
resource: createRuleAlarm(
`EventsThrottledRulesAlarm_${ruleName}`, // alarmName
`Throttled RulesAlarm for ${ruleName} breaches ${threshold}`, // alarmDescription
ruleName,
config.ComparisonOperator,
threshold,
'ThrottledRules', // metricName
config.Statistic,
config.Period,
config.EvaluationPeriods,
config.TreatMissingData
)
}
}
}
6 changes: 5 additions & 1 deletion serverless-plugin/alarms.js
Expand Up @@ -11,6 +11,7 @@ const kinesisAlarms = require('./alarms-kinesis')
const sqsAlarms = require('./alarms-sqs')
const ecsAlarms = require('./alarms-ecs')
const snsAlarms = require('./alarms-sns')
const ruleAlarms = require('./alarms-eventbridge')

module.exports = function alarms (serverless, alarmConfig, functionAlarmConfigs, context) {
const {
Expand All @@ -21,7 +22,8 @@ module.exports = function alarms (serverless, alarmConfig, functionAlarmConfigs,
SQS: sqsConfig,
Lambda: lambdaConfig,
ECS: ecsConfig,
SNS: snsConfig
SNS: snsConfig,
Events: ruleConfig
} = cascade(alarmConfig)

const cascadedFunctionAlarmConfigs = applyAlarmConfig(lambdaConfig, functionAlarmConfigs)
Expand All @@ -33,6 +35,7 @@ module.exports = function alarms (serverless, alarmConfig, functionAlarmConfigs,
const { createSQSAlarms } = sqsAlarms(sqsConfig, context, serverless)
const { createECSAlarms } = ecsAlarms(ecsConfig, context, serverless)
const { createSNSAlarms } = snsAlarms(snsConfig, context, serverless)
const { createRuleAlarms } = ruleAlarms(ruleConfig, context, serverless)

return {
addAlarms
Expand All @@ -54,6 +57,7 @@ module.exports = function alarms (serverless, alarmConfig, functionAlarmConfigs,
sqsConfig.enabled && createSQSAlarms(cfTemplate)
ecsConfig.enabled && createECSAlarms(cfTemplate)
snsConfig.enabled && createSNSAlarms(cfTemplate)
ruleConfig.enabled && createRuleAlarms(cfTemplate)
}
}
}
6 changes: 4 additions & 2 deletions serverless-plugin/config-schema.js
Expand Up @@ -19,7 +19,8 @@ const supportedAlarms = {
Kinesis: ['GetRecords.IteratorAgeMilliseconds', 'ReadProvisionedThroughputExceeded', 'WriteProvisionedThroughputExceeded', 'PutRecord.Success', 'PutRecords.Success', 'GetRecords.Success'],
SQS: ['AgeOfOldestMessage', 'InFlightMessagesPc'],
ECS: ['MemoryUtilization', 'CPUUtilization'],
SNS: ['NumberOfNotificationsFilteredOut-InvalidAttributes', 'NumberOfNotificationsFailed']
SNS: ['NumberOfNotificationsFilteredOut-InvalidAttributes', 'NumberOfNotificationsFailed'],
Events: ['FailedInvocations', 'ThrottledRules']
}

const supportedWidgets = {
Expand All @@ -30,7 +31,8 @@ const supportedWidgets = {
Kinesis: ['GetRecords.IteratorAgeMilliseconds', 'ReadProvisionedThroughputExceeded', 'WriteProvisionedThroughputExceeded', 'PutRecord.Success', 'PutRecords.Success', 'GetRecords.Success'],
SQS: ['NumberOfMessagesSent', 'NumberOfMessagesReceived', 'NumberOfMessagesDeleted', 'ApproximateAgeOfOldestMessage', 'ApproximateNumberOfMessagesVisible'],
ECS: ['MemoryUtilization', 'CPUUtilization'],
SNS: ['NumberOfNotificationsFilteredOut-InvalidAttributes', 'NumberOfNotificationsFailed']
SNS: ['NumberOfNotificationsFilteredOut-InvalidAttributes', 'NumberOfNotificationsFailed'],
Events: ['FailedInvocations', 'ThrottledRules', 'Invocations']
}

const commonAlarmProperties = {
Expand Down
47 changes: 45 additions & 2 deletions serverless-plugin/dashboard.js
Expand Up @@ -22,7 +22,8 @@ module.exports = function dashboard (serverless, dashboardConfig, functionDashbo
Kinesis: kinesisDashConfig,
SQS: sqsDashConfig,
ECS: ecsDashConfig,
SNS: snsDashConfig
SNS: snsDashConfig,
Events: ruleDashConfig
}
} = cascade(dashboardConfig)

Expand Down Expand Up @@ -61,6 +62,9 @@ module.exports = function dashboard (serverless, dashboardConfig, functionDashbo
const topicResources = cfTemplate.getResourcesByType(
'AWS::SNS::Topic'
)
const ruleResources = cfTemplate.getResourcesByType(
'AWS::Events::Rule'
)

const eventSourceMappingFunctions = cfTemplate.getEventSourceMappingFunctions()
const apiWidgets = createApiWidgets(apiResources)
Expand All @@ -74,6 +78,7 @@ module.exports = function dashboard (serverless, dashboardConfig, functionDashbo
const queueWidgets = createQueueWidgets(queueResources)
const ecsWidgets = createEcsWidgets(ecsServiceResources)
const topicWidgets = createTopicWidgets(topicResources)
const ruleWidgets = createRuleWidgets(ruleResources)

const positionedWidgets = layOutWidgets([
...apiWidgets,
Expand All @@ -83,7 +88,8 @@ module.exports = function dashboard (serverless, dashboardConfig, functionDashbo
...streamWidgets,
...queueWidgets,
...ecsWidgets,
...topicWidgets
...topicWidgets,
...ruleWidgets
])

if (positionedWidgets.length > 0) {
Expand Down Expand Up @@ -501,6 +507,43 @@ module.exports = function dashboard (serverless, dashboardConfig, functionDashbo
return topicWidgets
}

/**
* Create a set of CloudWatch Dashboard widgets for SNS services.
*
* @param {object} ruleResources Object of SNS Service resources by resource name
*/
function createRuleWidgets (ruleResources) {
const ruleWidgets = []
for (const res of Object.values(ruleResources)) {
const ruleName = res.Properties.Name

const widgetMetrics = []
for (const [metric, metricConfig] of Object.entries(getConfiguredMetrics(ruleDashConfig))) {
if (metricConfig.enabled) {
for (const stat of metricConfig.Statistic) {
widgetMetrics.push({
namespace: 'AWS/Events',
metric,
dimensions: {
RuleName: ruleName
},
stat
})
}
}
}
if (widgetMetrics.length > 0) {
const metricStatWidget = createMetricWidget(
`Events rule ${ruleName}`,
widgetMetrics,
ruleDashConfig
)
ruleWidgets.push(metricStatWidget)
}
}
return ruleWidgets
}

/**
* Set the location and dimension properties of each provided widget
*
Expand Down
17 changes: 16 additions & 1 deletion serverless-plugin/default-config.yaml
Expand Up @@ -101,6 +101,14 @@ alarms:
NumberOfNotificationsFailed:
Statistic: Sum
Threshold: 1
Events:
#EventBridge
FailedInvocations:
Statistic: Sum
Threshold: 1
ThrottledRules:
Statistic: Sum
Threshold: 1

dashboard:
enabled: true
Expand Down Expand Up @@ -185,5 +193,12 @@ dashboard:
Statistic: ["Sum"]
NumberOfNotificationsFailed:
Statistic: ["Sum"]

Events:
#EventBridge
FailedInvocations:
Statistic: ["Sum"]
ThrottledRules:
Statistic: ["Sum"]
Invocations:
Statistic: ["Sum"]

0 comments on commit 61a373c

Please sign in to comment.