diff --git a/README.md b/README.md index 2e45fa7..55914e9 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,12 @@ endpoints: cachet: api_url: http://status.cachethq.io/api/v1 token: mytoken +webhooks: + - url: "https://push.example.com/message?token=" + params: + title: "{title}" + message: "{message}" + priority: 5 ``` - **endpoints**, the configuration about the URL/Urls that will be monitored. @@ -88,6 +94,9 @@ cachet: - **cachet**, this is the settings for our cachet server. - **api_url**, the cachet API endpoint. *mandatory* - **token**, the API token. *mandatory* +- **webhooks**, generic webhooks to be notified about incident updates + - **url**, webhook URL, will be interpolated + - **params**, POST parameters, will be interpolated Each `expectation` has their own default incident status. It can be overridden by setting the `incident` property to any of the following values: - `PARTIAL` @@ -102,6 +111,13 @@ By choosing any of the aforementioned statuses, it will let you control the kind | LATENCY | PERFORMANCE | | REGEX | PARTIAL | +Following parameters are available in webhook interpolation + +| Parameter | Description | +| --------- | ----------- | +| `{title}` | Event title, includes endpoint name and short status | +| `{message}` | Event message, same as sent to Cachet | + ## Setting up The application should be installed using **virtualenv**, through the following command: diff --git a/cachet_url_monitor/client.py b/cachet_url_monitor/client.py old mode 100644 new mode 100755 diff --git a/cachet_url_monitor/configuration.py b/cachet_url_monitor/configuration.py index eb19744..1139eca 100755 --- a/cachet_url_monitor/configuration.py +++ b/cachet_url_monitor/configuration.py @@ -3,6 +3,8 @@ import logging import time from typing import Dict +from typing import List +from typing import Optional import requests from yaml import dump @@ -12,6 +14,7 @@ from cachet_url_monitor.exceptions import ConfigurationValidationError from cachet_url_monitor.expectation import Expectation from cachet_url_monitor.status import ComponentStatus +from cachet_url_monitor.webhook import Webhook # This is the mandatory fields that must be in the configuration file in this # same exact structure. @@ -25,6 +28,7 @@ class Configuration(object): endpoint_index: int endpoint: str client: CachetClient + webhooks: List[Webhook] current_fails: int trigger_update: bool @@ -43,11 +47,12 @@ class Configuration(object): previous_status: ComponentStatus message: str - def __init__(self, config, endpoint_index: int, client: CachetClient): + def __init__(self, config, endpoint_index: int, client: CachetClient, webhooks : Optional[List[Webhook]] = None): self.endpoint_index = endpoint_index self.data = config self.endpoint = self.data['endpoints'][endpoint_index] self.client = client + self.webhooks = webhooks or [] self.current_fails = 0 self.trigger_update = True @@ -232,6 +237,24 @@ def push_metrics(self): else: self.logger.warning(f'Metric upload failed with status [{metrics_request.status_code}]') + def trigger_webhooks(self): + """Trigger webhooks.""" + if self.status == st.ComponentStatus.PERFORMANCE_ISSUES: + message = self.message + title = f'{self.endpoint["name"]} degraded' + elif self.status == st.ComponentStatus.OPERATIONAL: + message = 'Incident resolved' + title = f'{self.endpoint["name"]} OK' + else: + message = self.message + title = f'{self.endpoint["name"]} unavailable' + for webhook in self.webhooks: + webhook_request = webhook.push_incident(title, message) + if webhook_request.ok: + self.logger.info(f'Webhook {webhook.url} triggered with {title}') + else: + self.logger.warning(f'Webhook {webhook.url} failed with status [{webhook_request.status_code}]') + def push_incident(self): """If the component status has changed, we create a new incident (if this is the first time it becomes unstable) or updates the existing incident once it becomes healthy again. @@ -250,6 +273,8 @@ def push_incident(self): else: self.logger.warning( f'Incident update failed with status [{incident_request.status_code}], message: "{self.message}"') + + self.trigger_webhooks() elif not hasattr(self, 'incident_id') and self.status != st.ComponentStatus.OPERATIONAL: incident_request = self.client.push_incident(self.status, self.public_incidents, self.component_id, message=self.message) @@ -261,3 +286,5 @@ def push_incident(self): else: self.logger.warning( f'Incident upload failed with status [{incident_request.status_code}], message: "{self.message}"') + + self.trigger_webhooks() diff --git a/cachet_url_monitor/scheduler.py b/cachet_url_monitor/scheduler.py index 355e9c7..54439fe 100755 --- a/cachet_url_monitor/scheduler.py +++ b/cachet_url_monitor/scheduler.py @@ -9,6 +9,7 @@ from cachet_url_monitor.client import CachetClient from cachet_url_monitor.configuration import Configuration +from cachet_url_monitor.webhook import Webhook cachet_mandatory_fields = ['api_url', 'token'] @@ -133,9 +134,13 @@ def fatal_error(message): validate_config() + webhooks = [] + for webhook in config_data.get('webhooks', []): + webhooks.append(Webhook(webhook['url'], webhook.get('params', {}))) + for endpoint_index in range(len(config_data['endpoints'])): token = os.environ.get('CACHET_TOKEN') or config_data['cachet']['token'] api_url = os.environ.get('CACHET_API_URL') or config_data['cachet']['api_url'] - configuration = Configuration(config_data, endpoint_index, CachetClient(api_url, token)) + configuration = Configuration(config_data, endpoint_index, CachetClient(api_url, token), webhooks) NewThread(Scheduler(configuration, build_agent(configuration, logging.getLogger('cachet_url_monitor.scheduler')))).start() diff --git a/cachet_url_monitor/webhook.py b/cachet_url_monitor/webhook.py new file mode 100644 index 0000000..f5bbaab --- /dev/null +++ b/cachet_url_monitor/webhook.py @@ -0,0 +1,26 @@ +from typing import Dict, Optional + +import requests + + +class Webhook: + url: str + params: Dict[str, str] + + def __init__(self, url: str, params: Dict[str, str]): + self.url = url + self.params = params + + def push_incident(self, title: str, message: str): + format_args = { + "title": title, + "message": message, + } + # Interpolate URL and params + url = self.url.format(**format_args) + params = { + name: str(value).format(**format_args) + for name, value in self.params.items() + } + + return requests.post(url, params=params) diff --git a/tests/configs/config_webhooks.yml b/tests/configs/config_webhooks.yml new file mode 100644 index 0000000..d2d8fc4 --- /dev/null +++ b/tests/configs/config_webhooks.yml @@ -0,0 +1,32 @@ + +endpoints: + - name: foo + url: http://localhost:8080/swagger + method: GET + header: + SOME-HEADER: SOME-VALUE + timeout: 0.01 + expectation: + - type: HTTP_STATUS + status_range: 200-300 + incident: MAJOR + - type: LATENCY + threshold: 1 + - type: REGEX + regex: '.*(" + params: + title: "{title}" diff --git a/tests/test_configuration.py b/tests/test_configuration.py index a7b87b1..2e8e178 100755 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -9,6 +9,7 @@ import cachet_url_monitor.exceptions import cachet_url_monitor.status +from cachet_url_monitor.webhook import Webhook sys.modules['logging'] = mock.Mock() from cachet_url_monitor.configuration import Configuration @@ -43,6 +44,13 @@ def invalid_config_file(): yield config_file_data +@pytest.fixture() +def webhooks_config_file(): + with open(os.path.join(os.path.dirname(__file__), 'configs/config_webhooks.yml'), 'rt') as yaml_file: + config_file_data = load(yaml_file, SafeLoader) + yield config_file_data + + @pytest.fixture() def mock_logger(): mock_logger = mock.Mock() @@ -59,6 +67,14 @@ def configuration(config_file, mock_client, mock_logger): yield Configuration(config_file, 0, mock_client) +@pytest.fixture() +def webhooks_configuration(webhooks_config_file, mock_client, mock_logger): + webhooks = [] + for webhook in webhooks_config_file.get('webhooks', []): + webhooks.append(Webhook(webhook['url'], webhook.get('params', {}))) + yield Configuration(webhooks_config_file, 0, mock_client, webhooks) + + @pytest.fixture() def multiple_urls_configuration(multiple_urls_config_file, mock_client, mock_logger): yield [Configuration(multiple_urls_config_file, index, mock_client) for index in @@ -130,6 +146,23 @@ def test_evaluate_with_http_error(configuration, mock_logger): mock_logger.exception.assert_called_with('Unexpected HTTP response') +def test_webhooks(webhooks_configuration, mock_logger, mock_client): + assert len(webhooks_configuration.webhooks) == 2 + push_incident_response = mock.Mock() + push_incident_response.ok = False + mock_client.push_incident.return_value = push_incident_response + with requests_mock.mock() as m: + m.get('http://localhost:8080/swagger', exc=requests.HTTPError) + m.post('https://push.example.com/foo%20unavailable', text='') + m.post('https://push.example.com/message?token=%3Capptoken%3E&title=foo+unavailable', text='') + webhooks_configuration.evaluate() + + assert webhooks_configuration.status == cachet_url_monitor.status.ComponentStatus.PARTIAL_OUTAGE, 'Component status set incorrectly' + mock_logger.exception.assert_called_with('Unexpected HTTP response') + webhooks_configuration.push_incident() + mock_logger.info.assert_called_with('Webhook https://push.example.com/message?token= triggered with foo unavailable') + + def test_push_status(configuration, mock_client): mock_client.get_component_status.return_value = cachet_url_monitor.status.ComponentStatus.PARTIAL_OUTAGE push_status_response = mock.Mock()