From f1cf85c5189a8c5ed16db5f50ba546b7f9a170bc Mon Sep 17 00:00:00 2001 From: Ryxias Date: Tue, 7 May 2019 10:23:23 -0700 Subject: [PATCH] Make PagerDuty output idempotent; adds support for responder requests (#935) * Alot of changes but requires testing * i funcional 2nd iteration * More fixupes * More improvements * oh god i have so many tests to fix * Fixup more tests * Awesome improvements * Fixes the tests * fixup * Further improves tests and test coverage * Improve test coverage * yep * Coveregae * mgaoebfalwkejlk * More test coverage --- .../alert_processor/outputs/output_base.py | 10 +- .../alert_processor/outputs/pagerduty.py | 566 +++++--- .../test_outputs/test_pagerduty.py | 1283 ++++++++++------- 3 files changed, 1071 insertions(+), 788 deletions(-) diff --git a/stream_alert/alert_processor/outputs/output_base.py b/stream_alert/alert_processor/outputs/output_base.py index ea2d92fe8..bc3f8cfc2 100644 --- a/stream_alert/alert_processor/outputs/output_base.py +++ b/stream_alert/alert_processor/outputs/output_base.py @@ -43,6 +43,10 @@ class OutputRequestFailure(Exception): """OutputRequestFailure handles any HTTP failures""" + def __init__(self, response): + super(OutputRequestFailure, self).__init__() + self.response = response + def retry_on_exception(exceptions): """Decorator function to attempt retry based on passed exceptions""" @@ -222,7 +226,7 @@ def do_put_request(): resp = cls._put_request(url, params, headers, verify) success = cls._check_http_response(resp) if not success: - raise OutputRequestFailure() + raise OutputRequestFailure(resp) return resp return do_put_request() @@ -263,7 +267,7 @@ def do_get_request(): resp = cls._get_request(url, params, headers, verify) success = cls._check_http_response(resp) if not success: - raise OutputRequestFailure() + raise OutputRequestFailure(resp) return resp return do_get_request() @@ -304,7 +308,7 @@ def do_post_request(): resp = cls._post_request(url, data, headers, verify) success = cls._check_http_response(resp) if not success: - raise OutputRequestFailure() + raise OutputRequestFailure(resp) return resp return do_post_request() diff --git a/stream_alert/alert_processor/outputs/pagerduty.py b/stream_alert/alert_processor/outputs/pagerduty.py index 72c76f674..6d6c6c34a 100644 --- a/stream_alert/alert_processor/outputs/pagerduty.py +++ b/stream_alert/alert_processor/outputs/pagerduty.py @@ -109,15 +109,22 @@ def events_v2_data(self, alert, descriptor, routing_key, with_record=True): group = publication.get('@pagerduty-v2.group', None) alert_class = publication.get('@pagerduty-v2.class', None) + # We namespace the dedup_key by the descriptor, preventing situations where a single + # alert sending to multiple PagerDuty services from having colliding dedup_keys, which + # would PROBABLY be ok (because of segregated environments) but why take the risk? + dedup_key = '{}:{}'.format(descriptor, alert.alert_id) + # Structure: https://v2.developer.pagerduty.com/docs/send-an-event-events-api-v2 return { 'routing_key': routing_key, 'event_action': 'trigger', - # Beware of providing this; when this is provided, even if empty string, this will - # cause the dedup_key to be bound to the ALERT, not the incident. The implication - # is that the incident will no longer be searchable with incident_key=dedup_key - # 'dedup_key': '', + # Passing a dedup_key will ensure that only one event is ever created. Any subsequent + # request with the same dedup_key + routing_key + event_action will simply return + # the original result. + # Once the alert is resolved, the dedup_key can be re-used. + # https://v2.developer.pagerduty.com/docs/events-api-v2#alert-de-duplication + 'dedup_key': dedup_key, 'payload': { 'summary': summary, 'source': alert.log_source, @@ -257,6 +264,7 @@ def _dispatch(self, alert, descriptor): { 'type': 'link', 'href': 'https://streamalert.io/', + 'text': 'Link Text' } Image embed @@ -411,14 +419,62 @@ def _dispatch(self, alert, descriptor): class PagerDutyIncidentOutput(OutputDispatcher, EventsV2DataProvider): """PagerDutyIncidentOutput handles all alert dispatching for PagerDuty Incidents REST API - In addition to using the REST API, this PagerDuty implementation also performs automatic - assignment of the incident, based upon context parameters. + In addition to creating an Alert through the EventsV2 API, this output will then find the + PagerDuty Incident that is created and automatically reassign, add more details, set priority, + add a note, and attach any additional responders to the incident. + + + Context: + - assigned_user (string): + Email address of user to assign the incident to. If omitted will default to + the service's default escalation policy. If the email address is not + associated with a user in PagerDuty, it will log a warning and default to + the service's escalation policy. + + - with_record (bool): + True to include the entire record in the Alert's payload. False to omit it. + Will be superseded by certain @pagerduty-v2 fields. + + - note (bool): + A text note that is added to the Incident. Will be superseded by publisher + fields (see below). + + - responders (list): + A list of email addresses of users to add as Requested Responders. If any + email address is not associated with a user in PagerDuty, it will be omitted + and a warning will be logged. + + - responder_message (string) + Text string that shows up in Response Request messages sent to requested + responders. + + + Publishing: + This output has a more complex workflow. The magic publisher fields for @pagerduty-v2 + ALSO are respected by this output. + + - @pagerduty-incident.incident_title (str): + The provided string will show up in the PagerDuty incident's title. + + The child Alert's + title is controlled by other publisher magic fields. - context = { - 'assigned_user': 'somebody@somewhere.somewhere', - 'with_record': True|False, - 'note': 'String goes here' - } + - @pagerduty-incident.note (str): + Due to legacy reasons, this PagerDuty services adds a note containing + "Creating SOX Incident" to the final PagerDuty incident. Providing a string + to this magic field will override that behavior. + + - @pagerduty-incident.urgency (str): + Either "low" or "high". By default urgency is "high" for all incidents. + + + - @pagerduty-incident.incident_body (str): + @deprecated + This is a legacy field that no longer serves any functionality. It populates + a field on the PagerDuty Incident that is never visible. + + + @see Also EventsV2DataProvider for more details """ __service__ = 'pagerduty-incident' INCIDENTS_ENDPOINT = 'incidents' @@ -511,41 +567,6 @@ def get_user_defined_properties(cls): def _dispatch(self, alert, descriptor): """Send incident to Pagerduty Incidents REST API v2 - Context: - - - with_record (bool): - - note (bool): - - Publishing: - This output has a more complex workflow. The magic publisher fields for @pagerduty-v2 - ALSO are respected by this output. - - - @pagerduty-incident.incident_title (str): - The provided string will override the PARENT INCIDENT's title. The child Alert's - title is controlled by other publisher magic fields. - - - @pagerduty-incident.incident_body (str): - This is text that shows up in the body of the newly created incident. - - (!) NOTE: Due to the way incidents are merged, this text is almost never - displayed properly on PagerDuty's UI. The only instance where it - shows up correctly is when incident merging fails and the newly - created incident does not have an alert attached to it. - - - @pagerduty-incident.note (str): - Due to legacy reasons, this PagerDuty services adds a note containing - "Creating SOX Incident" to the final PagerDuty incident. Providing a string - to this magic field will override that behavior. - - - @pagerduty-incident.urgency (str): - Either "low" or "high". By default urgency is "high" for all incidents. - - - In addition, the final event that is merged into the parent incident can be customized - as well. - @see EventsV2DataProvider for more details - - Args: alert (Alert): Alert instance which triggered a rule descriptor (str): Output descriptor @@ -587,20 +608,12 @@ def __init__(self, output_dispatcher, credentials): ) self._events_client = PagerDutyEventsV2ApiClient(http) + # We cache the API User because we may use it multiple times + self._api_user = None + def run(self, alert, descriptor): - """Sets up an assigned incident. - - FIXME (derek.wang): - This work routine is a large, non-atomic set of jobs that can sometimes partially fail. - Partial failures can have side effects on PagerDuty, including the creation of - incomplete or partially complete alerts. Because the Alert Processor will automatically - retry the entire routine from scratch, this can cause partial alerts to get created - redundantly forever. The temporary solution is to delete the erroneous record from - DynamoDB manually, but in the future we should consider writing back state into the - DynamoDB alert record to track the steps involved in "fulfilling" the dispatch of this - alert. - """ - if not self.verify_user_exists(): + """Sets up an assigned incident.""" + if not self._verify_user_exists(): return False # Extracting context data to assign the incident @@ -610,70 +623,131 @@ def run(self, alert, descriptor): publication = compose_alert(alert, self._output, descriptor) - incident = self._create_base_incident(alert, publication, rule_context) - incident_id = incident.get('id') - if not incident or not incident_id: - LOGGER.error('[%s] Could not create main incident', self._output.__service__) - return False - # Create alert to hold all the incident details event = self._create_base_alert_event(alert, descriptor, rule_context) if not event: LOGGER.error('[%s] Could not create incident event', self._output.__service__) return False - # FIXME (derek.wang), see above - # At this point, both the incident and the relevant alert event have been successfully - # created. Any further work that fails the dispatch call will cause the alert to retry - # and redundantly create more incidents and alert events. - # Therefore, the hack is to simply let further failures go by always returning True. - # The tradeoff is that incidents can be created on pagerduty in an incomplete state, - # but this is easier to manage than StreamAlert redundantly creating hundreds (or more!) - # redundant alerts. - stable = True - - # Merge the incident with the event, so we can have a rich context incident - # assigned to a specific person, which the PagerDuty REST API v2 does not allow - merged_incident = self._merge_event_into_incident(incident, event) - if not merged_incident: + # Create an incident to house the alert + incident = self._update_base_incident(event, alert, publication, rule_context) + if not incident: LOGGER.error( - '[%s] Failed to merge alert [%s] into [%s]', - self._output.__service__, - event.get('dedup_key'), - incident_id + '[%s] Failed to update container incident for event', + self._output.__service__ ) - stable = False + return False - if merged_incident: - note = self._add_incident_note(incident, publication, rule_context) - if not note: - LOGGER.error( - '[%s] Failed to add note to incident (%s)', - self._output.__service__, - incident_id + incident_id = incident.get('id', False) + if not incident_id: + LOGGER.error( + '[%s] Incident is missing "id"??', + self._output.__service__ + ) + return False + + # At this point, both the incident and the relevant alert event have been successfully + # created. + # + # All of the code above this line is considered idempotent and can be called repeatedly + # without adverse side effects. Code BELOW this line is neither atomic nor idempotent, so + # we will not retry if any of the below code fails. Instead, we log an error and make a + # best-effort attempt to attach an error note to the PagerDuty incident, signalling that + # it was not setup properly. + # + # In the fateful event the alert gets stuck ANYWAY, the easiest solution is to destroy the + # associated record on the DynamoDB table. + errors = [] + + # Add responder requests + responders = rule_context.get('responders', []) + if responders and not isinstance(responders, list): + responders = [responders] + + if responders: + # The message shows up in the email + default_message = 'An incident was reported that requires your attention.' + responder_message = rule_context.get('responder_message', default_message) + + for responder_email in responders: + result = self._add_incident_response_request( + incident_id, + responder_email, + responder_message ) - stable = False + if not result: + error = '[{}] Failed to request a responder ({}) on incident ({})'.format( + self._output.__service__, + responder_email, + incident_id + ) + LOGGER.error(error) + errors.append(error) + + # Add a note to the incident + note = self._add_incident_note(incident_id, publication, rule_context) + if not note: + error = '[{}] Failed to add note to incident ({})'.format( + self._output.__service__, + incident_id + ) + LOGGER.error(error) + errors.append(error) # If something went wrong, we can't throw an error anymore; log it on the Incident - if not stable: - self._add_instability_note(incident_id) + if errors: + self._add_instability_note(incident_id, errors) return True - def _add_instability_note(self, incident_id): + def _add_instability_note(self, incident_id, errors): + error_section = '\n'.join(['- {}'.format(err) for err in errors]) instability_note = ''' StreamAlert failed to correctly setup this incident. Please contact your StreamAlert administrator. - '''.strip() + +Errors: +{} + '''.format(error_section).strip() self._api_client.add_note(incident_id, instability_note) - def _create_base_incident(self, alert, publication, rule_context): - """Creates a container incident for this alert + def _update_base_incident(self, event, alert, publication, rule_context): + """Given an event, will find the container incident and update it. In PagerDuty's REST API design, Incidents are designed to behave like containers for many - alerts. Unlike alerts, which must obey service escalation policies, Incidents can be given - custom assignments. + alerts. Unlike alerts, Incidents can be given custom assignments and escalation policies. - Returns the newly created incident as a JSON dict. Returns False if anything goes wrong. + When an alert is created through the EventsV2 API, PagerDuty automatically creates an + incident to contain it. The Incident resource that is created is given an "incident_key" + that is identical to the "dedup_key" of the Event. + + Returns the updated incident as a JSON dict. Returns False if anything goes wrong. + """ + incident_key = event.get('dedup_key') + if not incident_key: + LOGGER.error( + '[%s] Event created is missing its "dedup_key"? %s', + self._output.__service__, + event + ) + return False + + event_incident_id = self._get_incident_id_from_event_incident_key(incident_key) + if not event_incident_id: + LOGGER.error( + '[%s] Failed to retrieve Event Incident Id from dedup_key (%s)', + self._output.__service__, + incident_key + ) + return False + + incident_data = self._construct_incident_put_request_data(alert, publication, rule_context) + + return self._api_client.modify_incident(event_incident_id, incident_data) + + def _construct_incident_put_request_data(self, alert, publication, rule_context): + """Builds the payload for an HTTP PUT /incidents/:incident_id request + + Returns it as a JSON dict """ # Presentation defaults @@ -689,13 +763,6 @@ def _create_base_incident(self, alert, publication, rule_context): incident_body = publication.get('@pagerduty-incident.incident_body', default_incident_body) incident_urgency = publication.get('@pagerduty-incident.urgency', default_urgency) - # FIXME (derek.wang) use publisher to set priority instead of context - # Use the priority provided in the context, use it or the incident will be low priority - incident_priority = self.get_standardized_priority(rule_context) - - # FIXME (derek.wang) use publisher to set priority instead of context - assigned_key, assigned_value = self.get_incident_assignment(rule_context) - # https://api-reference.pagerduty.com/#!/Incidents/post_incidents incident_data = { 'incident': { @@ -705,16 +772,50 @@ def _create_base_incident(self, alert, publication, rule_context): 'id': self._incident_service, 'type': 'service_reference' }, - 'priority': incident_priority, - 'incident_key': '', 'body': { 'type': 'incident_body', + # Notably, the incident body is basically useless and doesn't show up on the + # UI if the Incident has an alert attached to it. 'details': incident_body, }, - assigned_key: assigned_value + + # The incident_key behaves very similarly to the deduplication key, but subsequent + # requests to create a second incident with the same incident_key will return an + # HTTP 400 instead of returning the original result. + # https://v2.developer.pagerduty.com/docs/incident-creation-api#making-a-request + # + # The incident_key is a super bizarre field. + # + # AS-FAR-AS-I-CAN-TELL it functions something like this: + # + # - If you create an incident with incident_key A, any subsequent requests to + # create another incident with the same incident_key will return an HTTP 400 + # - If you create an event using EventsV2 API (with or without a dedup_key), the + # associated incident_key of the incident that is automatically created from + # the event will be the same as the dedup_key + # - If you create an event with EventsV2 API and attempt to then create an incident + # with an incident_key that is the same as the dedup_key, instead of returning + # an HTTP 400, it will return the incident that was originally created from the + # EventsV2 API... "idempotently". + # + # 'incident_key': '', } } + incident_priority = self._get_standardized_priority(rule_context) + if incident_priority: + incident_data['incident']['priority'] = incident_priority + + assignments = self._get_incident_assignments(rule_context) + if assignments: + incident_data['incident']['assignments'] = assignments + else: + # Important detail; + # 'assignments' and 'escalation_policy' seem to be exclusive. If you send both, the + # 'escalation_policy' seems to supersede any custom assignments you have. + escalation_policy = self._get_incident_escalation_policy(rule_context) + incident_data['incident']['escalation_policy'] = escalation_policy + # Urgency, if provided, must always be 'high' or 'low' or the API will error if incident_urgency: if incident_urgency in ['low', 'high']: @@ -726,7 +827,39 @@ def _create_base_incident(self, alert, publication, rule_context): incident_urgency ) - return self._api_client.create_incident(incident_data) + return incident_data + + def _get_incident_assignments(self, rule_context): + assignments = False + user_to_assign = rule_context.get('assigned_user', False) + + # If provided, verify the user and get the id from API + if user_to_assign: + user = self._api_client.get_user_by_email(user_to_assign) + if user and user.get('id'): + assignments = [{'assignee': { + 'id': user.get('id'), + 'type': 'user_reference', + }}] + else: + LOGGER.warn( + '[%s] Assignee (%s) could not be found in PagerDuty', + self._output.__service__, + user_to_assign + ) + return assignments + + def _get_incident_escalation_policy(self, rule_context): + # If escalation policy ID was not provided, use default one + policy_id_to_assign = rule_context.get( + 'assigned_policy_id', + self._default_escalation_policy_id + ) + # Assigned to escalation policy ID, return tuple + return { + 'id': policy_id_to_assign, + 'type': 'escalation_policy_reference' + } def _create_base_alert_event(self, alert, descriptor, rule_context): """Creates an alert on REST API v2 @@ -735,6 +868,9 @@ def _create_base_alert_event(self, alert, descriptor, rule_context): either the alert nor the incident itself, but rather a small acknowledgement structure containing a "dedup_key". This key can be used to find the incident that is created. + This method is idempotent. The calls out to PagerDuty will create a new alert+incident, + or return the existing one if this method has already been called. + Returns False if event was not created. """ with_record = rule_context.get('with_record', True) @@ -742,43 +878,28 @@ def _create_base_alert_event(self, alert, descriptor, rule_context): alert, descriptor, self._credentials['integration_key'], - with_record + with_record=with_record ) return self._events_client.enqueue_event(event_data) - def _merge_event_into_incident(self, incident, event): - """Merges the given event into the incident. - - Returns the final, merged incident as a JSON dict. Returns False if anything goes wrong. - """ - # Extract the incident id from the incident that was just created - incident_id = incident.get('id') - if not incident_id: - LOGGER.error('[%s] Incident missing Id?', self._output.__service__) - return False - - # Lookup the incident_key returned as dedup_key to get the incident id - incident_key = event.get('dedup_key') - if not incident_key: - LOGGER.error('[%s] Event missing dedup_key', self._output.__service__) - return False - - # Keep that id to be merged later with the created incident - event_incident_id = self.get_incident_id_from_event_incident_key(incident_key) - if not event_incident_id: + def _add_incident_response_request(self, incident_id, responder_email, message): + responder = self._api_client.get_user_by_email(responder_email) + if not responder: LOGGER.error( - '[%s] Failed to retrieve Event Incident Id from dedup_key (%s)', - self._output.__service__, - incident_key + 'Could not verify if requested incident responder "%s" exists', + responder_email ) return False - # Merge the incident with the event, so we can have a rich context incident - # assigned to a specific person, which the PagerDuty REST API v2 does not allow - return self._api_client.merge_incident(incident_id, event_incident_id) + return bool(self._api_client.request_responder( + incident_id, + self._api_user.get('id'), + message, + responder.get('id') + )) - def _add_incident_note(self, incident, publication, rule_context): + def _add_incident_note(self, incident_id, publication, rule_context): """Adds a note to the incident, when applicable. Returns: @@ -786,11 +907,6 @@ def _add_incident_note(self, incident, publication, rule_context): """ # Add a note to the combined incident to help with triage - merged_id = incident.get('id') - if not merged_id: - LOGGER.error('[%s] Merged incident missing Id?', self._output.__service__) - return False - default_incident_note = 'Creating SOX Incident' # For reverse compatibility reasons incident_note = publication.get( '@pagerduty-incident.note', @@ -804,8 +920,7 @@ def _add_incident_note(self, incident, publication, rule_context): # Simply return early without adding a note; no need to add a blank one return True - return bool(self._api_client.add_note(merged_id, incident_note)) - + return bool(self._api_client.add_note(incident_id, incident_note)) @backoff.on_exception(backoff.constant, PagerdutySearchDelay, @@ -814,12 +929,19 @@ def _add_incident_note(self, incident, publication, rule_context): on_backoff=backoff_handler(), on_success=success_handler(), on_giveup=giveup_handler()) - def get_incident_id_from_event_incident_key(self, incident_key): + def _get_incident_id_from_event_incident_key(self, incident_key): """Queries the API to get the incident id from an incident key When creating an EVENT from the events-v2 API, events are created alongside an incident, but only an incident_key is returned, which is not the same as the incident's REST API resource id. + + (!) WARNING: This method can sometimes fail even if an event was successfully created. + Pagerduty can sometimes experience a small amount of "lag time" between when an + Event is created and when its containing Incident is searchable via this API. + Therefore, all code that calls this method must account for the possibility that this + method can be inconsistent with the state of the "real world", and should retry as + appropriate. """ if not incident_key: return False @@ -830,7 +952,7 @@ def get_incident_id_from_event_incident_key(self, incident_key): return event_incident.get('id') - def verify_user_exists(self): + def _verify_user_exists(self): """Verifies that the 'email_from' provided in the creds is valid and exists.""" user = self._api_client.get_user_by_email(self._email_from) @@ -842,29 +964,35 @@ def verify_user_exists(self): ) return False + self._api_user = user + return True - def get_standardized_priority(self, context): + def _get_standardized_priority(self, context): """Method to verify the existence of a incident priority with the API + Uses the priority provided in the context. When omitted the incident defaults to low + priority. + Args: context (dict): Context provided in the alert record Returns: - dict: JSON object be used in the API call, containing the priority id - and the priority reference, empty if it fails or it does not exist + dict|False: JSON object be used in the API call, containing the priority id + and the priority reference, False if it fails or it does not exist """ if not context: - return dict() + return False + # FIXME (derek.wang) use publisher to set priority instead of context priority_name = context.get('incident_priority', False) if not priority_name: - return dict() + return False priorities = self._api_client.get_priorities() if not priorities: - return dict() + return False # If the requested priority is in the list, get the id priority_id = next( @@ -875,42 +1003,7 @@ def get_standardized_priority(self, context): if priority_id: return {'id': priority_id, 'type': 'priority_reference'} - return dict() - - def get_incident_assignment(self, context): - """Method to determine if the incident gets assigned to a user or an escalation policy - - Incident assignment goes in this order: - Provided user -> provided policy -> default escalation policy - - Args: - context (dict): Context provided in the alert record - - Returns: - tuple: assigned_key (str), assigned_value (dict to assign incident to an escalation - policy or array of dicts to assign incident to users) - """ - # Check if a user to assign the incident is provided - user_to_assign = context.get('assigned_user', False) - - # If provided, verify the user and get the id from API - if user_to_assign: - user = self._api_client.get_user_by_email(user_to_assign) - if user and user.get('id'): - return 'assignments', [{'assignee': { - 'id': user.get('id'), - 'type': 'user_reference', - }}] - - # If escalation policy ID was not provided, use default one - policy_id_to_assign = context.get( - 'assigned_policy_id', - self._default_escalation_policy_id - ) - - # Assigned to escalation policy ID, return tuple - return 'escalation_policy', { - 'id': policy_id_to_assign, 'type': 'escalation_policy_reference'} + return False # pylint: disable=protected-access @@ -929,7 +1022,8 @@ def get(self, url, params, headers=None, verify=False): """Returns the JSON response of the given request, or FALSE on failure""" try: result = self._output_dispatcher._get_request_retry(url, params, headers, verify) - except OutputRequestFailure: + except OutputRequestFailure as e: + LOGGER.error('Encountered HTTP error on GET %s: %s', url, e.response) return False response = result.json() @@ -942,7 +1036,8 @@ def post(self, url, data, headers=None, verify=False): """Returns the JSON response of the given request, or FALSE on failure""" try: result = self._output_dispatcher._post_request_retry(url, data, headers, verify) - except OutputRequestFailure: + except OutputRequestFailure as e: + LOGGER.error('Encountered HTTP error on POST %s: %s', url, e.response) return False response = result.json() @@ -955,7 +1050,8 @@ def put(self, url, params, headers=None, verify=False): """Returns the JSON response of the given request, or FALSE on failure""" try: result = self._output_dispatcher._put_request_retry(url, params, headers, verify) - except OutputRequestFailure: + except OutputRequestFailure as e: + LOGGER.error('Encountered HTTP error on PUT %s: %s', url, e.response) return False response = result.json() @@ -1092,55 +1188,21 @@ def get_escalation_policy_by_id(self, escalation_policy_id): return escalation_policies[0] if escalation_policies else False - def merge_incident(self, parent_incident_id, merged_incident_id): - """Given two incident ids, notifies PagerDuty to merge them into a single incident - - Returns the json representation of the merged incident, or False on failure. - """ - data = { - 'source_incidents': [ - { - 'id': merged_incident_id, - 'type': 'incident_reference' - } - ] - } - merged_incident = self._http_provider.put( - self._get_incident_merge_url(parent_incident_id), - data, - headers=self._construct_headers(), - verify=self._should_do_ssl_verify() - ) - self._update_ssl_verified(merged_incident) - - if not merged_incident: - return False - - return merged_incident.get('incident', False) - - def create_incident(self, incident_data): - """Creates a new incident + def modify_incident(self, incident_id, incident_data): + """Modifies an existing Incident Returns the incident json representation on success, or False on failure. Reference: https://api-reference.pagerduty.com/#!/Incidents/post_incidents - (!) FIXME (derek.wang) - The legacy implementation utilizes this POST /incidents endpoint to create - incidents and merge them with events created through the events-v2 API, but - the PagerDuty API documentation explicitly says to NOT use the REST API - to create incidents. Research if our use of the POST /incidents endpoint is - incorrect. - Reference: https://v2.developer.pagerduty.com/docs/getting-started - Args: incident_data (dict) Returns: dict """ - incident = self._http_provider.post( - self._get_incidents_url(), + incident = self._http_provider.put( + self._get_incident_url(incident_id), incident_data, headers=self._construct_headers(), verify=self._should_do_ssl_verify() @@ -1182,6 +1244,38 @@ def add_note(self, incident_id, note): return note.get('note', False) + def request_responder(self, incident_id, requester_user_id, message, responder_user_id): + # Be very careful with this API endpoint, there are several things you will need to know: + # + # 1) The requester_id MUST match the user associated with the API token + # 2) Both the requester_id and the responder id must have pagerduty accounts. If EITHER + # of them don't, this API endpoint actually exhibits strange behavior; instead of + # returning an HTTP 400 with a useful error message, it will return an HTTP 404. + # 3) You cannot add a requester to an incident that is resolved, it will also 404. + responder_request = self._http_provider.post( + self._get_incident_responder_requests_url(incident_id), + { + 'requester_id': requester_user_id, + 'message': message, + 'responder_request_targets': [ + { + 'responder_request_target': { + 'id': responder_user_id, + 'type': 'user_reference', + } + } + ] + }, + self._construct_headers(), + verify=self._should_do_ssl_verify() + ) + self._update_ssl_verified(responder_request) + + if not responder_request: + return False + + return responder_request.get('responder_request', False) + def _construct_headers(self, omit_email=False): """Returns a dict containing all headers to send for PagerDuty requests @@ -1214,12 +1308,14 @@ def _get_incident_url(self, incident_id): incident_id=incident_id ) - def _get_incident_merge_url(self, incident_id): - return '{incident_url}/merge'.format(incident_url=self._get_incident_url(incident_id)) - def _get_incident_notes_url(self, incident_id): return '{incident_url}/notes'.format(incident_url=self._get_incident_url(incident_id)) + def _get_incident_responder_requests_url(self, incident_id): + return '{incident_url}/responder_requests'.format( + incident_url=self._get_incident_url(incident_id) + ) + def _get_users_url(self): return '{base_url}/users'.format(base_url=self._base_url) diff --git a/tests/unit/stream_alert_alert_processor/test_outputs/test_pagerduty.py b/tests/unit/stream_alert_alert_processor/test_outputs/test_pagerduty.py index cebd0948a..e4382a444 100644 --- a/tests/unit/stream_alert_alert_processor/test_outputs/test_pagerduty.py +++ b/tests/unit/stream_alert_alert_processor/test_outputs/test_pagerduty.py @@ -14,8 +14,9 @@ limitations under the License. """ # pylint: disable=protected-access,attribute-defined-outside-init,too-many-lines,invalid-name +import re from collections import OrderedDict -from mock import patch, PropertyMock, Mock, MagicMock, call +from mock import patch, Mock, MagicMock, call from nose.tools import assert_equal, assert_false, assert_true from stream_alert.alert_processor.outputs.output_base import OutputDispatcher, OutputRequestFailure @@ -111,6 +112,65 @@ def test_dispatch_bad_descriptor(self, log_mock): log_mock.assert_called_with('Failed to send alert to %s:%s', self.SERVICE, 'bad_descriptor') + @patch('stream_alert.alert_processor.outputs.pagerduty.compose_alert') + @patch('logging.Logger.info') + @patch('requests.post') + def test_dispatch_success_with_contexts(self, post_mock, log_mock, compose_alert): + """PagerDutyOutput - Dispatch Success""" + compose_alert.return_value = { + '@pagerduty.contexts': [ + { + 'type': 'link', + 'href': 'https://streamalert.io', + 'text': 'Link text' + }, + { + 'type': 'image', + 'src': 'https://streamalert.io/en/stable/_images/sa-complete-arch.png', + } + ] + } + + RequestMocker.setup_mock(post_mock) + + assert_true(self._dispatcher.dispatch(get_alert(), self.OUTPUT)) + + log_mock.assert_called_with('Successfully sent alert to %s:%s', + self.SERVICE, self.DESCRIPTOR) + + post_mock.assert_called_with( + 'http://pagerduty.foo.bar/create_event.json', + headers=None, + json={ + 'client_url': '', + 'event_type': 'trigger', + 'contexts': [ + { + 'text': 'Link text', + 'href': 'https://streamalert.io', 'type': 'link' + }, + { + 'src': 'https://streamalert.io/en/stable/_images/sa-complete-arch.png', + 'type': 'image' + } + ], + 'client': 'streamalert', + 'details': { + 'record': { + 'compressed_size': '9982', 'node_id': '1', 'cb_server': 'cbserver', + 'timestamp': '1496947381.18', 'md5': '0F9AA55DA3BDE84B35656AD8911A22E1', + 'type': 'binarystore.file.added', + 'file_path': '/tmp/5DA/AD8/0F9AA55DA3BDE84B35656AD8911A22E1.zip', + 'size': '21504' + }, + 'description': 'Info about this rule and what actions to take' + }, + 'service_key': 'mocked_service_key', + 'description': 'StreamAlert Rule Triggered - cb_binarystore_file_added' + }, + timeout=3.05, verify=True + ) + @patch('stream_alert.alert_processor.outputs.output_base.OutputDispatcher.MAX_RETRY_ATTEMPTS', 1) class TestPagerDutyOutputV2(object): @@ -176,6 +236,7 @@ def test_dispatch_sends_correct_request(self, post_mock): 'routing_key': 'mocked_routing_key', 'images': [], 'links': [], + 'dedup_key': 'unit_test_pagerduty-v2:79192344-4a6d-4850-8d06-9c3fef1060a4', }, timeout=3.05, verify=True ) @@ -252,106 +313,119 @@ def test_get_default_properties(self): @patch('requests.post') @patch('requests.get') def test_dispatch_sends_correct_create_request(self, get_mock, post_mock, put_mock): - """PagerDutyIncidentOutput - Dispatch Success, Good User, Sends Correct Create Request""" - # GET /users, /users - json_user = {'users': [{'id': 'valid_user_id'}]} + """PagerDutyIncidentOutput - Dispatch Success, Good User, Sends Correct Create Request - # GET /incidents - json_lookup = {'incidents': [{'id': 'incident_id'}]} + This test ensures that the POST /v2/enqueue call is called with the proper params. + """ + RequestMocker.setup_mock(get_mock) + RequestMocker.setup_mock(post_mock) + RequestMocker.setup_mock(put_mock) - get_mock.return_value.status_code = 200 - get_mock.return_value.json.side_effect = [json_user, json_user, json_lookup] + ctx = {} + self._dispatcher.dispatch(get_alert(context=ctx), self.OUTPUT) - # POST /incidents, /v2/enqueue, /incidents/incident_id/notes - post_mock.return_value.status_code = 200 - json_incident = {'incident': {'id': 'incident_id'}} - json_event = {'dedup_key': 'returned_dedup_key'} - json_note = {'note': {'id': 'note_id'}} - post_mock.return_value.json.side_effect = [json_incident, json_event, json_note] + post_mock.assert_any_call( + 'https://events.pagerduty.com/v2/enqueue', + headers=None, + json={ + 'client_url': None, + 'client': 'StreamAlert', + 'payload': { + 'custom_details': OrderedDict( + [ + ('description', 'Info about this rule and what actions to take'), + ('record', + { + 'compressed_size': '9982', + 'node_id': '1', + 'cb_server': 'cbserver', + 'timestamp': '1496947381.18', + 'md5': '0F9AA55DA3BDE84B35656AD8911A22E1', + 'type': 'binarystore.file.added', + 'file_path': '/tmp/5DA/AD8/0F9AA55DA3BDE84B35656AD8911A22E1.zip', + 'size': '21504' + }) + ] + ), + 'group': None, + 'severity': 'critical', + 'component': None, + 'summary': 'StreamAlert Rule Triggered - cb_binarystore_file_added', + 'source': 'carbonblack:binarystore.file.added', + 'class': None + }, + 'links': [], + 'images': [], + 'event_action': 'trigger', + 'routing_key': 'mocked_key', + 'dedup_key': 'unit_test_pagerduty-incident:79192344-4a6d-4850-8d06-9c3fef1060a4' + }, timeout=3.05, verify=True + ) - # PUT /incidents/indicent_id/merge - put_mock.return_value.status_code = 200 + @patch('requests.put') + @patch('requests.post') + @patch('requests.get') + def test_dispatch_sends_correct_put_incident_request(self, get_mock, post_mock, put_mock): + """PagerDutyIncidentOutput - Dispatch Success, Good User, Sends Correct Update Request - ctx = {'pagerduty-incident': {'assigned_user': 'valid_user'}} + This test ensures that the PUT /incidents/## call is called with the proper params. + """ + RequestMocker.setup_mock(get_mock) + RequestMocker.setup_mock(post_mock) + RequestMocker.setup_mock(put_mock) + ctx = {} self._dispatcher.dispatch(get_alert(context=ctx), self.OUTPUT) - # Useful tidbit; for writing fixtures for implement multiple sequential calls, you can use - # mock.assert_has_calls() to render out all of the calls in order: - # post_mock.assert_has_calls([call()]) - post_mock.assert_any_call( - 'https://api.pagerduty.com/incidents', + put_mock.assert_any_call( + 'https://api.pagerduty.com/incidents/incident_id', headers={ 'From': 'email@domain.com', 'Content-Type': 'application/json', 'Authorization': 'Token token=mocked_token', - 'Accept': 'application/vnd.pagerduty+json;version=2' - }, + 'Accept': 'application/vnd.pagerduty+json;version=2'}, json={ 'incident': { 'body': { 'type': 'incident_body', 'details': 'Info about this rule and what actions to take' }, + 'escalation_policy': { + 'type': 'escalation_policy_reference', + 'id': 'mocked_escalation_policy_id' + }, + 'type': 'incident', 'service': { 'type': 'service_reference', 'id': 'mocked_service_id' }, - 'title': 'StreamAlert Incident - Rule triggered: cb_binarystore_file_added', - 'priority': {}, - 'assignments': [ - { - 'assignee': { - 'type': 'user_reference', 'id': 'valid_user_id' - } - } - ], - 'type': 'incident', - 'incident_key': '', + 'title': 'StreamAlert Incident - Rule triggered: cb_binarystore_file_added' } }, timeout=3.05, verify=False ) - @patch('stream_alert.alert_processor.outputs.pagerduty.compose_alert') @patch('requests.put') @patch('requests.post') @patch('requests.get') - def test_dispatch_sends_correct_with_urgency(self, get_mock, post_mock, put_mock, - compose_alert): - """PagerDutyIncidentOutput - Dispatch Success, Good User, Sends Correct Urgency""" - compose_alert.return_value = { - '@pagerduty-incident.urgency': 'low' - } + def test_dispatch_sends_correct_with_assigned_user(self, get_mock, post_mock, put_mock): + """PagerDutyIncidentOutput - Dispatch Success with Assignee - # GET /users, /users - json_user = {'users': [{'id': 'valid_user_id'}]} - - # GET /incidents - json_lookup = {'incidents': [{'id': 'incident_id'}]} - - get_mock.return_value.status_code = 200 - get_mock.return_value.json.side_effect = [json_user, json_user, json_lookup] - - # POST /incidents, /v2/enqueue, /incidents/incident_id/notes - post_mock.return_value.status_code = 200 - json_incident = {'incident': {'id': 'incident_id'}} - json_event = {'dedup_key': 'returned_dedup_key'} - json_note = {'note': {'id': 'note_id'}} - post_mock.return_value.json.side_effect = [json_incident, json_event, json_note] - - # PUT /incidents/indicent_id/merge - put_mock.return_value.status_code = 200 - - ctx = {'pagerduty-incident': {'assigned_user': 'valid_user'}} + Ensure the PUT call includes assignments when there is an "assigned_user" + """ + RequestMocker.setup_mock(get_mock) + RequestMocker.setup_mock(post_mock) + RequestMocker.setup_mock(put_mock) + ctx = { + 'pagerduty-incident': { + 'assigned_user': 'valid_user' + } + } self._dispatcher.dispatch(get_alert(context=ctx), self.OUTPUT) - # Useful tidbit; for writing fixtures for implement multiple sequential calls, you can use - # mock.assert_has_calls() to render out all of the calls in order: - # post_mock.assert_has_calls([call()]) - post_mock.assert_any_call( - 'https://api.pagerduty.com/incidents', + put_mock.assert_any_call( + 'https://api.pagerduty.com/incidents/incident_id', headers={ 'From': 'email@domain.com', 'Content-Type': 'application/json', @@ -369,7 +443,9 @@ def test_dispatch_sends_correct_with_urgency(self, get_mock, post_mock, put_mock 'id': 'mocked_service_id' }, 'title': 'StreamAlert Incident - Rule triggered: cb_binarystore_file_added', - 'priority': {}, + 'type': 'incident', + + # Assignment here; the valid_user_id comes from the /users API 'assignments': [ { 'assignee': { @@ -377,54 +453,47 @@ def test_dispatch_sends_correct_with_urgency(self, get_mock, post_mock, put_mock } } ], - 'urgency': 'low', - 'type': 'incident', - 'incident_key': '', } }, timeout=3.05, verify=False ) @patch('logging.Logger.warn') - @patch('stream_alert.alert_processor.outputs.pagerduty.compose_alert') + @patch('logging.Logger.info') @patch('requests.put') @patch('requests.post') @patch('requests.get') - def test_dispatch_sends_correct_bad_urgency(self, get_mock, post_mock, put_mock, - compose_alert, log_mock): - """PagerDutyIncidentOutput - Dispatch Success, Good User, Sends Correct Urgency""" - compose_alert.return_value = { - '@pagerduty-incident.urgency': 'asdf' - } + def test_dispatch_sends_correct_with_invalid_assigned_user(self, get_mock, post_mock, put_mock, + log_info_mock, log_warn_mock): + """PagerDutyIncidentOutput - Dispatch Success with invalid Assignee - # GET /users, /users - json_user = {'users': [{'id': 'valid_user_id'}]} + When a user is assigned but the email address is not found in PagerDuty, the call + should still succeed. It should log a warning and default to using the escalation policy. + """ - # GET /incidents - json_lookup = {'incidents': [{'id': 'incident_id'}]} + def invalid_user_matcher(*args, **kwargs): + if args[0] != 'https://api.pagerduty.com/users': + return False - get_mock.return_value.status_code = 200 - get_mock.return_value.json.side_effect = [json_user, json_user, json_lookup] + query = kwargs.get('params', {}).get('query', None) + return query == 'invalid_user' - # POST /incidents, /v2/enqueue, /incidents/incident_id/notes - post_mock.return_value.status_code = 200 - json_incident = {'incident': {'id': 'incident_id'}} - json_event = {'dedup_key': 'returned_dedup_key'} - json_note = {'note': {'id': 'note_id'}} - post_mock.return_value.json.side_effect = [json_incident, json_event, json_note] - - # PUT /incidents/indicent_id/merge - put_mock.return_value.status_code = 200 - - ctx = {'pagerduty-incident': {'assigned_user': 'valid_user'}} + RequestMocker.setup_mock(post_mock) + RequestMocker.setup_mock(put_mock) + RequestMocker.setup_mock( + get_mock, + [ + [invalid_user_matcher, 200, {'users': []}], + ['/users', 200, RequestMocker.USERS_JSON], + ['/incidents', 200, RequestMocker.INCIDENTS_JSON], + ] + ) - self._dispatcher.dispatch(get_alert(context=ctx), self.OUTPUT) + ctx = {'pagerduty-incident': {'assigned_user': 'invalid_user'}} + assert_true(self._dispatcher.dispatch(get_alert(context=ctx), self.OUTPUT)) - # Useful tidbit; for writing fixtures for implement multiple sequential calls, you can use - # mock.assert_has_calls() to render out all of the calls in order: - # post_mock.assert_has_calls([call()]) - post_mock.assert_any_call( - 'https://api.pagerduty.com/incidents', + put_mock.assert_any_call( + 'https://api.pagerduty.com/incidents/incident_id', headers={ 'From': 'email@domain.com', 'Content-Type': 'application/json', @@ -442,186 +511,183 @@ def test_dispatch_sends_correct_bad_urgency(self, get_mock, post_mock, put_mock, 'id': 'mocked_service_id' }, 'title': 'StreamAlert Incident - Rule triggered: cb_binarystore_file_added', - 'priority': {}, - 'assignments': [ - { - 'assignee': { - 'type': 'user_reference', 'id': 'valid_user_id' - } - } - ], 'type': 'incident', - 'incident_key': '', + + # Cannot do assignment, so default to escalation policy + 'escalation_policy': { + 'type': 'escalation_policy_reference', + 'id': 'mocked_escalation_policy_id' + }, } }, timeout=3.05, verify=False ) - log_mock.assert_called_with('[%s] Invalid pagerduty incident urgency: "%s"', - 'pagerduty-incident', 'asdf') + log_warn_mock.assert_called_with( + '[%s] Assignee (%s) could not be found in PagerDuty', + self.SERVICE, + 'invalid_user' + ) + log_info_mock.assert_called_with('Successfully sent alert to %s:%s', + self.SERVICE, self.DESCRIPTOR) + + @patch('stream_alert.alert_processor.outputs.pagerduty.compose_alert') @patch('requests.put') @patch('requests.post') @patch('requests.get') - def test_dispatch_sends_correct_enqueue_event_request(self, get_mock, post_mock, put_mock): - """PagerDutyIncidentOutput - Dispatch Success, Good User, Sends Correct Event Request""" - # GET /users, /users - json_user = {'users': [{'id': 'valid_user_id'}]} - - # GET /incidents - json_lookup = {'incidents': [{'id': 'incident_id'}]} - - get_mock.return_value.status_code = 200 - get_mock.return_value.json.side_effect = [json_user, json_user, json_lookup] - - # POST /incidents, /v2/enqueue, /incidents/incident_id/notes - post_mock.return_value.status_code = 200 - json_incident = {'incident': {'id': 'incident_id'}} - json_event = {'dedup_key': 'returned_dedup_key'} - json_note = {'note': {'id': 'note_id'}} - post_mock.return_value.json.side_effect = [json_incident, json_event, json_note] + def test_dispatch_sends_correct_with_urgency(self, get_mock, post_mock, put_mock, + compose_alert): + """PagerDutyIncidentOutput - Dispatch Success, Good User, Sends Correct Urgency - # PUT /incidents/indicent_id/merge - put_mock.return_value.status_code = 200 + Ensure the PUT call respects a publisher urgency. + """ + RequestMocker.setup_mock(get_mock) + RequestMocker.setup_mock(post_mock) + RequestMocker.setup_mock(put_mock) - ctx = {'pagerduty-incident': {'assigned_user': 'valid_user'}} + compose_alert.return_value = { + '@pagerduty-incident.urgency': 'low' + } + ctx = {} self._dispatcher.dispatch(get_alert(context=ctx), self.OUTPUT) - # post_mock.assert_has_calls([call()]) - post_mock.assert_any_call( - 'https://events.pagerduty.com/v2/enqueue', - headers=None, + put_mock.assert_any_call( + 'https://api.pagerduty.com/incidents/incident_id', + headers={ + 'From': 'email@domain.com', + 'Content-Type': 'application/json', + 'Authorization': 'Token token=mocked_token', + 'Accept': 'application/vnd.pagerduty+json;version=2' + }, json={ - 'event_action': 'trigger', - 'client': 'StreamAlert', - 'client_url': None, - 'payload': { - 'custom_details': OrderedDict( - [ - ('description', 'Info about this rule and what actions to take'), - ('record', { - 'compressed_size': '9982', 'node_id': '1', 'cb_server': 'cbserver', - 'timestamp': '1496947381.18', - 'md5': '0F9AA55DA3BDE84B35656AD8911A22E1', - 'type': 'binarystore.file.added', - 'file_path': '/tmp/5DA/AD8/0F9AA55DA3BDE84B35656AD8911A22E1.zip', - 'size': '21504' - }) - ] - ), - 'source': 'carbonblack:binarystore.file.added', - 'severity': 'critical', - 'summary': 'StreamAlert Rule Triggered - cb_binarystore_file_added', - 'component': None, - 'group': None, - 'class': None, - }, - 'routing_key': 'mocked_key', - 'images': [], - 'links': [], + 'incident': { + 'body': { + 'type': 'incident_body', + 'details': 'Info about this rule and what actions to take' + }, + 'service': { + 'type': 'service_reference', + 'id': 'mocked_service_id' + }, + 'title': 'StreamAlert Incident - Rule triggered: cb_binarystore_file_added', + 'escalation_policy': { + 'type': 'escalation_policy_reference', + 'id': 'mocked_escalation_policy_id' + }, + 'type': 'incident', + + # This field should exist + 'urgency': 'low', + } }, - timeout=3.05, verify=True + timeout=3.05, verify=False ) + @patch('logging.Logger.warn') + @patch('stream_alert.alert_processor.outputs.pagerduty.compose_alert') @patch('requests.put') @patch('requests.post') @patch('requests.get') - def test_dispatch_sends_correct_merge_request(self, get_mock, post_mock, put_mock): - """PagerDutyIncidentOutput - Dispatch Success, Good User, Sends Correct Merge Request""" - # GET /users, /users - json_user = {'users': [{'id': 'valid_user_id'}]} - - # GET /incidents - json_lookup = {'incidents': [{'id': 'incident_id'}]} - - get_mock.return_value.status_code = 200 - get_mock.return_value.json.side_effect = [json_user, json_user, json_lookup] - - # POST /incidents, /v2/enqueue, /incidents/incident_id/notes - post_mock.return_value.status_code = 200 - json_incident = {'incident': {'id': 'incident_id'}} - json_event = {'dedup_key': 'returned_dedup_key'} - json_note = {'note': {'id': 'note_id'}} - post_mock.return_value.json.side_effect = [json_incident, json_event, json_note] + def test_dispatch_sends_correct_bad_urgency(self, get_mock, post_mock, put_mock, + compose_alert, log_mock): + """PagerDutyIncidentOutput - Dispatch Success, Good User, Omit Bad Urgency - # PUT /incidents/indicent_id/merge - put_mock.return_value.status_code = 200 + When an urgency is provided that is not valid, it should omit it entirely. + """ + RequestMocker.setup_mock(get_mock) + RequestMocker.setup_mock(post_mock) + RequestMocker.setup_mock(put_mock) - ctx = {'pagerduty-incident': {'assigned_user': 'valid_user'}} + compose_alert.return_value = { + '@pagerduty-incident.urgency': 'asdf' + } + ctx = {} self._dispatcher.dispatch(get_alert(context=ctx), self.OUTPUT) - put_mock.assert_called_with( - 'https://api.pagerduty.com/incidents/incident_id/merge', - headers={'From': 'email@domain.com', 'Content-Type': 'application/json', - 'Authorization': 'Token token=mocked_token', - 'Accept': 'application/vnd.pagerduty+json;version=2'}, - json={'source_incidents': [{'type': 'incident_reference', 'id': 'incident_id'}]}, - timeout=3.05, - verify=False - ) - - @patch('logging.Logger.info') - @patch('requests.put') - @patch('requests.post') - @patch('requests.get') - def test_dispatch_success_good_user(self, get_mock, post_mock, put_mock, log_mock): - """PagerDutyIncidentOutput - Dispatch Success, Good User""" - # GET /users, /users - json_user = {'users': [{'id': 'valid_user_id'}]} - - # GET /incidents - json_lookup = {'incidents': [{'id': 'incident_id'}]} - - get_mock.return_value.status_code = 200 - get_mock.return_value.json.side_effect = [json_user, json_user, json_lookup] - - # POST /incidents, /v2/enqueue, /incidents/incident_id/notes - post_mock.return_value.status_code = 200 - json_incident = {'incident': {'id': 'incident_id'}} - json_event = {'dedup_key': 'returned_dedup_key'} - json_note = {'note': {'id': 'note_id'}} - post_mock.return_value.json.side_effect = [json_incident, json_event, json_note] - - # PUT /incidents/indicent_id/merge - put_mock.return_value.status_code = 200 - - ctx = {'pagerduty-incident': {'assigned_user': 'valid_user'}} - - assert_true(self._dispatcher.dispatch(get_alert(context=ctx), self.OUTPUT)) + put_mock.assert_any_call( + 'https://api.pagerduty.com/incidents/incident_id', + headers={ + 'From': 'email@domain.com', + 'Content-Type': 'application/json', + 'Authorization': 'Token token=mocked_token', + 'Accept': 'application/vnd.pagerduty+json;version=2' + }, + json={ + 'incident': { + 'body': { + 'type': 'incident_body', + 'details': 'Info about this rule and what actions to take' + }, + 'service': { + 'type': 'service_reference', + 'id': 'mocked_service_id' + }, + 'title': 'StreamAlert Incident - Rule triggered: cb_binarystore_file_added', + 'escalation_policy': { + 'type': 'escalation_policy_reference', + 'id': 'mocked_escalation_policy_id' + }, + 'type': 'incident', - log_mock.assert_called_with('Successfully sent alert to %s:%s', - self.SERVICE, self.DESCRIPTOR) + # urgency is omitted here because the original urgency was invalid + } + }, + timeout=3.05, verify=False + ) + log_mock.assert_called_with('[%s] Invalid pagerduty incident urgency: "%s"', + 'pagerduty-incident', 'asdf') @patch('logging.Logger.info') @patch('requests.put') @patch('requests.post') @patch('requests.get') def test_dispatch_success_good_policy(self, get_mock, post_mock, put_mock, log_mock): - """PagerDutyIncidentOutput - Dispatch Success, Good Policy""" - # GET /users - json_user = {'users': [{'id': 'user_id'}]} + """PagerDutyIncidentOutput - Dispatch Success, Good Policy - # GET /incidents - json_lookup = {'incidents': [{'id': 'incident_id'}]} - - get_mock.return_value.status_code = 200 - get_mock.return_value.json.side_effect = [json_user, json_lookup] - - # POST /incidents, /v2/enqueue, /incidents/incident_id/notes - post_mock.return_value.status_code = 200 - json_incident = {'incident': {'id': 'incident_id'}} - json_event = {'dedup_key': 'returned_dedup_key'} - json_note = {'note': {'id': 'note_id'}} - post_mock.return_value.json.side_effect = [json_incident, json_event, json_note] - - # PUT /incidents/{incident_id}/merge - put_mock.return_value.status_code = 200 - - ctx = {'pagerduty-incident': {'assigned_policy_id': 'valid_policy_id'}} + Ensures that we respect a custom escalation policy + """ + RequestMocker.setup_mock(get_mock) + RequestMocker.setup_mock(post_mock) + RequestMocker.setup_mock(put_mock) + ctx = { + 'pagerduty-incident': { + 'assigned_policy_id': 'valid_policy_id' + } + } assert_true(self._dispatcher.dispatch(get_alert(context=ctx), self.OUTPUT)) + put_mock.assert_any_call( + 'https://api.pagerduty.com/incidents/incident_id', + headers={ + 'From': 'email@domain.com', + 'Content-Type': 'application/json', + 'Authorization': 'Token token=mocked_token', + 'Accept': 'application/vnd.pagerduty+json;version=2' + }, + json={ + 'incident': { + 'body': { + 'type': 'incident_body', + 'details': 'Info about this rule and what actions to take' + }, + 'service': { + 'type': 'service_reference', + 'id': 'mocked_service_id' + }, + 'title': 'StreamAlert Incident - Rule triggered: cb_binarystore_file_added', + 'escalation_policy': { + 'type': 'escalation_policy_reference', + 'id': 'valid_policy_id', # Policy is sent here + }, + 'type': 'incident', + } + }, + timeout=3.05, verify=False + ) + log_mock.assert_called_with('Successfully sent alert to %s:%s', self.SERVICE, self.DESCRIPTOR) @@ -630,57 +696,56 @@ def test_dispatch_success_good_policy(self, get_mock, post_mock, put_mock, log_m @patch('requests.post') @patch('requests.get') def test_dispatch_success_with_priority(self, get_mock, post_mock, put_mock, log_mock): - """PagerDutyIncidentOutput - Dispatch Success With Priority""" - # GET /priorities, /users - json_user = {'users': [{'id': 'user_id'}]} - json_priority = {'priorities': [{'id': 'priority_id', 'name': 'priority_name'}]} - json_lookup = {'incidents': [{'id': 'incident_id'}]} - - def setup_post_mock(mock, json_incident, json_event, json_note): - def post(*args, **_): - url = args[0] - if url == 'https://api.pagerduty.com/incidents': - response = json_incident - elif url == 'https://events.pagerduty.com/v2/enqueue': - response = json_event - elif ( - url.startswith('https://api.pagerduty.com/incidents/') and - url.endswith('/notes') - ): - response = json_note - else: - raise RuntimeError('Misconfigured mock: {}'.format(url)) - - _mock = MagicMock() - _mock.status_code = 200 - _mock.json.return_value = response - return _mock - - mock.side_effect = post - - get_mock.return_value.status_code = 200 - get_mock.return_value.json.side_effect = [json_user, json_priority, json_lookup] - - # POST /incidents, /v2/enqueue, /incidents/{incident_id}/notes - json_incident = {'incident': {'id': 'incident_id'}} - json_event = {'dedup_key': 'returned_dedup_key'} - json_note = {'note': {'id': 'note_id'}} - # post_mock.return_value.status_code = 200 - # post_mock.return_value.json.side_effect = [json_incident, json_event, json_note] - setup_post_mock(post_mock, json_incident, json_event, json_note) + """PagerDutyIncidentOutput - Dispatch Success With Priority - # PUT /incidents/{incident_id}/merge - put_mock.return_value.status_code = 200 + Ensure the PUT call respects a priority, if a valid one is given. + """ + RequestMocker.setup_mock(get_mock) + RequestMocker.setup_mock(post_mock) + RequestMocker.setup_mock(put_mock) ctx = { 'pagerduty-incident': { - 'assigned_policy_id': 'valid_policy_id', - 'incident_priority': 'priority_name' + 'incident_priority': 'priority_name', } } - assert_true(self._dispatcher.dispatch(get_alert(context=ctx), self.OUTPUT)) + put_mock.assert_any_call( + 'https://api.pagerduty.com/incidents/incident_id', + headers={ + 'From': 'email@domain.com', + 'Content-Type': 'application/json', + 'Authorization': 'Token token=mocked_token', + 'Accept': 'application/vnd.pagerduty+json;version=2' + }, + json={ + 'incident': { + 'body': { + 'type': 'incident_body', + 'details': 'Info about this rule and what actions to take' + }, + 'service': { + 'type': 'service_reference', + 'id': 'mocked_service_id' + }, + 'title': 'StreamAlert Incident - Rule triggered: cb_binarystore_file_added', + 'escalation_policy': { + 'type': 'escalation_policy_reference', + 'id': 'mocked_escalation_policy_id', + }, + 'type': 'incident', + + # This priority node should show up + 'priority': { + 'type': 'priority_reference', + 'id': 'priority_id' + } + } + }, + timeout=3.05, verify=False + ) + log_mock.assert_called_with('Successfully sent alert to %s:%s', self.SERVICE, self.DESCRIPTOR) @@ -690,53 +755,17 @@ def post(*args, **_): @patch('requests.get') def test_dispatch_success_with_note(self, get_mock, post_mock, put_mock, log_mock): """PagerDutyIncidentOutput - Dispatch Success With Note""" - # GET /priorities, /users - json_user = {'users': [{'id': 'user_id'}]} - json_lookup = {'incidents': [{'id': 'incident_id'}]} - - def setup_post_mock(mock, json_incident, json_event, json_note): - def post(*args, **_): - url = args[0] - if url == 'https://api.pagerduty.com/incidents': - response = json_incident - elif url == 'https://events.pagerduty.com/v2/enqueue': - response = json_event - elif ( - url.startswith('https://api.pagerduty.com/incidents/') and - url.endswith('/notes') - ): - response = json_note - else: - raise RuntimeError('Misconfigured mock: {}'.format(url)) - - _mock = MagicMock() - _mock.status_code = 200 - _mock.json.return_value = response - return _mock - - mock.side_effect = post - - get_mock.return_value.status_code = 200 - get_mock.return_value.json.side_effect = [json_user, json_lookup] - - # POST /incidents, /v2/enqueue, /incidents/{incident_id}/notes - json_incident = {'incident': {'id': 'incident_id'}} - json_event = {'dedup_key': 'returned_dedup_key'} - json_note = {'note': {'id': 'note_id'}} - setup_post_mock(post_mock, json_incident, json_event, json_note) - - # PUT /incidents/{incident_id}/merge - put_mock.return_value.status_code = 200 + RequestMocker.setup_mock(get_mock) + RequestMocker.setup_mock(post_mock) + RequestMocker.setup_mock(put_mock) ctx = { 'pagerduty-incident': { 'note': 'This is just a note' } } - assert_true(self._dispatcher.dispatch(get_alert(context=ctx), self.OUTPUT)) - # post_mock.assert_has_calls([call()]) post_mock.assert_any_call( 'https://api.pagerduty.com/incidents/incident_id/notes', headers={'From': 'email@domain.com', @@ -756,84 +785,21 @@ def post(*args, **_): @patch('requests.get') def test_dispatch_success_none_note(self, get_mock, post_mock, put_mock, log_mock): """PagerDutyIncidentOutput - Dispatch Success With No Note""" - # GET /priorities, /users - json_user = {'users': [{'id': 'user_id'}]} - json_lookup = {'incidents': [{'id': 'incident_id'}]} - - def setup_post_mock(mock, json_incident, json_event): - def post(*args, **_): - url = args[0] - if url == 'https://api.pagerduty.com/incidents': - response = json_incident - elif url == 'https://events.pagerduty.com/v2/enqueue': - response = json_event - elif ( - url.startswith('https://api.pagerduty.com/incidents/') and - url.endswith('/notes') - ): - # assert the /notes endpoint is never called - raise RuntimeError('This endpoint is not intended to be called') - else: - raise RuntimeError('Misconfigured mock: {}'.format(url)) - - _mock = MagicMock() - _mock.status_code = 200 - _mock.json.return_value = response - return _mock - - mock.side_effect = post - - get_mock.return_value.status_code = 200 - get_mock.return_value.json.side_effect = [json_user, json_lookup] - - # POST /incidents, /v2/enqueue, /incidents/{incident_id}/notes - json_incident = {'incident': {'id': 'incident_id'}} - json_event = {'dedup_key': 'returned_dedup_key'} - setup_post_mock(post_mock, json_incident, json_event) - - # PUT /incidents/{incident_id}/merge - put_mock.return_value.status_code = 200 + RequestMocker.setup_mock(get_mock) + RequestMocker.setup_mock(post_mock) + RequestMocker.setup_mock(put_mock) ctx = { 'pagerduty-incident': { 'note': None } } - assert_true(self._dispatcher.dispatch(get_alert(context=ctx), self.OUTPUT)) - log_mock.assert_called_with('Successfully sent alert to %s:%s', - self.SERVICE, self.DESCRIPTOR) - - @patch('logging.Logger.info') - @patch('requests.put') - @patch('requests.post') - @patch('requests.get') - def test_dispatch_success_bad_user(self, get_mock, post_mock, put_mock, log_mock): - """PagerDutyIncidentOutput - Dispatch Success, Bad User""" - # GET /users, /users - json_user = {'users': [{'id': 'user_id'}]} - json_not_user = {'not_users': [{'id': 'user_id'}]} - - # GET /incidents - json_lookup = {'incidents': [{'id': 'incident_id'}]} + def notes_api_call(*args, **_): + return args[0].endswith('/notes') - get_mock.return_value.status_code = 200 - get_mock.return_value.json.side_effect = [json_user, json_not_user, json_lookup] - - # POST /incidents, /v2/enqueue, /incidents/incident_id/notes - post_mock.return_value.status_code = 200 - json_incident = {'incident': {'id': 'incident_id'}} - json_event = {'dedup_key': 'returned_dedup_key'} - json_note = {'note': {'id': 'note_id'}} - post_mock.return_value.json.side_effect = [json_incident, json_event, json_note] - - # PUT /incidents/indicent_id/merge - put_mock.return_value.status_code = 200 - - ctx = {'pagerduty-incident': {'assigned_user': 'invalid_user'}} - - assert_true(self._dispatcher.dispatch(get_alert(context=ctx), self.OUTPUT)) + RequestMocker.assert_mock_with_no_calls_like(post_mock, notes_api_call) log_mock.assert_called_with('Successfully sent alert to %s:%s', self.SERVICE, self.DESCRIPTOR) @@ -844,24 +810,9 @@ def test_dispatch_success_bad_user(self, get_mock, post_mock, put_mock, log_mock @patch('requests.get') def test_dispatch_success_no_context(self, get_mock, post_mock, put_mock, log_mock): """PagerDutyIncidentOutput - Dispatch Success, No Context""" - # GET /users - json_user = {'users': [{'id': 'user_id'}]} - - # GET /incidents - json_lookup = {'incidents': [{'id': 'incident_id'}]} - - get_mock.return_value.status_code = 200 - get_mock.return_value.json.side_effect = [json_user, json_lookup] - - # POST /incidents, /v2/enqueue, /incidents/incident_id/notes - post_mock.return_value.status_code = 200 - json_incident = {'incident': {'id': 'incident_id'}} - json_event = {'dedup_key': 'returned_dedup_key'} - json_note = {'note': {'id': 'note_id'}} - post_mock.return_value.json.side_effect = [json_incident, json_event, json_note] - - # PUT /incidents/indicent_id/merge - put_mock.return_value.status_code = 200 + RequestMocker.setup_mock(get_mock) + RequestMocker.setup_mock(post_mock) + RequestMocker.setup_mock(put_mock) assert_true(self._dispatcher.dispatch(get_alert(), self.OUTPUT)) @@ -871,147 +822,291 @@ def test_dispatch_success_no_context(self, get_mock, post_mock, put_mock, log_mo @patch('logging.Logger.error') @patch('requests.post') @patch('requests.get') - def test_dispatch_failure_bad_everything(self, get_mock, post_mock, log_mock): - """PagerDutyIncidentOutput - Dispatch Failure: No User""" - # GET /users, /users - type(get_mock.return_value).status_code = PropertyMock(side_effect=[200, 400]) - - # Only set the return_value here since there will only be one successful call - # that makes it to the point of calling the .json() method - get_mock.return_value.json.return_value = {'users': [{'id': 'user_id'}]} - - # POST /incidents - post_mock.return_value.status_code = 400 + def test_dispatch_failure_bad_from_user(self, get_mock, post_mock, log_mock): + """PagerDutyIncidentOutput - Dispatch Failure: No User + + This fixtures the behavior if the "from_email" configuration param is invalid. + This causes significant problems on the PagerDuty API so we want to error out early. + """ + RequestMocker.setup_mock(post_mock) + RequestMocker.setup_mock( + get_mock, + [ + ['/users', 200, {'users': []}], + ['/incidents', 200, RequestMocker.INCIDENTS_JSON], + ] + ) assert_false(self._dispatcher.dispatch(get_alert(), self.OUTPUT)) log_mock.assert_called_with('Failed to send alert to %s:%s', self.SERVICE, self.DESCRIPTOR) - @patch('logging.Logger.info') - @patch('requests.put') + @patch('logging.Logger.error') @patch('requests.post') @patch('requests.get') - def test_dispatch_success_no_merge_response(self, get_mock, post_mock, put_mock, log_mock): - """PagerDutyIncidentOutput - Dispatch Success, No Merge Response""" - # GET /users - get_mock.return_value.status_code = 200 - json_user = {'users': [{'id': 'user_id'}]} - json_lookup = {'incidents': [{'id': 'existing_incident_id'}]} - get_mock.return_value.json.side_effect = [json_user, json_lookup] - - # POST /incidents, /v2/enqueue - post_mock.return_value.status_code = 200 - json_incident = {'incident': {'id': 'incident_id'}} - json_event = {'dedup_key': 'returned_dedup_key'} - json_note = {'note': {'aa'}} - post_mock.return_value.json.side_effect = [json_incident, json_event, json_note] - - # PUT /incidents/{incident_id}/merge - put_mock.return_value.status_code = 200 - put_mock.return_value.json.return_value = {} - - ctx = {'pagerduty-incident': {'assigned_policy_id': 'valid_policy_id'}} + def test_dispatch_no_dispatch_no_event_response(self, get_mock, post_mock, log_mock): + """PagerDutyIncidentOutput - Dispatch Failure, Event Enqueue No Response + + Tests behavior if the /enqueue API call fails. + """ + RequestMocker.setup_mock(get_mock) + RequestMocker.setup_mock( + post_mock, + [ + ['/enqueue', 400, {}], + ] + ) - assert_true(self._dispatcher.dispatch(get_alert(context=ctx), self.OUTPUT)) + assert_false(self._dispatcher.dispatch(get_alert(), self.OUTPUT)) - log_mock.assert_called_with('Successfully sent alert to %s:%s', - self.SERVICE, self.DESCRIPTOR) + log_mock.assert_called_with('Failed to send alert to %s:%s', self.SERVICE, self.DESCRIPTOR) @patch('logging.Logger.error') + @patch('requests.put') @patch('requests.post') @patch('requests.get') - def test_dispatch_no_dispatch_no_incident_response(self, get_mock, post_mock, log_mock): - """PagerDutyIncidentOutput - Dispatch Failure, No Incident Response""" - # /users - get_mock.return_value.status_code = 200 - json_user = {'users': [{'id': 'user_id'}]} - get_mock.return_value.json.return_value = json_user - - # /incidents - post_mock.return_value.status_code = 200 - post_mock.return_value.json.return_value = {} + def test_dispatch_no_dispatch_no_incident_response(self, get_mock, post_mock, put_mock, + log_mock): + """PagerDutyIncidentOutput - Dispatch Failure, No Incident Response + + Tests behavior if the PUT /incidents/# API call fails. + """ + RequestMocker.setup_mock(get_mock) + RequestMocker.setup_mock(post_mock) + RequestMocker.setup_mock( + put_mock, + [ + [re.compile(r'^.*/incidents/[a-zA-Z0-9-_]+$'), 400, {}], + ] + ) assert_false(self._dispatcher.dispatch(get_alert(), self.OUTPUT)) + log_mock.assert_any_call('[%s] Failed to update container incident for event', self.SERVICE) log_mock.assert_called_with('Failed to send alert to %s:%s', self.SERVICE, self.DESCRIPTOR) @patch('logging.Logger.error') + @patch('requests.put') @patch('requests.post') @patch('requests.get') - def test_dispatch_no_dispatch_no_incident_event(self, get_mock, post_mock, log_mock): - """PagerDutyIncidentOutput - Dispatch Failure, No Incident Event""" - # /users - get_mock.return_value.status_code = 200 - json_user = {'users': [{'id': 'user_id'}]} - get_mock.return_value.json.return_value = json_user - - # /incidents, /v2/enqueue - post_mock.return_value.status_code = 200 - json_incident = {'incident': {'id': 'incident_id'}} - json_event = {} - post_mock.return_value.json.side_effect = [json_incident, json_event] + def test_dispatch_no_dispatch_no_incident_id_in_response(self, get_mock, post_mock, put_mock, + log_mock): + """PagerDutyIncidentOutput - Dispatch Failure, No Incident Id in Response + + This is somewhat of a specific weird case when the response structure is not what we + expect and is missing the id. + """ + RequestMocker.setup_mock(get_mock) + RequestMocker.setup_mock(post_mock) + RequestMocker.setup_mock( + put_mock, + [ + [re.compile(r'^.*/incidents/[a-zA-Z0-9-_]+$'), 200, {'incident': {'not_id': '?'}}], + ] + ) assert_false(self._dispatcher.dispatch(get_alert(), self.OUTPUT)) - log_mock.assert_called_with('Failed to send alert to %s:%s', self.SERVICE, self.DESCRIPTOR) + RequestMocker.inspect_calls(log_mock) + log_mock.assert_any_call('[%s] Incident is missing "id"??', self.SERVICE) @patch('logging.Logger.error') + def test_dispatch_bad_descriptor(self, log_mock): + """PagerDutyIncidentOutput - Dispatch Failure, Bad Descriptor""" + assert_false( + self._dispatcher.dispatch(get_alert(), ':'.join([self.SERVICE, 'bad_descriptor']))) + + log_mock.assert_called_with('Failed to send alert to %s:%s', self.SERVICE, 'bad_descriptor') + + @patch('logging.Logger.info') + @patch('requests.put') @patch('requests.post') @patch('requests.get') - def test_dispatch_no_dispatch_no_incident_key(self, get_mock, post_mock, log_mock): - """PagerDutyIncidentOutput - Dispatch Failure, No Incident Key""" - # /users - get_mock.return_value.status_code = 200 - json_user = {'users': [{'id': 'user_id'}]} - get_mock.return_value.json.return_value = json_user + def test_dispatch_request_responder(self, get_mock, post_mock, put_mock, log_mock): + """PagerDutyIncidentOutput - Dispatch Success, Request Responder - # /incidents, /v2/enqueue - post_mock.return_value.status_code = 200 - json_incident = {'incident': {'id': 'incident_id'}} - json_event = {'not_dedup_key': 'returned_dedup_key'} - post_mock.return_value.json.side_effect = [json_incident, json_event] + Ensures the correct calls are made to Request Responders. + """ + RequestMocker.setup_mock(get_mock) + RequestMocker.setup_mock(post_mock) + RequestMocker.setup_mock(put_mock) - assert_false(self._dispatcher.dispatch(get_alert(), self.OUTPUT)) + ctx = { + 'pagerduty-incident': { + 'responders': ['responder1@airbnb.com'], + 'responder_message': 'I am tea kettle short and stout', + } + } + assert_true(self._dispatcher.dispatch(get_alert(context=ctx), self.OUTPUT)) - log_mock.assert_called_with('Failed to send alert to %s:%s', self.SERVICE, self.DESCRIPTOR) + get_mock.assert_any_call( + 'https://api.pagerduty.com/users', + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'Token token=mocked_token', + 'Accept': 'application/vnd.pagerduty+json;version=2' + }, + params={'query': 'responder1@airbnb.com'}, + timeout=3.05, verify=False + ) - @patch('logging.Logger.error') + post_mock.assert_any_call( + 'https://api.pagerduty.com/incidents/incident_id/responder_requests', + headers={ + 'From': 'email@domain.com', + 'Content-Type': 'application/json', + 'Authorization': 'Token token=mocked_token', + 'Accept': 'application/vnd.pagerduty+json;version=2' + }, + json={ + 'requester_id': 'valid_user_id', + 'message': 'I am tea kettle short and stout', + 'responder_request_targets': [ + { + 'responder_request_target': { + 'type': 'user_reference', + 'id': 'valid_user_id' + } + } + ] + }, + timeout=3.05, + verify=False + ) + + log_mock.assert_called_with( + 'Successfully sent alert to %s:%s', self.SERVICE, self.DESCRIPTOR + ) + + @patch('logging.Logger.info') + @patch('requests.put') @patch('requests.post') @patch('requests.get') - def test_dispatch_bad_dispatch(self, get_mock, post_mock, log_mock): - """PagerDutyIncidentOutput - Dispatch Failure, Bad Request""" - # /users - get_mock.return_value.status_code = 200 - json_user = {'users': [{'id': 'user_id'}]} - get_mock.return_value.json.return_value = json_user + def test_dispatch_request_responder_single(self, get_mock, post_mock, put_mock, log_mock): + """PagerDutyIncidentOutput - Dispatch Success, Support single responder and default message - # /incidents - post_mock.return_value.status_code = 400 + Ensure support for omitting the message and using string syntax for responder. + """ + RequestMocker.setup_mock(get_mock) + RequestMocker.setup_mock(post_mock) + RequestMocker.setup_mock(put_mock) - assert_false(self._dispatcher.dispatch(get_alert(), self.OUTPUT)) + ctx = { + 'pagerduty-incident': { + 'responders': 'responder1@airbnb.com', + } + } + assert_true(self._dispatcher.dispatch(get_alert(context=ctx), self.OUTPUT)) - log_mock.assert_called_with('Failed to send alert to %s:%s', self.SERVICE, self.DESCRIPTOR) + get_mock.assert_any_call( + 'https://api.pagerduty.com/users', + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'Token token=mocked_token', + 'Accept': 'application/vnd.pagerduty+json;version=2' + }, + params={'query': 'responder1@airbnb.com'}, + timeout=3.05, verify=False + ) + + post_mock.assert_any_call( + 'https://api.pagerduty.com/incidents/incident_id/responder_requests', + headers={ + 'From': 'email@domain.com', + 'Content-Type': 'application/json', + 'Authorization': 'Token token=mocked_token', + 'Accept': 'application/vnd.pagerduty+json;version=2' + }, + json={ + 'requester_id': 'valid_user_id', + 'message': 'An incident was reported that requires your attention.', + 'responder_request_targets': [ + { + 'responder_request_target': { + 'type': 'user_reference', + 'id': 'valid_user_id' + } + } + ] + }, + timeout=3.05, + verify=False + ) + + log_mock.assert_called_with( + 'Successfully sent alert to %s:%s', self.SERVICE, self.DESCRIPTOR + ) @patch('logging.Logger.error') + @patch('logging.Logger.info') + @patch('requests.put') + @patch('requests.post') @patch('requests.get') - def test_dispatch_bad_email(self, get_mock, log_mock): - """PagerDutyIncidentOutput - Dispatch Failure, Bad Email""" - # /users - get_mock.return_value.status_code = 400 - json_user = {'not_users': [{'id': 'no_user_id'}]} - get_mock.return_value.json.return_value = json_user + def test_dispatch_request_responder_not_valid(self, get_mock, post_mock, put_mock, + log_mock, log_error_mock): + """PagerDutyIncidentOutput - Dispatch Success, Responder not valid + + When a responer is requested but the email is not associated with a PD user, it will + omit the request, log an error, add an instability note, but the overall request will + still succeed. + """ + def invalid_user_matcher(*args, **kwargs): + if args[0] != 'https://api.pagerduty.com/users': + return False + + query = kwargs.get('params', {}).get('query', None) + return query == 'invalid_responder@airbnb.com' + + RequestMocker.setup_mock(post_mock) + RequestMocker.setup_mock(put_mock) + RequestMocker.setup_mock( + get_mock, + [ + [invalid_user_matcher, 200, {'users': []}], + ['/users', 200, RequestMocker.USERS_JSON], + ['/incidents', 200, RequestMocker.INCIDENTS_JSON], + ] + ) - assert_false(self._dispatcher.dispatch(get_alert(), self.OUTPUT)) + ctx = { + 'pagerduty-incident': { + 'responders': ['invalid_responder@airbnb.com'], + } + } + assert_true(self._dispatcher.dispatch(get_alert(context=ctx), self.OUTPUT)) - log_mock.assert_called_with('Failed to send alert to %s:%s', self.SERVICE, self.DESCRIPTOR) + def responder_request_calls(*args, **_): + return args[0].endswith('/responder_requests') - @patch('logging.Logger.error') - def test_dispatch_bad_descriptor(self, log_mock): - """PagerDutyIncidentOutput - Dispatch Failure, Bad Descriptor""" - assert_false( - self._dispatcher.dispatch(get_alert(), ':'.join([self.SERVICE, 'bad_descriptor']))) + RequestMocker.assert_mock_with_no_calls_like(post_mock, responder_request_calls) + log_error_mock.assert_called_with( + ( + '[pagerduty-incident] Failed to request a responder (invalid_responder@airbnb.com)' + ' on incident (incident_id)' + ) + ) - log_mock.assert_called_with('Failed to send alert to %s:%s', self.SERVICE, 'bad_descriptor') + expected_note = ( + 'StreamAlert failed to correctly setup this incident. Please contact your ' + 'StreamAlert administrator.\n\nErrors:\n- [pagerduty-incident] Failed to request ' + 'a responder (invalid_responder@airbnb.com) on incident (incident_id)' + ) + post_mock.assert_any_call( + 'https://api.pagerduty.com/incidents/incident_id/notes', + headers={'From': 'email@domain.com', + 'Content-Type': 'application/json', + 'Authorization': 'Token token=mocked_token', + 'Accept': 'application/vnd.pagerduty+json;version=2'}, + json={'note': {'content': expected_note}}, + timeout=3.05, verify=False + ) + + log_mock.assert_called_with('Successfully sent alert to %s:%s', + self.SERVICE, self.DESCRIPTOR) + + log_mock.assert_called_with( + 'Successfully sent alert to %s:%s', self.SERVICE, self.DESCRIPTOR + ) @patch('stream_alert.alert_processor.outputs.output_base.OutputDispatcher.MAX_RETRY_ATTEMPTS', 1) @@ -1050,7 +1145,7 @@ def test_get_standardized_priority_sends_correct_reuqest(self, get_mock): get_mock.return_value.status_code = 200 context = {'incident_priority': priority_name} - self._work.get_standardized_priority(context) + self._work._get_standardized_priority(context) get_mock.assert_called_with( 'https://api.pagerduty.com/priorities', @@ -1077,7 +1172,7 @@ def test_get_standardized_priority_success(self, get_mock): context = {'incident_priority': priority_name} - priority_verified = self._work.get_standardized_priority(context) + priority_verified = self._work._get_standardized_priority(context) assert_equal(priority_verified['id'], 'verified_priority_id') assert_equal(priority_verified['type'], 'priority_reference') @@ -1089,8 +1184,8 @@ def test_get_standardized_priority_fail(self, get_mock): context = {'incident_priority': 'priority_name'} - priority_not_verified = self._work.get_standardized_priority(context) - assert_equal(priority_not_verified, dict()) + priority_not_verified = self._work._get_standardized_priority(context) + assert_false(priority_not_verified) @patch('requests.get') def test_get_standardized_priority_empty(self, get_mock): @@ -1102,8 +1197,8 @@ def test_get_standardized_priority_empty(self, get_mock): context = {'incident_priority': 'priority_name'} - priority_not_verified = self._work.get_standardized_priority(context) - assert_equal(priority_not_verified, dict()) + priority_not_verified = self._work._get_standardized_priority(context) + assert_false(priority_not_verified) @patch('requests.get') def test_get_standardized_priority_not_found(self, get_mock): @@ -1115,8 +1210,8 @@ def test_get_standardized_priority_not_found(self, get_mock): context = {'incident_priority': 'priority_name'} - priority_not_verified = self._work.get_standardized_priority(context) - assert_equal(priority_not_verified, dict()) + priority_not_verified = self._work._get_standardized_priority(context) + assert_false(priority_not_verified) @patch('requests.get') def test_get_standardized_priority_invalid(self, get_mock): @@ -1128,8 +1223,8 @@ def test_get_standardized_priority_invalid(self, get_mock): context = {'incident_priority': 'priority_name'} - priority_not_verified = self._work.get_standardized_priority(context) - assert_equal(priority_not_verified, dict()) + priority_not_verified = self._work._get_standardized_priority(context) + assert_false(priority_not_verified) @patch('requests.get') def test_get_incident_assignment_user_sends_correct_rquest(self, get_mock): @@ -1137,7 +1232,7 @@ def test_get_incident_assignment_user_sends_correct_rquest(self, get_mock): context = {'assigned_user': 'user_to_assign'} get_mock.return_value.status_code = 400 - self._work.get_incident_assignment(context) + self._work._get_incident_assignments(context) get_mock.assert_called_with( 'https://api.pagerduty.com/users', @@ -1160,19 +1255,17 @@ def test_get_incident_assignment_user(self, get_mock): json_user = {'users': [{'id': 'verified_user_id'}]} get_mock.return_value.json.return_value = json_user - assigned_key, assigned_value = self._work.get_incident_assignment(context) + assigned_value = self._work._get_incident_assignments(context) - assert_equal(assigned_key, 'assignments') assert_equal(assigned_value[0]['assignee']['id'], 'verified_user_id') assert_equal(assigned_value[0]['assignee']['type'], 'user_reference') - def test_get_incident_assignment_policy_no_default(self): + def test_get_incident_escalation_policy_no_default(self): """PagerDutyIncidentOutput - Incident Assignment Policy (No Default)""" context = {'assigned_policy_id': 'policy_id_to_assign'} - assigned_key, assigned_value = self._work.get_incident_assignment(context) + assigned_value = self._work._get_incident_escalation_policy(context) - assert_equal(assigned_key, 'escalation_policy') assert_equal(assigned_value['id'], 'policy_id_to_assign') assert_equal(assigned_value['type'], 'escalation_policy_reference') @@ -1183,7 +1276,7 @@ def test_user_verify_success(self, get_mock): json_check = {'users': [{'id': 'verified_user_id'}]} get_mock.return_value.json.return_value = json_check - user_verified = self._work.verify_user_exists() + user_verified = self._work._verify_user_exists() assert_true(user_verified) @patch('requests.get') @@ -1193,7 +1286,7 @@ def test_user_verify_fail(self, get_mock): json_check = {'not_users': [{'not_id': 'verified_user_id'}]} get_mock.return_value.json.return_value = json_check - user_verified = self._work.verify_user_exists() + user_verified = self._work._verify_user_exists() assert_false(user_verified) @@ -1445,7 +1538,7 @@ def setup(self): } ) work.verify_user_exists = MagicMock(return_value=True) - work._create_base_incident = MagicMock(return_value=incident) + work._update_base_incident = MagicMock(return_value=incident) work._create_base_alert_event = MagicMock(return_value=event) work._merge_event_into_incident = MagicMock(return_value=merged_incident) work._add_incident_note = MagicMock(return_value=note) @@ -1468,25 +1561,22 @@ def test_positive_case(self, compose_alert_mock, log_error): @patch('logging.Logger.error') @patch('stream_alert.alert_processor.outputs.pagerduty.compose_alert') - def test_unstable_merge_fail(self, compose_alert_mock, log_error): - """PagerDuty WorkContext - Unstable - Merge Failed""" + def test_unstable_note_fail(self, compose_alert_mock, log_error): + """PagerDuty WorkContext - Unstable - Add Note Failed""" publication = {} compose_alert_mock.return_value = publication - self._work._merge_event_into_incident = MagicMock(return_value=False) + self._work._add_incident_note = MagicMock(return_value=False) alert = get_alert() result = self._work.run(alert, 'descriptor') assert_true(result) - log_error.assert_called_with( - '[%s] Failed to merge alert [%s] into [%s]', 'test', '000000ppppdpdpdpdpd', 'ABCDEFGH' - ) + log_error.assert_called_with(StringThatStartsWith("[test] Failed to add note to incident")) - @patch('logging.Logger.error') @patch('stream_alert.alert_processor.outputs.pagerduty.compose_alert') - def test_unstable_note_fail(self, compose_alert_mock, log_error): - """PagerDuty WorkContext - Unstable - Add Node Failed""" + def test_unstable_adds_instability_note(self, compose_alert_mock): + """PagerDuty WorkContext - Unstable - Add Instability Note""" publication = {} compose_alert_mock.return_value = publication @@ -1496,20 +1586,113 @@ def test_unstable_note_fail(self, compose_alert_mock, log_error): result = self._work.run(alert, 'descriptor') assert_true(result) - log_error.assert_called_with( - '[%s] Failed to add note to incident (%s)', 'test', 'ABCDEFGH' + self._work._add_instability_note.assert_called_with( + 'ABCDEFGH', ['[test] Failed to add note to incident (ABCDEFGH)'] ) - @patch('stream_alert.alert_processor.outputs.pagerduty.compose_alert') - def test_unstable_adds_instability_note(self, compose_alert_mock): - """PagerDuty WorkContext - Unstable - Add Instability Note""" - publication = {} - compose_alert_mock.return_value = publication - self._work._add_incident_note = MagicMock(return_value=False) +class StringThatStartsWith(str): + def __eq__(self, other): + return other.startswith(self) + + +class RequestMocker(object): + CREATE_EVENT_JSON = {'something': '?'} + USERS_JSON = {'users': [{'id': 'valid_user_id'}]} + INCIDENTS_JSON = {'incidents': [{'id': 'incident_id'}]} + INCIDENT_JSON = {'incident': {'id': 'incident_id'}} + PRIORITIES_JSON = {'priorities': [{'id': 'priority_id', 'name': 'priority_name'}]} + EVENT_JSON = {'dedup_key': 'returned_dedup_key'} + NOTE_JSON = {'note': {'id': 'note_id'}} + RESPONDER_JSON = {'responder_request': { + 'incident': {'id': 'incident_id'}, + 'requester': {'id': 'responder_user_id'}, + 'responder_request_targets': [] + }} + + @staticmethod + def inspect_calls(mock): + """Prints out all of the calls to this mock, in the order of call. + """ + print mock.call_args_list + + @staticmethod + def assert_mock_with_no_calls_like(mock, condition): + calls = mock.call_args_list + failed = [] + + for index, _call in enumerate(calls, start=1): + args, kwargs = _call + if condition(*args, **kwargs): + failed.append(index) - alert = get_alert() - result = self._work.run(alert, 'descriptor') - assert_true(result) + assert_false( + failed, + ( + 'Failed to assert that mock was not called.\nOut of {} calls, ' + 'calls {} failed the condition.' + ).format(len(calls), ', '.join(['#{}'.format(idx) for idx in failed])) + ) + + @classmethod + def setup_mock(cls, get_mock, conditions=None): + """Sets up a magic mock to return values based upon the conditions provided + + The "conditions" arg is an array of structures. + Each structure is another array of exactly 3 elements: - self._work._add_instability_note.assert_called_with('ABCDEFGH') + 1) The first element can be one of three types: + function + regex + string + + 2) The second element is an integer, the HTTP response code + + 3) The third element is a JSON dict, the response of the HTTP request + + When the magic mock is called, it will iterate through each condition to find the first + condition that matches the given input arguments. It then sets up a mock response object + and returns it. If all conditions are false, it mocks a 404 response. + + When conditions are omitted, it assumes some defaults that return positive cases. + """ + + if conditions is None: + conditions = [ + ['/create_event.json', 200, cls.CREATE_EVENT_JSON], + ['/users', 200, cls.USERS_JSON], + ['/incidents', 200, cls.INCIDENTS_JSON], + ['/priorities', 200, cls.PRIORITIES_JSON], + ['/enqueue', 200, cls.EVENT_JSON], + ['/notes', 200, cls.NOTE_JSON], + ['/responder_requests', 200, cls.RESPONDER_JSON], + [re.compile(r'^.*/incidents/[a-zA-Z0-9_-]+$'), 200, cls.INCIDENT_JSON], + ] + + def _mocked_call(*args, **kwargs): + for condition in conditions: + matcher, status_code, response = condition + + if callable(matcher): + # Lambda or function + is_condition_match = matcher(*args, **kwargs) + else: + try: + # I couldn't find an easy way to determine if a variable was an instance + # of a regex type, so this was the best I could do + is_condition_match = matcher.match(args[0]) + except AttributeError: + # String + is_condition_match = args[0].endswith(matcher) + + if is_condition_match: + _mock_response = MagicMock() + _mock_response.status_code = status_code + _mock_response.json.return_value = response + return _mock_response + + _404_response = MagicMock() + _404_response.status_code = 404 + return _404_response + + get_mock.side_effect = _mocked_call