Add new Gmail API Service Account Source type

.gitignore

@@ -304,3 +304,4 @@ demo/
 
 # Secrets
 development/creds.env
+development/gmail_service_account_credentials.json

CHANGELOG.md

@@ -5,6 +5,7 @@
 ### Changed
 
 - #23 - Make notifications more agnostic to multiple source types and improve `RawNotification` model
+- #26 - Add Gmail API Service Account Source type to Notification Sources
 
 ## v0.1.3 - 2021-06-10
 

README.md

@@ -57,14 +57,14 @@ Notification Sources are defined in two steps:
 
 #### 2.1 Define Notification Sources in `configuration.py`
 
-In the `PLUGINS_CONFIG`, under the `nautobot_circuit_maintenance` key, we should define the Notification Sources that will be able later in the UI.
+In the `PLUGINS_CONFIG`, under the `nautobot_circuit_maintenance` key, we should define the Notification Sources that will be able later in the UI, where you will be able to **validate** if the authentication credentials provided are working fine or not.
 
 There are two mandatory attributes (other keys are dependent on the integration type, and will be documented below):
 
 - `name`: Name to identify the Source and will be available in the UI.
 - `url`: URL to reach the Notification Source (i.e. `imap://imap.gmail.com:993`)
 
-> Currently, only IMAP email box integration is supported as URL scheme,
+> Currently, only IMAP and HTTPS (accounts.google.com) integrations are supported as URL scheme
 
 ##### IMAP
 
@@ -90,6 +90,37 @@ PLUGINS_CONFIG = {
 }
 ```
 
+##### Gmail API Service Account
+
+There are 2 extra attributes:
+
+- `account`: Identifier (i.e. email address) to use to impersonate as service account.
+- `credentials_file`: JSON file containing all the necessary data to identify the Service Account.
+
+Enabling Gmail API access and creating a [Service Account](https://support.google.com/a/answer/7378726?hl=en) involves multiple steps that could be summarized as:
+
+1. Create a **New Project** in [Google Cloud Console](https://console.cloud.google.com/).
+2. Under **APIs and Services**, enable **Gmail API** for the selected project.
+3. Still under **APIs and Services**, in **Credentials**, create a new **Service Account** and save the credentials file generated.
+4. With Admin rights, edit the newly created Service Account and expand the **Show Domain-Wide Delegation** section. Enable Google Workspace Domain-wide Delegation and save the changes. Copy the Client ID shown.
+5. With Super Admin rights, open the [Google Workspace admin console](https://admin.google.com). Navigate to **Security**, **API controls**, and select the **Manage Domain Wide Delegation** at the bottom of the page.
+6. Add a new API client and paste in the Client ID copied earlier. In the **OAuth scopes** field add the scopes `https://www.googleapis.com/auth/gmail.readonly` and `https://mail.google.com/`. Save the new client configuration by clicking _Authorize_.
+
+```py
+PLUGINS_CONFIG = {
+    "nautobot_circuit_maintenance": {
+        "notification_sources": [
+            {
+                "name": "my custom name",
+                "account": os.environ.get("CM_NS_1_ACCOUNT", ""),
+                "credentials_file": os.environ.get("CM_NS_1_CREDENTIALS_FILE", ""),
+                "url": os.environ.get("CM_NS_1_URL", ""),
+            }
+        ]
+    }
+}
+```
+
 #### 2.2 Add `Providers` to the Notification Sources
 
 In the Circuit Maintenance plugin UI section, there is a **Notification Sources** button (yellow) where you can configure the Notification Sources to track new circuit maintenance notifications from specific providers.

development/creds.example.env

@@ -12,3 +12,6 @@ NAUTOBOT_SUPERUSER_PASSWORD=admin
 CM_NS_1_ACCOUNT=me@example.com
 CM_NS_1_SECRET=supersecret
 CM_NS_1_URL=imap://imap.gmail.com
+CM_NS_2_ACCOUNT=me@example.com
+CM_NS_2_URL=https://accounts.google.com/o/oauth2/auth
+CM_NS_2_CREDENTIALS_FILE=file_path

development/nautobot_config.py

@@ -307,7 +307,13 @@ def is_truthy(arg):
                 "account": os.environ.get("CM_NS_1_ACCOUNT", ""),
                 "secret": os.environ.get("CM_NS_1_SECRET", ""),
                 "url": os.environ.get("CM_NS_1_URL", ""),
-            }
+            },
+            {
+                "name": "my gmail api source",
+                "url": os.environ.get("CM_NS_2_URL", ""),
+                "account": os.environ.get("CM_NS_2_ACCOUNT", ""),
+                "credentials_file": os.environ.get("CM_NS_2_CREDENTIALS_FILE", ""),
+            },
         ]
     }
 }

nautobot_circuit_maintenance/fixtures/handle_notifications_job.yaml

@@ -14,6 +14,21 @@
     noc_contact: ""
     admin_contact: ""
     comments: ""
+- model: circuits.provider
+  pk: 6de61a64-f873-482a-9e33-97000b7f7515
+  fields:
+    created: 2021-04-30
+    last_updated: 2021-04-30 07:43:14.510402+00:00
+    _custom_field_data:
+      emails_circuit_maintenances: sender2@example.com
+    name: packetfabric
+    slug: packetfabric
+    asn: null
+    account: ""
+    portal_url: ""
+    noc_contact: ""
+    admin_contact: ""
+    comments: ""
 - model: circuits.circuittype
   pk: d1c2ed9d-9376-485b-b058-d26c29b5f22f
   fields:
@@ -59,7 +74,17 @@
     created: 2021-05-06
     last_updated: 2021-05-06 05:19:11.314056+00:00
     _custom_field_data: {}
-    name: example
-    slug: example
+    name: "source imap"
+    slug: "source-imap"
     providers:
       - 0ec89992-0de0-483f-8f53-bdcd179eaecd
+- model: nautobot_circuit_maintenance.notificationsource
+  pk: 5fb2d3e4-4693-472c-8116-b4d7904eba66
+  fields:
+    created: 2021-06-16
+    last_updated: 2021-06-16 09:13:59.564410+00:00
+    _custom_field_data: {}
+    name: "source gmail api"
+    slug: "source-gmail-api"
+    providers:
+      - 6de61a64-f873-482a-9e33-97000b7f7515

nautobot_circuit_maintenance/fixtures/source_gmail_api.yaml

@@ -0,0 +1,65 @@
+---
+- model: circuits.provider
+  pk: 0ec89992-0de0-483f-8f53-bdcd179eaecd
+  fields:
+    created: 2021-04-30
+    last_updated: 2021-04-30 07:43:14.510402+00:00
+    _custom_field_data:
+      emails_circuit_maintenances: sender@example.com
+    name: ntt
+    slug: ntt
+    asn: null
+    account: ""
+    portal_url: ""
+    noc_contact: ""
+    admin_contact: ""
+    comments: ""
+- model: circuits.circuittype
+  pk: d1c2ed9d-9376-485b-b058-d26c29b5f22f
+  fields:
+    created: 2021-05-06
+    last_updated: 2021-05-06 05:16:27.538979+00:00
+    _custom_field_data: {}
+    name: Transit
+    slug: transit
+    description: ""
+- model: circuits.circuit
+  pk: 3ef990ab-6ae0-4e26-bd76-8017f643ab74
+  fields:
+    created: 2021-04-30
+    last_updated: 2021-04-30 07:43:34.845865+00:00
+    _custom_field_data: {}
+    status: null
+    cid: "000001"
+    provider: 0ec89992-0de0-483f-8f53-bdcd179eaecd
+    type: d1c2ed9d-9376-485b-b058-d26c29b5f22f
+    tenant: null
+    install_date: null
+    commit_rate: null
+    description: ""
+    comments: ""
+- model: circuits.circuit
+  pk: fe87375e-a292-4a44-9bc9-830e0fb7a0ec
+  fields:
+    created: 2021-05-06
+    last_updated: 2021-05-06 05:18:36.057186+00:00
+    _custom_field_data: {}
+    status: null
+    cid: "000002"
+    provider: 0ec89992-0de0-483f-8f53-bdcd179eaecd
+    type: d1c2ed9d-9376-485b-b058-d26c29b5f22f
+    tenant: null
+    install_date: null
+    commit_rate: null
+    description: ""
+    comments: ""
+- model: nautobot_circuit_maintenance.notificationsource
+  pk: eb82df7e-6df6-405a-a798-d12e16cc1a50
+  fields:
+    created: 2021-05-06
+    last_updated: 2021-05-06 05:19:11.314056+00:00
+    _custom_field_data: {}
+    name: "source gmail api"
+    slug: "source-gmail-api"
+    providers:
+      - 0ec89992-0de0-483f-8f53-bdcd179eaecd

nautobot_circuit_maintenance/fixtures/source_imap.yaml

@@ -0,0 +1,65 @@
+---
+- model: circuits.provider
+  pk: 0ec89992-0de0-483f-8f53-bdcd179eaecd
+  fields:
+    created: 2021-04-30
+    last_updated: 2021-04-30 07:43:14.510402+00:00
+    _custom_field_data:
+      emails_circuit_maintenances: sender@example.com
+    name: ntt
+    slug: ntt
+    asn: null
+    account: ""
+    portal_url: ""
+    noc_contact: ""
+    admin_contact: ""
+    comments: ""
+- model: circuits.circuittype
+  pk: d1c2ed9d-9376-485b-b058-d26c29b5f22f
+  fields:
+    created: 2021-05-06
+    last_updated: 2021-05-06 05:16:27.538979+00:00
+    _custom_field_data: {}
+    name: Transit
+    slug: transit
+    description: ""
+- model: circuits.circuit
+  pk: 3ef990ab-6ae0-4e26-bd76-8017f643ab74
+  fields:
+    created: 2021-04-30
+    last_updated: 2021-04-30 07:43:34.845865+00:00
+    _custom_field_data: {}
+    status: null
+    cid: "000001"
+    provider: 0ec89992-0de0-483f-8f53-bdcd179eaecd
+    type: d1c2ed9d-9376-485b-b058-d26c29b5f22f
+    tenant: null
+    install_date: null
+    commit_rate: null
+    description: ""
+    comments: ""
+- model: circuits.circuit
+  pk: fe87375e-a292-4a44-9bc9-830e0fb7a0ec
+  fields:
+    created: 2021-05-06
+    last_updated: 2021-05-06 05:18:36.057186+00:00
+    _custom_field_data: {}
+    status: null
+    cid: "000002"
+    provider: 0ec89992-0de0-483f-8f53-bdcd179eaecd
+    type: d1c2ed9d-9376-485b-b058-d26c29b5f22f
+    tenant: null
+    install_date: null
+    commit_rate: null
+    description: ""
+    comments: ""
+- model: nautobot_circuit_maintenance.notificationsource
+  pk: eb82df7e-6df6-405a-a798-d12e16cc1a50
+  fields:
+    created: 2021-05-06
+    last_updated: 2021-05-06 05:19:11.314056+00:00
+    _custom_field_data: {}
+    name: "source imap"
+    slug: "source-imap"
+    providers:
+      - 0ec89992-0de0-483f-8f53-bdcd179eaecd

nautobot_circuit_maintenance/handle_notifications/handler.py

@@ -3,6 +3,7 @@
 import traceback
 from typing import Union
 from django.conf import settings
+from django.db import OperationalError
 from circuit_maintenance_parser import ParsingError, init_parser, MaintenanceNotification
 from nautobot.circuits.models import Circuit, Provider
 from nautobot.extras.jobs import Job
@@ -180,7 +181,6 @@ def process_raw_notification(logger: Job, notification: MaintenanceNotification)
     It creates a RawNotification and if it could be parsed, create the corresponding ParsedNotification.
     """
     parser = init_parser(**notification.__dict__)
-
     if not parser:
         logger.log_warning(message=f"Notification Parser not found for {notification.provider_type}")
         return None
@@ -196,13 +196,18 @@ def process_raw_notification(logger: Job, notification: MaintenanceNotification)
         return None
 
     # Insert raw notification in DB
-    raw_entry, created = RawNotification.objects.get_or_create(
-        subject=parser.subject,
-        provider=provider,
-        raw=parser.raw,
-        sender=parser.sender,
-        source=NotificationSource.objects.filter(name=parser.source).last(),
-    )
+    try:
+        raw_entry, created = RawNotification.objects.get_or_create(
+            subject=parser.subject,
+            provider=provider,
+            raw=parser.raw,
+            sender=parser.sender,
+            source=NotificationSource.objects.filter(name=parser.source).last(),
+        )
+    except OperationalError as exc:
+        logger.log_warning(message=f"Raw notification '{raw_entry.subject}' not created because {str(exc)}")
+        return None
+
     if not created:
         logger.log_warning(message=f"Raw notification '{raw_entry.subject}' already existed with id {raw_entry.pk}")
         return None
@@ -258,8 +263,6 @@ def run(self, data=None, commit=None):
             last_time_processed = None
 
         try:
-            # TODO: get_notifications should be replaced by get_notifications_from_source when new types
-            # are supported
             notifications = get_notifications(
                 logger=self,
                 notification_sources=notification_sources,