Add Duo intel module (#1172)

Adds a Duo intel module

- users, also linking to humans
- groups
- endpoints
- tokens
- webauthncredentials

---------

Co-authored-by: Alex Chantavy <achantavy@lyft.com>

README.md

@@ -33,6 +33,8 @@ Start [here](https://lyft.github.io/cartography/install.html).
 - [NIST CVE](https://lyft.github.io/cartography/modules/cve/index.html) - Common Vulnerabilities and Exposures (CVE) data from NIST database
 - [Lastpass](https://lyft.github.io/cartography/modules/lastpass/index.html) - users
 - [BigFix](https://lyft.github.io/cartography/modules/bigfix/index.html) - Computers
+- [Duo](https://lyft.github.io/cartography/modules/duo/index.html) - Users, Groups, Endpoints
+
 
 ## Usage
 Start with our [tutorial](https://lyft.github.io/cartography/usage/tutorial.html). Our [data schema](https://lyft.github.io/cartography/usage/schema.html) is a helpful reference when you get stuck.

cartography/cli.py

@@ -476,6 +476,30 @@ def _build_parser(self):
                 'The BigFix Root URL, a.k.a the BigFix API URL'
             ),
         )
+        parser.add_argument(
+            '--duo-api-key-env-var',
+            type=str,
+            default=None,
+            help=(
+                'The name of environment variable containing the Duo api key'
+            ),
+        )
+        parser.add_argument(
+            '--duo-api-secret-env-var',
+            type=str,
+            default=None,
+            help=(
+                'The name of environment variable containing the Duo api secret'
+            ),
+        )
+        parser.add_argument(
+            '--duo-api-hostname',
+            type=str,
+            default=None,
+            help=(
+                'The Duo api hostname'
+            ),
+        )
         return parser
 
     def main(self, argv: str) -> int:
@@ -633,6 +657,15 @@ def main(self, argv: str) -> int:
             logger.debug(f"Reading BigFix password from environment variable {config.bigfix_password_env_var}")
             config.bigfix_password = os.environ.get(config.bigfix_password_env_var)
 
+        # Duo config
+        if config.duo_api_key_env_var and config.duo_api_secret_env_var and config.duo_api_hostname:
+            logger.debug(
+                f"Reading Duo api key and secret from environment variables {config.duo_api_key_env_var}"
+                f", {config.duo_api_secret_env_var}",
+            )
+            config.duo_api_key = os.environ.get(config.duo_api_key_env_var)
+            config.duo_api_secret = os.environ.get(config.duo_api_secret_env_var)
+
         # Run cartography
         try:
             return cartography.sync.run_with_config(self.sync, config)

cartography/config.py

@@ -97,6 +97,12 @@ class Config:
     :param bigfix_password: The password to authenticate to BigFix. Optional.
     :type bigfix_root_url: str
     :param bigfix_root_url: The API URL to use for BigFix, e.g. "https://example.com:52311". Optional.
+    :type duo_api_key: str
+    :param duo_api_key: The Duo api key. Optional.
+    :type duo_api_key: str
+    :param duo_api_key: The Duo api secret. Optional.
+    :type duo_api_hostname: str
+    :param duo_api_hostname: The Duo api hostname, e.g. "api-abc123.duosecurity.com". Optional.
     """
 
     def __init__(
@@ -148,6 +154,9 @@ def __init__(
         bigfix_username=None,
         bigfix_password=None,
         bigfix_root_url=None,
+        duo_api_key=None,
+        duo_api_secret=None,
+        duo_api_hostname=None,
     ):
         self.neo4j_uri = neo4j_uri
         self.neo4j_user = neo4j_user
@@ -196,3 +205,6 @@ def __init__(
         self.bigfix_username = bigfix_username
         self.bigfix_password = bigfix_password
         self.bigfix_root_url = bigfix_root_url
+        self.duo_api_key = duo_api_key
+        self.duo_api_secret = duo_api_secret
+        self.duo_api_hostname = duo_api_hostname

cartography/intel/duo/__init__.py

@@ -0,0 +1,90 @@
+import logging
+
+import duo_client
+import neo4j
+
+from cartography.config import Config
+from cartography.intel.duo.api_host import sync_duo_api_host
+from cartography.intel.duo.endpoints import sync_duo_endpoints
+from cartography.intel.duo.groups import sync_duo_groups
+from cartography.intel.duo.phones import sync as sync_duo_phones
+from cartography.intel.duo.tokens import sync as sync_duo_tokens
+from cartography.intel.duo.users import sync_duo_users
+from cartography.intel.duo.web_authn_credentials import sync as sync_duo_web_authn_credentials
+from cartography.util import timeit
+
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get_client(config: Config) -> duo_client.Admin:
+    '''
+    Return a duo Admin client with the creds in the config object
+    '''
+    return duo_client.Admin(
+        ikey=config.duo_api_key,
+        skey=config.duo_api_secret,
+        host=config.duo_api_hostname,
+    )
+
+
+@timeit
+def start_duo_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    '''
+    If this module is configured, perform ingestion of duo data. Otherwise warn and exit
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    '''
+    if not all([
+        config.duo_api_key,
+        config.duo_api_secret,
+        config.duo_api_hostname,
+    ]):
+        logger.info(
+            'Duo import is not configured - skipping this module. '
+            'See docs to configure.',
+        )
+        return
+
+    client = get_client(config)
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+        "DUO_API_HOSTNAME": config.duo_api_hostname,
+    }
+
+    sync_duo_api_host(
+        neo4j_session,
+        common_job_parameters,
+    )
+    sync_duo_tokens(
+        client,
+        neo4j_session,
+        common_job_parameters,
+    )
+    sync_duo_web_authn_credentials(
+        client,
+        neo4j_session,
+        common_job_parameters,
+    )
+    sync_duo_endpoints(
+        client,
+        neo4j_session,
+        common_job_parameters,
+    )
+    sync_duo_phones(
+        client,
+        neo4j_session,
+        common_job_parameters,
+    )
+    sync_duo_groups(
+        client,
+        neo4j_session,
+        common_job_parameters,
+    )
+    sync_duo_users(
+        client,
+        neo4j_session,
+        common_job_parameters,
+    )

cartography/intel/duo/api_host.py

@@ -0,0 +1,38 @@
+import logging
+from typing import Any
+from typing import Dict
+
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.models.duo.api_host import DuoApiHostSchema
+from cartography.util import timeit
+
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync_duo_api_host(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
+    '''
+    Add the DuoApiHost subresource
+    '''
+    _load_api_host(neo4j_session, common_job_parameters)
+
+
+@timeit
+def _load_api_host(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
+    '''
+    Load the host node into the graph
+    '''
+    data = [
+        {
+            'id': common_job_parameters['DUO_API_HOSTNAME'],
+        },
+    ]
+    load(
+        neo4j_session,
+        DuoApiHostSchema(),
+        data,
+        lastupdated=common_job_parameters['UPDATE_TAG'],
+    )

cartography/intel/duo/endpoints.py

@@ -0,0 +1,110 @@
+import logging
+from json import dumps
+from typing import Any
+from typing import Dict
+from typing import List
+
+import duo_client
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.duo.endpoint import DuoEndpointSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync_duo_endpoints(
+    client: duo_client.Admin,
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    '''
+    Sync Duo Endpoints
+    '''
+    endpoints = _get_endpoints(client)
+    transformed_endpoints = _transform_endpoints(endpoints)
+    _load_endpoints(neo4j_session, transformed_endpoints, common_job_parameters)
+    _cleanup_endpoints(neo4j_session, common_job_parameters)
+
+
+@timeit
+def _get_endpoints(client: duo_client.Admin) -> List[Dict[str, Any]]:
+    '''
+    Fetch all endpoint data
+    https://duo.com/docs/adminapi#endpoints
+    '''
+    logger.info("Fetching Duo endpoints")
+    return client.get_endpoints()
+
+
+@timeit
+def _transform_endpoints(endpoints: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    '''
+    Reformat the data before loading
+    '''
+    logger.info(f'Transforming {len(endpoints)} duo endpoints')
+    transformed_endpoints = []
+    for endpoint in endpoints:
+        transformed_endpoint = {
+            'browsers': [dumps(browser) for browser in endpoint['browsers']],
+            'computer_sid': endpoint['computer_sid'],
+            'cpu_id': endpoint['cpu_id'],
+            'device_id': endpoint['device_id'],
+            'device_identifier': endpoint['device_identifier'],
+            'device_identifier_type': endpoint['device_identifier_type'],
+            'device_name': endpoint['device_name'],
+            'device_udid': endpoint['device_udid'],
+            'device_username': endpoint['device_username'],
+            'device_username_type': endpoint['device_username_type'],
+            'disk_encryption_status': endpoint['disk_encryption_status'],
+            'domain_sid': endpoint['domain_sid'],
+            'email': endpoint['email'],
+            'epkey': endpoint['epkey'],
+            'firewall_status': endpoint['firewall_status'],
+            'hardware_uuid': endpoint['hardware_uuid'],
+            'health_app_client_version': endpoint['health_app_client_version'],
+            'health_data_last_collected': endpoint['health_data_last_collected'],
+            'last_updated': endpoint['last_updated'],
+            'machine_guid': endpoint['machine_guid'],
+            'model': endpoint['model'],
+            'os_build': endpoint['os_build'],
+            'os_family': endpoint['os_family'],
+            'os_version': endpoint['os_version'],
+            'password_status': endpoint['password_status'],
+            'security_agents': [dumps(agent) for agent in endpoint['security_agents']],
+            'trusted_endpoint': endpoint['trusted_endpoint'],
+            'type': endpoint['type'],
+            'username': endpoint['username'],
+        }
+        transformed_endpoints.append(transformed_endpoint)
+    return transformed_endpoints
+
+
+@timeit
+def _load_endpoints(
+    neo4j_session: neo4j.Session,
+    endpoints: List[Dict[str, Any]],
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    '''
+    Load the endpoints into the database
+    '''
+    logger.info(f'Loading {len(endpoints)} duo endpoints')
+    load(
+        neo4j_session,
+        DuoEndpointSchema(),
+        endpoints,
+        DUO_API_HOSTNAME=common_job_parameters['DUO_API_HOSTNAME'],
+        lastupdated=common_job_parameters['UPDATE_TAG'],
+    )
+
+
+@timeit
+def _cleanup_endpoints(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
+    '''
+    Cleanup endpoints
+    '''
+    GraphJob.from_node_schema(DuoEndpointSchema(), common_job_parameters).run(neo4j_session)

cartography/intel/duo/groups.py

@@ -0,0 +1,66 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import duo_client
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.duo.group import DuoGroupSchema
+from cartography.util import timeit
+
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync_duo_groups(
+    client: duo_client.Admin,
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    '''
+    Sync Duo groups
+    '''
+    groups = _get_groups(client)
+    _load_groups(neo4j_session, groups, common_job_parameters)
+    _cleanup_groups(neo4j_session, common_job_parameters)
+
+
+@timeit
+def _get_groups(client: duo_client.Admin) -> List[Dict[str, Any]]:
+    '''
+    Fetch all group data
+    https://duo.com/docs/adminapi#users
+    '''
+    logger.info("Fetching Duo groups")
+    return client.get_groups()
+
+
+@timeit
+def _load_groups(
+    neo4j_session: neo4j.Session,
+    groups: List[Dict[str, Any]],
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    '''
+    Load the groups into the graph
+    '''
+    logger.info(f'Loading {len(groups)} duo groups')
+    load(
+        neo4j_session,
+        DuoGroupSchema(),
+        groups,
+        DUO_API_HOSTNAME=common_job_parameters['DUO_API_HOSTNAME'],
+        lastupdated=common_job_parameters['UPDATE_TAG'],
+    )
+
+
+@timeit
+def _cleanup_groups(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
+    '''
+    Cleanup endpoints
+    '''
+    GraphJob.from_node_schema(DuoGroupSchema(), common_job_parameters).run(neo4j_session)