Add Rapid7 InsightVM source

cartography/cli.py

@@ -393,6 +393,54 @@ def _build_parser(self):
                 'The crowdstrike URL, if using self-hosted. Defaults to the public crowdstrike API URL otherwise.'
             ),
         )
+        parser.add_argument(
+            '--rapid7-user',
+            type=str,
+            default=None,
+            help=(
+                'The rapid7 user for authentication.'
+            ),
+        )
+        parser.add_argument(
+            '--rapid7-password-env-var',
+            type=str,
+            default=None,
+            help=(
+                'The name of environment variable containing the rapid7 user password for authentication.'
+            ),
+        )
+        parser.add_argument(
+            '--rapid7-server-url',
+            type=str,
+            default=None,
+            help=(
+                'The url of the Rapid7 InsightsVM server. Required in all Rapid7 data retrieval.'
+            ),
+        )
+        parser.add_argument(
+            '--rapid7-verify-cert',
+            type=str,
+            default="True",
+            help=(
+                'Validate https certificate of Rapid7 InsightsVM server.'
+            ),
+        )
+        parser.add_argument(
+            '--rapid7-report-id',
+            type=str,
+            default=None,
+            help=(
+                'Rapid7 report id if downloading report option. Use a negative id to list accessible reports.'
+            ),
+        )
+        parser.add_argument(
+            '--rapid7-dirpath',
+            type=str,
+            default=None,
+            help=(
+                'Directory path where to find Rapid7 csv data if using that option.'
+            ),
+        )
         parser.add_argument(
             '--experimental-neo4j-4x-support',
             default=False,
@@ -537,6 +585,14 @@ def main(self, argv: str) -> int:
                 ' and this option will be removed when code hard-coded syntax upgrades are completed',
             )
 
+        if config.rapid7_password_env_var:
+            logger.debug(
+                f"Reading password for rapid7 from environment variable {config.rapid7_password_env_var}",
+            )
+            config.rapid7_password = os.environ.get(config.rapid7_password_env_var)
+        else:
+            config.rapid7_password = None
+
         # Run cartography
         try:
             return cartography.sync.run_with_config(self.sync, config)

cartography/config.py

@@ -119,6 +119,12 @@ def __init__(
         crowdstrike_client_id=None,
         crowdstrike_client_secret=None,
         crowdstrike_api_url=None,
+        rapid7_user=None,
+        rapid7_password=None,
+        rapid7_server_url=None,
+        rapid7_verify_cert=None,
+        rapid7_dirpath=None,
+        rapid7_report_id=None,
     ):
         self.neo4j_uri = neo4j_uri
         self.neo4j_user = neo4j_user
@@ -158,3 +164,9 @@ def __init__(
         self.crowdstrike_client_id = crowdstrike_client_id
         self.crowdstrike_client_secret = crowdstrike_client_secret
         self.crowdstrike_api_url = crowdstrike_api_url
+        self.rapid7_user = rapid7_user
+        self.rapid7_password = rapid7_password
+        self.rapid7_server_url = rapid7_server_url
+        self.rapid7_verify_cert = rapid7_verify_cert
+        self.rapid7_dirpath = rapid7_dirpath
+        self.rapid7_report_id = rapid7_report_id

cartography/data/indexes.cypher

@@ -313,6 +313,8 @@ CREATE INDEX IF NOT EXISTS FOR (n:PublicIpAddress) ON (n.ip);
 CREATE INDEX IF NOT EXISTS FOR (n:PublicIpAddress) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:PythonLibrary) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:PythonLibrary) ON (n.lastupdated);
+CREATE INDEX IF NOT EXISTS FOR (n:Rapid7Host) ON (n.id);
+CREATE INDEX IF NOT EXISTS FOR (n:Rapid7Host) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:RedshiftCluster) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:RedshiftCluster) ON (n.arn);
 CREATE INDEX IF NOT EXISTS FOR (n:RedshiftCluster) ON (n.lastupdated);

cartography/data/jobs/cleanup/rapid7_import_cleanup.json

@@ -0,0 +1,15 @@
+{
+  "statements": [
+    {
+      "query": "MATCH (h:Rapid7Host) WHERE h.lastupdated <> $UPDATE_TAG WITH h LIMIT $LIMIT_SIZE DETACH DELETE (h)",
+      "iterative": true,
+      "iterationsize": 100
+    },
+    {
+      "query": "MATCH (:AzureVirtualMachine)-[r:PRESENT_IN]->(:Rapid7Host{id: $AZURE_RESOURCE_ID}) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
+      "iterative": true,
+      "iterationsize": 100
+    }
+  ],
+  "name": "cleanup rapid7"
+}

cartography/intel/rapid7/__init__.py

@@ -0,0 +1,75 @@
+"""
+cartography/intel/rapid7
+"""
+import logging
+
+import neo4j
+
+from cartography.config import Config
+from cartography.intel.rapid7.endpoints import sync_hosts
+from cartography.stats import get_stats_client
+from cartography.util import merge_module_sync_metadata
+from cartography.util import run_cleanup_job
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+stat_handler = get_stats_client(__name__)
+
+
+@timeit
+def start_rapid7_ingestion(
+    neo4j_session: neo4j.Session,
+    config: Config,
+) -> None:
+    """
+    Perform ingestion of Rapid7 data.
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+    }
+    if (
+        (
+            not config.rapid7_user or
+            not config.rapid7_password or
+            not config.rapid7_server_url
+        ) and
+        (not config.rapid7_dirpath or not config.rapid7_server_url) and
+        (not config.rapid7_report_id or not config.rapid7_server_url)
+    ):
+        logger.error("rapid7 config not found")
+        return
+
+    authorization = (
+        config.rapid7_user,
+        config.rapid7_password,
+        config.rapid7_server_url,
+        config.rapid7_verify_cert,
+        config.rapid7_dirpath,
+        config.rapid7_report_id,
+    )
+    # pylint: disable=too-many-function-args
+    sync_hosts(
+        neo4j_session,
+        config.update_tag,
+        authorization,
+    )
+    run_cleanup_job(
+        "rapid7_import_cleanup.json",
+        neo4j_session,
+        common_job_parameters,
+    )
+
+    group_id = "public"
+    if config.rapid7_server_url:
+        group_id = config.rapid7_server_url
+    merge_module_sync_metadata(
+        neo4j_session,
+        group_type="rapid7",
+        group_id=group_id,
+        synced_type="rapid7",
+        update_tag=config.update_tag,
+        stat_handler=stat_handler,
+    )

cartography/intel/rapid7/endpoints.py

@@ -0,0 +1,82 @@
+"""
+cartography/intel/rapid7/endpoints
+"""
+# pylint: disable=missing-function-docstring,too-many-arguments
+import logging
+from typing import Dict
+from typing import List
+from typing import Tuple
+
+import neo4j
+
+from .util import rapid7_hosts
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync_hosts(
+    neo4j_session: neo4j.Session,
+    update_tag: int,
+    authorization: Tuple[str, str, str, str, str, int],
+) -> None:
+    r7_hosts = rapid7_hosts(authorization)
+    for host_data in r7_hosts:
+        load_host_data(neo4j_session, host_data, update_tag)
+
+
+def load_host_data(
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    update_tag: int,
+) -> None:
+    """
+    Transform and load scan information
+    """
+    ingestion_cypher_query = """
+    UNWIND $Hosts AS host
+        MERGE (h:Rapid7Host{id: host.id})
+        ON CREATE SET h.r7_id = host.id,
+            h.firstseen = timestamp()
+        SET h.r7_assessedForPolicies = host.assessedForPolicies,
+            h.r7_assessedForVulnerabilities = host.assessedForVulnerabilities,
+            h.hostname = host.hostName,
+            h.short_hostname = host.short_hostname,
+            h.r7_ip = host.ip,
+            h.r7_mac = host.mac,
+            h.r7_os = host.os,
+            h.r7_rawriskscore = host.rawRiskScore,
+            h.r7_riskscore = host.riskScore,
+            h.r7_architecture = host.osFingerprint_architecture,
+            h.r7_os_product = host.osFingerprint_product,
+            h.r7_os_version = host.osFingerprint_version,
+            h.r7_vulnerabilities_critical = host.vulnerabilities_critical,
+            h.r7_vulnerabilities_exploits = host.vulnerabilities_exploits,
+            h.r7_vulnerabilities_malwareKits = host.vulnerabilities_malwareKits,
+            h.r7_vulnerabilities_moderate = host.vulnerabilities_moderate,
+            h.r7_vulnerabilities_severe = host.vulnerabilities_severe,
+            h.r7_vulnerabilities_total = host.vulnerabilities_total,
+            h.tool_first_seen = host.tool_first_seen,
+            h.tool_last_seen = host.tool_last_seen,
+            h.r7_type = host.type,
+            h.r7_sites = host.sites,
+            h.cloud_provider = host.cloud_provider,
+            h.instance_id = host.instance_id,
+            h.subscription_id = host.subscription_id,
+            h.resource_id = host.resource_id,
+            h.resource_group = host.resource_group,
+            h.modified_timestamp = host.modified_timestamp,
+            h.lastupdated = $update_tag
+        WITH h
+        MATCH (s:AzureVirtualMachine{id: h.resource_id})
+        MERGE (s)-[r:PRESENT_IN]->(h)
+        ON CREATE SET r.firstseen = timestamp()
+        SET r.lastupdated = $update_tag
+    """
+    logger.debug("Loading %s rapid7 hosts.", len(data))
+    neo4j_session.run(
+        ingestion_cypher_query,
+        Hosts=data,
+        update_tag=update_tag,
+    )