Add Okta intel module. (#137)

* Fix for issue #31

* internal_commit

* adding factors

* trusted origin, full sync ok

* api doc

* run clean - added transform layer

* Update oktaintel.py

working full sync

* Update oktaintel.py

* updated doc

* unit test foundation

* Update __init__.py

* lint

* Update oktaintel.py

* Update oktaintel.py

* Update oktaintel.py

* Update oktaintel.py

* unit tests

* cred setup

* sync integration

* Update okta_import_cleanup.json

* lint bs

* lint bs

* Group member bug fix

* PR Feedback

* Update oktaintel.py

* Update README.md

* removing cli parameter

* evan review part 1

* evan feedback part 2 - refactoring into smaller chunks

* evan feedback - part 2 - CLI parameters

* lint

* utils

* testing

* bug fix

* Update cli.py

* splitting get  and transform

* Update users.py

* Update groups.py

* Update roles.py

* fix doc

* change unit test

* Update test_syntax.py

* fix unit test

* fix index

* Address Alex feedback - store data in memory vs graph call

* fix

* lint

* Update okta_import_cleanup.json

README.md

@@ -98,8 +98,12 @@ Time to set up the server that will run Cartography.  Cartography _should_ work
         1. `GSUITE_GOOGLE_APPLICATION_CREDENTIALS` - location of the credentials file.
         2. `GSUITE_DELEGATED_ADMIN` - email address that you created in step 2
 
+6. If you're using Okta intel module, **prepare your Okta API token**
+    1. Generate your API token by following the steps from [Okta Create An API Token documentation](https://developer.okta.com/docs/guides/create-an-api-token/overview/)
+    2. Populate an environment variable with the API token. You can pass the environment variable name via the cli --okta-api-key-env-var parameter
+    3. Use the cli --okta-org-id parameter with the organization id you want to query. The organization id is the first part of the Okta url for your organization.
 
-5. **Get and run Cartography**
+7. **Get and run Cartography**
 
 	1. Run `pip install cartography` to install our code.
 
@@ -371,7 +375,6 @@ There are many ways to allow Cartography to pull from more than one AWS account.
 			region=<the region of your Hub account, e.g. us-east-1>
 			output=json
 
-
 	3.  Add a profile for each AWS account you want Cartography to sync with to your `AWS_CONFIG_FILE`.  It will look something like this:
 
 		```

cartography/cli.py

@@ -124,6 +124,23 @@ def _build_parser(self):
                 'jobs are executed.'
             ),
         )
+        parser.add_argument(
+            '--okta-org-id',
+            type=str,
+            default=None,
+            help=(
+                'Okta organizational id to sync. Required if you are using the Okta intel module. Ignored otherwise.'
+            ),
+        )
+        parser.add_argument(
+            '--okta-api-key-env-var',
+            type=str,
+            default=None,
+            help=(
+                'The name of an environment variable containing a key with which to auth to the Okta API.'
+                'Required if you are using the Okta intel module. Ignored otherwise.'
+            ),
+        )
         return parser
 
     def main(self, argv):
@@ -158,6 +175,12 @@ def main(self, argv):
                 logger.warning("Neo4j username was provided but a password could not be found.")
         else:
             config.neo4j_password = None
+        # Okta config
+        if config.okta_org_id and config.okta_api_key_env_var:
+            logger.debug(f"Reading API key for Okta from environment variable {config.okta_api_key_env_var}")
+            config.okta_api_key = os.environ.get(config.okta_api_key_env_var)
+
+        # Run cartography
         try:
             return cartography.sync.run_with_config(self.sync, config)
         except KeyboardInterrupt:

cartography/config.py

@@ -19,6 +19,10 @@ class Config:
         False (default), AWS sync will run using the default credentials only. Optional.
     :type analysis_job_directory: str
     :param analysis_job_directory: Path to a directory tree containing analysis jobs to run. Optional.
+    :type okta_org_id: str
+    :param okta_org_id: Okta organization id. Optional.
+    :type okta_api_key: str
+    :param okta_api_key: Okta API key. Optional.
     """
 
     def __init__(
@@ -29,10 +33,14 @@ def __init__(
         update_tag=None,
         aws_sync_all_profiles=False,
         analysis_job_directory=None,
+        okta_org_id=None,
+        okta_api_key=None,
     ):
         self.neo4j_uri = neo4j_uri
         self.neo4j_user = neo4j_user
         self.neo4j_password = neo4j_password
         self.update_tag = update_tag
         self.aws_sync_all_profiles = aws_sync_all_profiles
         self.analysis_job_directory = analysis_job_directory
+        self.okta_org_id = okta_org_id
+        self.okta_api_key = okta_api_key

cartography/data/indexes.cypher

@@ -58,6 +58,14 @@ CREATE INDEX ON :IpRule(ruleid);
 CREATE INDEX ON :LoadBalancer(dnsname);
 CREATE INDEX ON :LoadBalancer(id);
 CREATE INDEX ON :NetworkInterface(id);
+CREATE INDEX ON :OktaOrganization(id);
+CREATE INDEX ON :OktaUser(id);
+CREATE INDEX ON :OktaUser(email);
+CREATE INDEX ON :OktaGroup(id);
+CREATE INDEX ON :OktaApplication(id);
+CREATE INDEX ON :OktaUserFactor(id);
+CREATE INDEX ON :OktaTrustedOrigin(id);
+CREATE INDEX ON: OktaAdministrationRole(id);
 CREATE INDEX ON :PublicIpAddress(ip);
 CREATE INDEX ON :RDSInstance(db_instance_identifier);
 CREATE INDEX ON :RDSInstance(id);

cartography/data/jobs/cleanup/okta_import_cleanup.json

@@ -0,0 +1,70 @@
+{
+  "statements": [
+    {
+      "query": "MATCH (:OktaOrganization{id: {OKTA_ORG_ID}})-[:RESOURCE]->(n:OktaUser)-[:FACTOR]->(n:OktaUserFactor) WHERE n.lastupdated <> {UPDATE_TAG} WITH n LIMIT {LIMIT_SIZE} DETACH DELETE (n) return COUNT(*) as TotalCompleted",
+      "iterative": true,
+      "iterationsize": 100,
+      "__comment__": "Deletate stale OktaUserFactor"
+    },
+    {
+      "query": "MATCH (:OktaOrganization{id: {OKTA_ORG_ID}})-[:RESOURCE]->(:OktaUser)-[r:FACTOR]->(:OktaUserFactor)  WHERE r.lastupdated <> {UPDATE_TAG} WITH r LIMIT {LIMIT_SIZE} DELETE (r) return COUNT(*) as TotalCompleted",
+      "iterative": true,
+      "iterationsize": 100,
+      "__comment__": "Delete stale OktaUserFactor to OktaUser relationship"
+    },
+    {
+      "query": "MATCH (:OktaOrganization{id: {OKTA_ORG_ID}})-[:RESOURCE]->(n:OktaUser) WHERE n.lastupdated <> {UPDATE_TAG} WITH n LIMIT {LIMIT_SIZE} DETACH DELETE (n) return COUNT(*) as TotalCompleted",
+      "iterative": true,
+      "iterationsize": 100,
+      "__comment__": "Delete stale OktaUser"
+    },
+    {
+      "query": "MATCH (:OktaOrganization{id: {OKTA_ORG_ID}})-[:RESOURCE]->(:OktaGroup)<-[r:MEMBER_OF_OKTA_GROUP]-(:OktaUser) WHERE r.lastupdated <> {UPDATE_TAG} WITH r LIMIT {LIMIT_SIZE} DELETE (r) return COUNT(*) as TotalCompleted",
+      "iterative": true,
+      "iterationsize": 100,
+      "__comment__": "Delete stale OktaUser relationship to OktaGroup"
+    },
+    {
+      "query": "MATCH (:OktaOrganization{id: {OKTA_ORG_ID}})-[:RESOURCE]->(n:OktaGroup) WHERE n.lastupdated <> {UPDATE_TAG} WITH n LIMIT {LIMIT_SIZE} DETACH DELETE (n) return COUNT(*) as TotalCompleted",
+      "iterative": true,
+      "iterationsize": 100,
+      "__comment__": "Delete stale OktaGroup"
+    },
+    {
+      "query": "MATCH (:OktaOrganization{id: {OKTA_ORG_ID}})-[:RESOURCE]->(n:OktaApplication) WHERE n.lastupdated <> {UPDATE_TAG} WITH n LIMIT {LIMIT_SIZE} DETACH DELETE (n) return COUNT(*) as TotalCompleted",
+      "iterative": true,
+      "iterationsize": 100,
+      "__comment__": "Delete stale OktaApplication"
+    },
+    {
+      "query": "MATCH (:OktaOrganization{id: {OKTA_ORG_ID}})-[:RESOURCE]->(:OktaApplication)<-[r:APPLICATION]-(n) WHERE r.lastupdated <> {UPDATE_TAG} WITH r LIMIT {LIMIT_SIZE} DELETE (r) return COUNT(*) as TotalCompleted",
+      "iterative": true,
+      "iterationsize": 100,
+      "__comment__": "Delete stale OktaApplication relationships"
+    },
+    {
+      "query": "MATCH (:OktaOrganization{id: {OKTA_ORG_ID}})-[:RESOURCE]->(n:OktaTrustedOrigin) WHERE n.lastupdated <> {UPDATE_TAG} WITH n LIMIT {LIMIT_SIZE} DETACH DELETE (n) return COUNT(*) as TotalCompleted",
+      "iterative": true,
+      "iterationsize": 100,
+      "__comment__": "Delete stale OktaTrustedOrigin"
+    },
+    {
+      "query": "MATCH (:OktaOrganization{id: {OKTA_ORG_ID}})-[:RESOURCE]->(n:OktaAdministrationRole) WHERE n.lastupdated <> {UPDATE_TAG} WITH n LIMIT {LIMIT_SIZE} DETACH DELETE (n) return COUNT(*) as TotalCompleted",
+      "iterative": true,
+      "iterationsize": 100,
+      "__comment__": "Delete stale OktaAdministrationRole"
+    },
+    {
+      "query": "MATCH (:OktaOrganization{id: {OKTA_ORG_ID}})-[:RESOURCE]->(:OktaAdministrationRole)<-[r:MEMBER_OF_OKTA_ROLE]-(n)  WHERE r.lastupdated <> {UPDATE_TAG} WITH r LIMIT {LIMIT_SIZE} DELETE (r) return COUNT(*) as TotalCompleted",
+      "iterative": true,
+      "iterationsize": 100,
+      "__comment__": "Delete stale OktaAdministrationRole relationships"
+    },
+    {
+      "query": "MATCH (n:OktaOrganization{id: {OKTA_ORG_ID}}) WHERE n.lastupdated <> {UPDATE_TAG} DELETE (n)",
+      "iterative": false,
+      "__comment__": "Delete stale OktaOrganization"
+    }
+  ],
+  "name": "Okta intel module cleanup"
+}

cartography/intel/okta/__init__.py

@@ -0,0 +1,66 @@
+import logging
+
+from okta.framework.OktaError import OktaError
+
+from cartography.intel.okta import applications
+from cartography.intel.okta import factors
+from cartography.intel.okta import groups
+from cartography.intel.okta import organization
+from cartography.intel.okta import origins
+from cartography.intel.okta import roles
+from cartography.intel.okta import users
+from cartography.intel.okta.sync_state import OktaSyncState
+from cartography.util import run_cleanup_job
+
+logger = logging.getLogger(__name__)
+
+
+def _cleanup_okta_organizations(session, common_job_parameters):
+    """
+    Remove stale Okta organization
+    :param session: The Neo4j session
+    :param common_job_parameters: Parameters to carry to the cleanup job
+    :return: Nothing
+    """
+
+    run_cleanup_job('okta_import_cleanup.json', session, common_job_parameters)
+
+
+def start_okta_ingestion(neo4j_session, config):
+    """
+    Starts the OKTA ingestion process
+    :param neo4j_session: The Neo4j session
+    :param config: A `cartography.config` object
+    :return: Nothing
+    """
+
+    logger.debug(f"Starting Okta sync on {config.okta_org_id}")
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+        "OKTA_ORG_ID": config.okta_org_id,
+    }
+
+    state = OktaSyncState()
+
+    organization.create_okta_organization(neo4j_session, config.okta_org_id, config.update_tag)
+    users.sync_okta_users(neo4j_session, config.okta_org_id, config.update_tag, config.okta_api_key, state)
+    groups.sync_okta_groups(neo4j_session, config.okta_org_id, config.update_tag, config.okta_api_key, state)
+    applications.sync_okta_applications(neo4j_session, config.okta_org_id, config.update_tag, config.okta_api_key)
+    factors.sync_users_factors(neo4j_session, config.okta_org_id, config.update_tag, config.okta_api_key, state)
+    origins.sync_trusted_origins(neo4j_session, config.okta_org_id, config.update_tag, config.okta_api_key)
+
+    # need creds with permission
+    # soft fail as some won't be able to get such high priv token
+    # when we get the E0000006 error
+    # see https://developer.okta.com/docs/reference/error-codes/
+    try:
+        roles.sync_roles(neo4j_session, config.okta_org_id, config.update_tag, config.okta_api_key, state)
+    except OktaError as okta_error:
+        logger.warning(f"Unable to pull admin roles got {okta_error}")
+
+        # Getting roles requires super admin which most won't be able to get easily
+        if okta_error.error_code == "E0000006":
+            logger.warning("Unable to sync admin roles - api token needs admin rights to pull admin roles data")
+
+    _cleanup_okta_organizations(neo4j_session, common_job_parameters)