Merge pull request #414 from cisagov/DJ-add_intelx-dev

Add intelX scan to Pe_source

src/pe_reports/data/database.ini

@@ -10,6 +10,9 @@ port=
 [dnstwist]
 [hibp]
 
+[intelx]
+api_key=
+
 [postgresql]
 host=
 database=

src/pe_source/data/pe_db/config.py

@@ -82,3 +82,21 @@ def cybersix_token():
     }
     resp = requests.post(url, headers=headers, data=payload).json()
     return resp["access_token"]
+
+
+def get_params(section):
+    """Get data source parameters."""
+    if os.path.isfile(REPORT_DB_CONFIG):
+        parser = ConfigParser()
+        parser.read(REPORT_DB_CONFIG, encoding="utf-8")
+        if parser.has_section(section):
+            params = parser.items(section)
+        else:
+            raise Exception(
+                "Section {} not found in the {} file".format(section, REPORT_DB_CONFIG)
+            )
+    else:
+        raise Exception(
+            "Database.ini file not found at this path: {}".format(REPORT_DB_CONFIG)
+        )
+    return params

src/pe_source/data/pe_db/db_query_source.py

@@ -44,11 +44,11 @@ def close(conn):
 
 
 def get_orgs():
-    """Query organizations table."""
+    """Query organizations that receive reports and demo organizations."""
     conn = connect()
     try:
         cur = conn.cursor()
-        sql = """SELECT * FROM organizations"""
+        sql = """SELECT * FROM organizations where report_on or demo"""
         cur.execute(sql)
         pe_orgs = cur.fetchall()
         keys = ("org_uid", "org_name", "cyhy_db_name")
@@ -224,7 +224,6 @@ def insert_sixgill_breaches(df):
     # SQL query to execute
     query = """INSERT INTO {}({}) VALUES %s
     ON CONFLICT (breach_name) DO UPDATE SET
-    exposed_cred_count = EXCLUDED.exposed_cred_count,
     password_included = EXCLUDED.password_included;"""
     cursor = conn.cursor()
     try:
@@ -271,7 +270,7 @@ def insert_sixgill_credentials(df):
     cols = ",".join(list(df.columns))
     # SQL query to execute
     query = """INSERT INTO {}({}) VALUES %s
-    ON CONFLICT (breach_name, email, name) DO UPDATE SET
+    ON CONFLICT (breach_name, email) DO UPDATE SET
     modified_date = EXCLUDED.modified_date;"""
     cursor = conn.cursor()
     try:
@@ -404,3 +403,82 @@ def org_root_domains(conn, org_uid):
     """
     df = pd.read_sql_query(sql, conn, params={"org_id": org_uid})
     return df
+
+
+def insert_intelx_breaches(df):
+    """Insert IntelX breach data."""
+    conn = connect()
+    table = "credential_breaches"
+    # Create a list of tuples from the dataframe values
+    tuples = [tuple(x) for x in df.to_numpy()]
+    # Comma-separated dataframe columns
+    cols = ",".join(list(df.columns))
+    # SQL query to execute
+    query = """INSERT INTO {}({}) VALUES %s
+    ON CONFLICT (breach_name) DO UPDATE SET
+    password_included = EXCLUDED.password_included;"""
+    cursor = conn.cursor()
+    try:
+        extras.execute_values(
+            cursor,
+            query.format(
+                table,
+                cols,
+            ),
+            tuples,
+        )
+        conn.commit()
+        LOGGER.info("Successfully inserted/updated IntelX breaches into PE database.")
+    except (Exception, psycopg2.DatabaseError) as error:
+        LOGGER.info(error)
+        conn.rollback()
+    cursor.close()
+
+
+def get_intelx_breaches(source_uid):
+    """Get IntelX credential breaches."""
+    conn = connect()
+    try:
+        cur = conn.cursor()
+        sql = """SELECT breach_name, credential_breaches_uid FROM credential_breaches where data_source_uid = %s"""
+        cur.execute(sql, [source_uid])
+        all_breaches = cur.fetchall()
+        cur.close()
+        return all_breaches
+    except (Exception, psycopg2.DatabaseError) as error:
+        LOGGER.error("There was a problem with your database query %s", error)
+    finally:
+        if conn is not None:
+            close(conn)
+
+
+def insert_intelx_credentials(df):
+    """Insert IntelX credential data."""
+    conn = connect()
+    table = "credential_exposures"
+    # Create a list of tuples from the dataframe values
+    tuples = [tuple(x) for x in df.to_numpy()]
+    # Comma-separated dataframe columns
+    cols = ",".join(list(df.columns))
+    # SQL query to execute
+    query = """INSERT INTO {}({}) VALUES %s
+    ON CONFLICT (breach_name, email) DO UPDATE SET
+    modified_date = EXCLUDED.modified_date;"""
+    cursor = conn.cursor()
+    try:
+        extras.execute_values(
+            cursor,
+            query.format(
+                table,
+                cols,
+            ),
+            tuples,
+        )
+        conn.commit()
+        LOGGER.info(
+            "Successfully inserted/updated exposed IntelX credentials into PE database."
+        )
+    except (Exception, psycopg2.DatabaseError) as error:
+        LOGGER.info(error)
+        conn.rollback()
+    cursor.close()