Merge 1c87c7d into dd5a870

setup.py

@@ -87,8 +87,14 @@ def get_version(version_file):
     package_dir={"": "src"},
     package_data={
         "pe_mailer": ["data/*"],
-        "pe_reports": ["data/shell/*.pptx", "data/*.ini", "helpers/*"],
-        "pe_source": ["data/*", "data/shodan/*", "data/sixgill/*", "data/pe_db/*"],
+        "pe_reports": ["data/shell/*.pptx", "data/*.ini"],
+        "pe_source": [
+            "data/*",
+            "data/shodan/*",
+            "data/sixgill/*",
+            "data/dnsmonitor/*",
+            "data/pe_db/*",
+        ],
     },
     py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")],
     include_package_data=True,

src/pe_reports/data/database.ini

@@ -27,5 +27,9 @@ key4=
 client_id=
 client_secret=
 
+[dnsmonitor]
+client_id=
+client_secret=
+
 [rapid7]
 [whoisxml]

src/pe_reports/data/db_query.py

@@ -98,30 +98,6 @@ def query_creds_view(org_uid, start_date, end_date):
             close(conn)
 
 
-def query_domMasq(org_uid, start_date, end_date):
-    """Query domain masquerading table."""
-    conn = connect()
-    try:
-        sql = """SELECT * FROM dnstwist_domain_masq
-        WHERE organizations_uid = %(org_uid)s
-        AND date_observed BETWEEN %(start_date)s AND %(end_date)s"""
-        df = pd.read_sql(
-            sql,
-            conn,
-            params={
-                "org_uid": org_uid,
-                "start_date": start_date,
-                "end_date": end_date,
-            },
-        )
-        return df
-    except (Exception, psycopg2.DatabaseError) as error:
-        LOGGER.error("There was a problem with your database query %s", error)
-    finally:
-        if conn is not None:
-            close(conn)
-
-
 # The 'table' parameter is used in query_shodan, query_darkweb and
 # query_darkweb_cves functions to call specific tables that relate to the
 # function name.  The result of this implementation reduces the code base,
@@ -226,3 +202,51 @@ def query_cyberSix_creds(org_uid, start_date, end_date):
     finally:
         if conn is not None:
             close(conn)
+
+
+def query_domMasq(org_uid, start_date, end_date):
+    """Query domain masquerading table."""
+    conn = connect()
+    try:
+        sql = """SELECT * FROM domain_permutations
+        WHERE organizations_uid = %(org_uid)s
+        AND date_active BETWEEN %(start_date)s AND %(end_date)s"""
+        df = pd.read_sql(
+            sql,
+            conn,
+            params={
+                "org_uid": org_uid,
+                "start_date": start_date,
+                "end_date": end_date,
+            },
+        )
+        return df
+    except (Exception, psycopg2.DatabaseError) as error:
+        logging.error("There was a problem with your database query %s", error)
+    finally:
+        if conn is not None:
+            close(conn)
+
+
+def query_domMasq_alerts(org_uid, start_date, end_date):
+    """Query domain alerts table."""
+    conn = connect()
+    try:
+        sql = """SELECT * FROM domain_alerts
+        WHERE organizations_uid = %(org_uid)s
+        AND date BETWEEN %(start_date)s AND %(end_date)s"""
+        df = pd.read_sql(
+            sql,
+            conn,
+            params={
+                "org_uid": org_uid,
+                "start_date": start_date,
+                "end_date": end_date,
+            },
+        )
+        return df
+    except (Exception, psycopg2.DatabaseError) as error:
+        logging.error("There was a problem with your database query %s", error)
+    finally:
+        if conn is not None:
+            close(conn)

src/pe_reports/metrics.py

@@ -12,6 +12,7 @@
     query_darkweb,
     query_darkweb_cves,
     query_domMasq,
+    query_domMasq_alerts,
     query_shodan,
 )
 
@@ -153,6 +154,7 @@ def __init__(self, start_date, end_date, org_uid):
         self.org_uid = org_uid
         df = query_domMasq(org_uid, start_date, end_date)
         self.df_mal = df[df["malicious"]]
+        self.dom_alerts_df = query_domMasq_alerts(org_uid, start_date, end_date)
 
     def count(self):
         """Return total count of malicious domains."""
@@ -171,7 +173,8 @@ def summary(self):
                     "name_server",
                 ]
             ]
-            domain_sum = domain_sum[:25]
+            domain_sum = domain_sum[:10]
+            domain_sum.loc[domain_sum["ipv6"] == "", "ipv6"] = "NA"
             domain_sum = domain_sum.rename(
                 columns={
                     "domain_permutation": "Domain",
@@ -193,23 +196,26 @@ def summary(self):
             )
         return domain_sum
 
-    def utlds(self):
-        """Return count of unique top level domains."""
-        mal_df = self.df_mal
-
-        if len(mal_df.index) > 0:
-            mal_df["tld"] = (
-                mal_df["domain_permutation"]
-                .str.split(".")
-                .str[-1]
-                .str.split("/")
-                .str[0]
-            )
-            utlds = len(mal_df["tld"].unique())
-        else:
-            utlds = 0
+    def alert_count(self):
+        """Return number of alerts."""
+        dom_alert_count = len(self.dom_alerts_df)
+        return dom_alert_count
 
-        return utlds
+    def alerts(self):
+        """Return domain alerts."""
+        dom_alerts_df = self.dom_alerts_df[["message", "date"]]
+        dom_alerts_df = dom_alerts_df.rename(
+            columns={"message": "Alert", "date": "Date"}
+        )
+        dom_alerts_df = dom_alerts_df[:10].reset_index(drop=True)
+        return dom_alerts_df
+
+    def alerts_sum(self):
+        """Return domain alerts summary."""
+        dom_alerts_sum = self.dom_alerts_df[
+            ["message", "date", "previous_value", "new_value"]
+        ]
+        return dom_alerts_sum
 
 
 class Malware_Vulns:

src/pe_reports/pages.py

@@ -105,11 +105,14 @@ def masquerading(chevron_dict, start_date, end_date, org_uid):
     chevron_dict.update(
         {
             "domain_table": buildTable(Domain_Masq.summary(), ["table"], []),
+            "domain_alerts_table": buildTable(
+                Domain_Masq.alerts(), ["table"], [75, 25]
+            ),
             "suspectedDomains": Domain_Masq.count(),
-            "uniqueTlds": Domain_Masq.utlds(),
+            "domain_alerts": Domain_Masq.alert_count(),
         }
     )
-    return chevron_dict, Domain_Masq.df_mal
+    return chevron_dict, Domain_Masq.df_mal, Domain_Masq.alerts_sum()
 
 
 def mal_vuln(chevron_dict, start_date, end_date, org_uid):
@@ -287,7 +290,9 @@ def init(datestring, org_name, org_uid):
         chevron_dict, trending_start_date, start_date, end_date, org_uid
     )
 
-    chevron_dict, masq_df = masquerading(chevron_dict, start_date, end_date, org_uid)
+    chevron_dict, masq_df, dom_alert_sum = masquerading(
+        chevron_dict, start_date, end_date, org_uid
+    )
 
     chevron_dict, insecure_df, vulns_df, assets_df = mal_vuln(
         chevron_dict, start_date, end_date, org_uid
@@ -303,6 +308,7 @@ def init(datestring, org_name, org_uid):
         html,
         creds_sum,
         masq_df,
+        dom_alert_sum,
         insecure_df,
         vulns_df,
         assets_df,

src/pe_reports/report_generator.py

@@ -144,6 +144,7 @@ def generate_reports(datestring, output_directory):
                 source_html,
                 creds_sum,
                 masq_df,
+                dom_alerts_sum,
                 insecure_df,
                 vulns_df,
                 assets_df,
@@ -172,6 +173,7 @@ def generate_reports(datestring, output_directory):
             da_xlsx = f"{output_directory}/{org_code}/domain_alerts.xlsx"
             domWriter = pd.ExcelWriter(da_xlsx, engine="xlsxwriter")
             masq_df.to_excel(domWriter, sheet_name="Suspected Domains", index=False)
+            dom_alerts_sum.to_excel(domWriter, sheet_name="Domain Alerts", index=False)
             domWriter.save()
 
             # Create Suspected vulnerability Excel file

src/pe_reports/style.css

@@ -227,12 +227,14 @@
   margin: 1cm;
   margin-bottom: 0cm;
   background-image: url("assets/masq-background.png");
+
   @frame title {
     left: 3.5cm;
     width: 16.5cm;
     height: 2cm;
     top: 1cm;
   }
+
   @frame footer {
     -pdf-frame-content: footerContent;
     bottom: -0.4cm;
@@ -241,48 +243,66 @@
     height: 1.7cm;
     left: 1cm;
   }
+
   @frame header {
     -pdf-frame-content: headerContent;
     top: 0.5cm;
     left: 1cm;
     height: 6cm;
   }
+
   @frame kpi1 {
     left: 4.3cm;
     width: 5.45cm;
     height: 3cm;
     top: 2.7cm;
   }
+
   @frame kpi2 {
     left: 11.9cm;
     width: 5.45cm;
     height: 2.3cm;
     top: 2.7cm;
   }
-  @frame content1 {
+
+  @frame content0 {
+    -pdf-frame-content: summaryDom;
     left: 1.5cm;
     width: 19cm;
     height: 3cm;
     top: 5.3cm;
   }
-  /* @frame graphbox1 {
-    left: 1.3cm;
-    width: 9cm;
-    height: 4.5cm;
-    top: 7.8cm;
+
+  @frame content1 {
+    -pdf-frame-content: domainAlertsContent;
+    left: 1.5cm;
+    width: 19cm;
+    height: 3cm;
+    top: 7.3cm;
   }
-  @frame listbox {
-    left: 11.3cm;
-    width: 9cm;
-    height: 4.5cm;
-    top: 7.8cm;
-  } */
+
+  @frame table {
+    -pdf-frame-content: domainAlertsTable;
+    left: 1.5cm;
+    width: 19cm;
+    height: 9.3cm;
+    top: 8.5cm;
+  }
+
+  @frame content2 {
+    -pdf-frame-content: domainContent;
+    left: 1.5cm;
+    width: 19cm;
+    height: 3cm;
+    top: 17cm;
+  }
+
   @frame table {
     -pdf-frame-content: domainTable;
     left: 1.5cm;
     width: 19cm;
-    height: 18.6cm;
-    top: 7.8cm;
+    height: 9.3cm;
+    top: 18.2cm;
   }
 }