Merge branch 'develop' into AL-datasource-to-dbschema

README.md

@@ -37,16 +37,11 @@ is gathered on the 1st and 15th of each month.
 
 ```console
 Usage:
-  pe-reports REPORT_DATE DATA_DIRECTORY OUTPUT_DIRECTORY [--db-creds-file=FILENAME] [--log-level=LEVEL]
+  pe-reports REPORT_DATE DATA_DIRECTORY OUTPUT_DIRECTORY [--log-level=LEVEL]
 
 Arguments:
   REPORT_DATE                   Date of the report, format YYYY-MM-DD.
-  DATA_DIRECTORY                The directory where the Excel data files are located.
-                                Organized by owner.
   OUTPUT_DIRECTORY              The directory where the final PDF reports should be saved.
-  -c --db-creds-file=FILENAME   A YAML file containing the Cyber
-                                Hygiene database credentials.
-                                [default: /secrets/database_creds.yml]
 Options:
   -h --help                     Show this message.
   -v --version                  Show version information.

setup.py

@@ -85,7 +85,7 @@ def get_version(version_file):
     packages=find_packages(where="src"),
     package_dir={"": "src"},
     package_data={
-        "pe_reports": ["data/shell/*.pptx", "data/*.config"],
+        "pe_reports": ["data/shell/*.pptx", "data/*.ini"],
         "pe_mailer": ["data/*"],
     },
     py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")],
@@ -96,6 +96,7 @@ def get_version(version_file):
         "chevron",
         "docopt",
         "glob2",
+        "importlib-resources",
         "matplotlib",
         "mongo-db-from-config@http://github.com/cisagov/mongo-db-from-config/tarball/develop",
         "openpyxl",
@@ -110,6 +111,7 @@ def get_version(version_file):
         "schema",
         "setuptools >= 24.2.0",
         "types-PyYAML",
+        "xhtml2pdf",
     ],
     extras_require={
         "test": [

src/pe_mailer/email_reports.py

@@ -76,15 +76,16 @@ def get_emails_from_request(request):
     for c in request["agency"]["contacts"]:
         if "type" not in c or "email" not in c or not c["email"].split():
             logging.warning(
-                f"Agency with ID {id} has a contact that is missing an email and/or type attribute!"
+                "Agency with ID %s has a contact that is missing an email and/or type attribute!",
+                id,
             )
 
     distro_emails = [c["email"] for c in contacts if c["type"] == "DISTRO"]
     technical_emails = [c["email"] for c in contacts if c["type"] == "TECHNICAL"]
 
     # There should be zero or one distro email
     if len(distro_emails) > 1:
-        logging.warning(f"More than one DISTRO email address for agency with ID {id}")
+        logging.warning("More than one DISTRO email address for agency with ID %s", id)
 
     # Send to the distro email, else send to the technical emails.
     to_emails = distro_emails
@@ -93,7 +94,7 @@ def get_emails_from_request(request):
 
     # At this point to_emails should contain at least one email
     if not to_emails:
-        logging.error(f"No emails found for ID {id}")
+        logging.error("No emails found for ID %s", id)
 
     return to_emails
 
@@ -262,7 +263,7 @@ def send_message(ses_client, message, counter=None):
     # Check for errors
     status_code = response["ResponseMetadata"]["HTTPStatusCode"]
     if status_code != 200:
-        logging.error(f"Unable to send message. Response from boto3 is: {response}")
+        logging.error("Unable to send message. Response from boto3 is: %s", response)
         raise UnableToSendError(response)
 
     if counter is not None:
@@ -310,7 +311,7 @@ def send_pe_reports(db, ses_client, pe_report_dir, to):
         cyhy_agencies = pe_requests.count()
         1 / cyhy_agencies
     except ZeroDivisionError:
-        logging.critical(f"No report data is found in {pe_report_dir}")
+        logging.critical("No report data is found in %s", pe_report_dir)
         sys.exit(1)
 
     agencies_emailed_pe_reports = 0
@@ -366,7 +367,8 @@ def send_pe_reports(db, ses_client, pe_report_dir, to):
                     )
                 except (UnableToSendError, ClientError):
                     logging.error(
-                        f"Unable to send Posture and Exposure report for agency with ID {id}",
+                        "Unable to send Posture and Exposure report for agency with ID %s",
+                        id,
                         exc_info=True,
                         stack_info=True,
                     )
@@ -389,30 +391,33 @@ def send_reports(pe_report_dir, db_creds_file, summary_to=None, test_emails=None
     try:
         db = db_from_config(db_creds_file)
     except OSError:
-        logging.critical(f"Database configuration file {db_creds_file} does not exist")
+        logging.critical("Database configuration file %s does not exist", db_creds_file)
         return 1
 
     except yaml.YAMLError:
         logging.critical(
-            f"Database configuration file {db_creds_file} does not contain valid YAML",
+            "Database configuration file %s does not contain valid YAML",
+            db_creds_file,
             exc_info=True,
         )
         return 1
     except KeyError:
         logging.critical(
-            f"Database configuration file {db_creds_file} does not contain the expected keys",
+            "Database configuration file %s does not contain the expected keys",
+            db_creds_file,
             exc_info=True,
         )
         return 1
     except pymongo.errors.ConnectionError:
         logging.critical(
-            f"Unable to connect to the database server in {db_creds_file}",
+            "Unable to connect to the database server in %s",
+            db_creds_file,
             exc_info=True,
         )
         return 1
     except pymongo.errors.InvalidName:
         logging.critical(
-            f"The database in {db_creds_file} does not exist", exc_info=True
+            "The database in %s does not exist", db_creds_file, exc_info=True
         )
         return 1
 

src/pe_reports/charts.py

@@ -1,12 +1,18 @@
 """Class methods for report charts."""
 
+# Standard Python Libraries
+import os
+
 # Third-Party Libraries
 import matplotlib.pyplot as plt
 from matplotlib.ticker import MaxNLocator
 
 # Factor to convert cm to inches
 CM_CONVERSION_FACTOR = 2.54
 
+# Get base directory to save images
+BASE_DIR = os.path.abspath(os.path.dirname(__file__))
+
 
 class Charts:
     """Build charts."""
@@ -60,7 +66,9 @@ def autopct(pct):
         plt.gcf().set_size_inches(
             width / CM_CONVERSION_FACTOR, height / CM_CONVERSION_FACTOR
         )
-        plt.savefig("assets/" + name, transparent=True, dpi=500, bbox_inches="tight")
+        plt.savefig(
+            BASE_DIR + "/assets/" + name, transparent=True, dpi=500, bbox_inches="tight"
+        )
         plt.clf()
 
     def stacked_bar(self):
@@ -85,7 +93,7 @@ def stacked_bar(self):
         plt.rc("axes", axisbelow=True)
         plt.grid(axis="y", zorder=0)
         plt.xticks(rotation=30, ha="right")
-        plt.savefig("assets/" + name, transparent=True, dpi=500)
+        plt.savefig(BASE_DIR + "/assets/" + name, transparent=True, dpi=500)
         plt.clf()
 
     def h_bar(self):
@@ -131,7 +139,9 @@ def h_bar(self):
                 )
 
         plt.gca().xaxis.set_major_locator(MaxNLocator(integer=True))
-        plt.savefig("assets/" + name, transparent=True, dpi=500, bbox_inches="tight")
+        plt.savefig(
+            BASE_DIR + "/assets/" + name, transparent=True, dpi=500, bbox_inches="tight"
+        )
         plt.clf()
 
     def line_chart(self):
@@ -159,8 +169,6 @@ def line_chart(self):
         plt.tight_layout()
 
         for i, j in df[df.columns[1]].items():
-            print(i)
-            print(j)
             ax.annotate(
                 j,
                 xy=(i, j),
@@ -170,5 +178,7 @@ def line_chart(self):
                 fontsize=7,
             )
 
-        plt.savefig("assets/" + name, transparent=True, dpi=500, bbox_inches="tight")
+        plt.savefig(
+            BASE_DIR + "/assets/" + name, transparent=True, dpi=500, bbox_inches="tight"
+        )
         plt.clf()

src/pe_reports/data/__init__.py

@@ -0,0 +1 @@
+"""The pe-reports data directory."""

src/pe_reports/data/config.py

@@ -2,9 +2,11 @@
 
 # Standard Python Libraries
 from configparser import ConfigParser
-from importlib.resources import files
 
-REPORT_DB_CONFIG = files("pe_reports").joinpath("data/dbconfig.config")
+# Third-Party Libraries
+from importlib_resources import files
+
+REPORT_DB_CONFIG = files("pe_reports").joinpath("data/database.ini")
 
 
 def config(filename=REPORT_DB_CONFIG, section="postgres"):

src/pe_reports/data/database.ini

@@ -0,0 +1,6 @@
+[postgres]
+host=
+database=
+user=
+password=
+port=

src/pe_reports/data/db_query.py

@@ -20,10 +20,10 @@
 
 def show_psycopg2_exception(err):
     """Handle errors for PostgreSQL issues."""
-    err_type, traceback = sys.exc_info()
-    line_n = traceback.tb_lineno
-    logging.error(f"\npsycopg2 ERROR: {err} on line number: {line_n}")
-    logging.error(f"psycopg2 traceback: {traceback} -- type: {err_type}")
+    err_type, err_obj, traceback = sys.exc_info()
+    logging.error(
+        "Database connection error: %s on line number: %s", err, traceback.tb_lineno
+    )
 
 
 def connect():
@@ -32,7 +32,7 @@ def connect():
     try:
         logging.info("Connecting to the PostgreSQL......")
         conn = psycopg2.connect(**CONN_PARAMS_DIC)
-        logging.info("Connection successful................\n")
+        logging.info("Connection successful......")
     except OperationalError as err:
         show_psycopg2_exception(err)
         conn = None
@@ -45,9 +45,8 @@ def close(conn):
     return
 
 
-def get_orgs():
+def get_orgs(conn):
     """Query organizations table."""
-    conn = connect()
     try:
         cur = conn.cursor()
         sql = """SELECT * FROM organizations"""
@@ -56,7 +55,7 @@ def get_orgs():
         cur.close()
         return pe_orgs
     except (Exception, psycopg2.DatabaseError) as error:
-        logging.error(f"There was a problem with your database query {error}")
+        logging.error("There was a problem with your database query %s", error)
     finally:
         if conn is not None:
             close(conn)
@@ -76,7 +75,7 @@ def query_hibp_view(org_uid, start_date, end_date):
         )
         return df
     except (Exception, psycopg2.DatabaseError) as error:
-        logging.error(f"There was a problem with your database query {error}")
+        logging.error("There was a problem with your database query %s", error)
     finally:
         if conn is not None:
             close(conn)
@@ -100,7 +99,7 @@ def query_domMasq(org_uid, start_date, end_date):
         )
         return df
     except (Exception, psycopg2.DatabaseError) as error:
-        logging.error(f"There was a problem with your database query {error}")
+        logging.error("There was a problem with your database query %s", error)
     finally:
         if conn is not None:
             close(conn)
@@ -133,7 +132,7 @@ def query_shodan(org_uid, start_date, end_date, table):
         )
         return df
     except (Exception, psycopg2.DatabaseError) as error:
-        logging.error(f"There was a problem with your database query {error}")
+        logging.error("There was a problem with your database query %s", error)
     finally:
         if conn is not None:
             close(conn)
@@ -158,7 +157,7 @@ def query_darkweb(org_uid, start_date, end_date, table):
         )
         return df
     except (Exception, psycopg2.DatabaseError) as error:
-        logging.error(f"There was a problem with your database query {error}")
+        logging.error("There was a problem with your database query %s", error)
     finally:
         if conn is not None:
             close(conn)
@@ -176,7 +175,7 @@ def query_darkweb_cves(table):
         )
         return df
     except (Exception, psycopg2.DatabaseError) as error:
-        logging.error(f"There was a problem with your database query {error}")
+        logging.error("There was a problem with your database query %s", error)
     finally:
         if conn is not None:
             close(conn)
@@ -206,7 +205,7 @@ def query_cyberSix_creds(org_uid, start_date, end_date):
         df["password_included"] = np.where(df["password"] != "", True, False)
         return df
     except (Exception, psycopg2.DatabaseError) as error:
-        logging.error(f"There was a problem with your database query {error}")
+        logging.error("There was a problem with your database query %s", error)
     finally:
         if conn is not None:
             close(conn)