Update logging configuration

src/pe_mailer/message.py

@@ -5,22 +5,14 @@
 from email.mime.application import MIMEApplication
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
-import logging
 import os.path
 
 # cisagov Libraries
-from pe_reports import CENTRAL_LOGGING_FILE
+from pe_reports import app
 
 # Setup logging to central file
-logging.basicConfig(
-    filename=CENTRAL_LOGGING_FILE,
-    filemode="a",
-    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
-    datefmt="%m/%d/%Y %I:%M:%S",
-    level="INFO",
-)
-
-LOGGER = logging.getLogger(__name__)
+
+LOGGER = app.config["LOGGER"]
 
 
 class Message(MIMEMultipart):

src/pe_reports/__init__.py

@@ -41,12 +41,15 @@
 
 CENTRAL_LOGGING_FILE = "pe_reports_logging.log"
 
+# Setup Logging
+"""Set up logging and call the run_pe_script function."""
+
 logging.basicConfig(
     filename=CENTRAL_LOGGING_FILE,
     filemode="a",
     format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
     datefmt="%m/%d/%Y %I:%M:%S",
-    level=logging.INFO,
+    level="INFO",
 )
 
 app.config["LOGGER"] = logging.getLogger(__name__)

src/pe_reports/data/db_query.py

@@ -1,7 +1,6 @@
 """Query the PE PostgreSQL database."""
 
 # Standard Python Libraries
-import logging
 import sys
 
 # Third-Party Libraries
@@ -12,20 +11,12 @@
 from psycopg2.extensions import AsIs
 
 # cisagov Libraries
-from pe_reports import CENTRAL_LOGGING_FILE
+from pe_reports import app
 
 from .config import config
 
 # Setup logging to central file
-logging.basicConfig(
-    filename=CENTRAL_LOGGING_FILE,
-    filemode="a",
-    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
-    datefmt="%m/%d/%Y %I:%M:%S",
-    level="INFO",
-)
-
-LOGGER = logging.getLogger(__name__)
+LOGGER = app.config["LOGGER"]
 
 CONN_PARAMS_DIC = config()
 

src/pe_reports/report_generator.py

@@ -27,13 +27,13 @@
 from xhtml2pdf import pisa
 
 # cisagov Libraries
-from pe_reports import CENTRAL_LOGGING_FILE
+from pe_reports import CENTRAL_LOGGING_FILE, app
 
 from ._version import __version__
 from .data.db_query import connect, get_orgs
 from .pages import init
 
-LOGGER = logging.getLogger(__name__)
+LOGGER = app.config["LOGGER"]
 
 
 def embed(
@@ -258,7 +258,7 @@ def main():
         level=log_level.upper(),
     )
 
-    logging.info("Loading Posture & Exposure Report, Version : %s", __version__)
+    LOGGER.info("Loading Posture & Exposure Report, Version : %s", __version__)
 
     # Create output directory
     if not os.path.exists(validated_args["OUTPUT_DIRECTORY"]):

src/pe_source/cybersixgill.py

@@ -2,11 +2,10 @@
 
 # Standard Python Libraries
 from datetime import date, datetime, timedelta
-import logging
 import sys
 
 # cisagov Libraries
-from pe_reports import CENTRAL_LOGGING_FILE
+from pe_reports import app
 
 from .data.pe_db.db_query import (
     get_breaches,
@@ -29,16 +28,7 @@
     top_cves,
 )
 
-# Setup logging to central file
-logging.basicConfig(
-    filename=CENTRAL_LOGGING_FILE,
-    filemode="a",
-    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
-    datefmt="%m/%d/%Y %I:%M:%S",
-    level="INFO",
-)
-
-LOGGER = logging.getLogger(__name__)
+LOGGER = app.config["LOGGER"]
 
 # Set todays date formatted YYYY-MM-DD and the start_date 30 days prior
 TODAY = date.today()

src/pe_source/data/pe_db/config.py

@@ -2,7 +2,6 @@
 
 # Standard Python Libraries
 from configparser import ConfigParser
-import logging
 import os
 
 # Third-Party Libraries
@@ -11,22 +10,14 @@
 import shodan
 
 # cisagov Libraries
-from pe_reports import CENTRAL_LOGGING_FILE
+from pe_reports import app
 
 # Configuration
 REPORT_DB_CONFIG = files("pe_reports").joinpath("data/database.ini")
 
 
 # Setup logging to central file
-logging.basicConfig(
-    filename=CENTRAL_LOGGING_FILE,
-    filemode="a",
-    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
-    datefmt="%m/%d/%Y %I:%M:%S",
-    level="INFO",
-)
-
-LOGGER = logging.getLogger(__name__)
+LOGGER = app.config["LOGGER"]
 
 
 def shodan_api_init():

src/pe_source/data/pe_db/db_query.py

@@ -2,7 +2,6 @@
 
 # Standard Python Libraries
 from datetime import datetime
-import logging
 import sys
 
 # Third-Party Libraries
@@ -12,22 +11,19 @@
 import psycopg2.extras as extras
 
 # cisagov Libraries
+from pe_reports import app
 from pe_reports.data.config import config
 
 # Setup logging to central file
-logging.basicConfig(
-    filename="pe_reports_Logging.log",
-    format="%(asctime)-15s %(levelname)s %(message)s",
-    level="INFO",
-)
+LOGGER = app.config["LOGGER"]
 
 CONN_PARAMS_DIC = config()
 
 
 def show_psycopg2_exception(err):
     """Handle errors for PostgreSQL issues."""
     err_type, err_obj, traceback = sys.exc_info()
-    logging.error(
+    LOGGER.error(
         "Database connection error: %s on line number: %s", err, traceback.tb_lineno
     )
 
@@ -60,7 +56,7 @@ def get_orgs():
         cur.close()
         return pe_orgs
     except (Exception, psycopg2.DatabaseError) as error:
-        logging.error("There was a problem with your database query %s", error)
+        LOGGER.error("There was a problem with your database query %s", error)
     finally:
         if conn is not None:
             close(conn)
@@ -140,9 +136,9 @@ def insert_sixgill_alerts(df):
             tuples,
         )
         conn.commit()
-        logging.info("Successfully inserted/updated alert data into PE database.")
+        LOGGER.info("Successfully inserted/updated alert data into PE database.")
     except (Exception, psycopg2.DatabaseError) as error:
-        logging.error(error)
+        LOGGER.error(error)
         conn.rollback()
     cursor.close()
 
@@ -175,7 +171,7 @@ def insert_sixgill_mentions(df):
             ]
         ]
     except Exception as e:
-        logging.error(e)
+        LOGGER.error(e)
         df = df[
             [
                 "organizations_uid",
@@ -223,9 +219,9 @@ def insert_sixgill_mentions(df):
             tuples,
         )
         conn.commit()
-        logging.info("Successfully inserted/updated mention data into PE database.")
+        LOGGER.info("Successfully inserted/updated mention data into PE database.")
     except (Exception, psycopg2.DatabaseError) as error:
-        logging.error(error)
+        LOGGER.error(error)
         conn.rollback()
     cursor.close()
 
@@ -254,9 +250,9 @@ def insert_sixgill_breaches(df):
             tuples,
         )
         conn.commit()
-        logging.info("Successfully inserted/updated breaches into PE database.")
+        LOGGER.info("Successfully inserted/updated breaches into PE database.")
     except (Exception, psycopg2.DatabaseError) as error:
-        logging.info(error)
+        LOGGER.info(error)
         conn.rollback()
     cursor.close()
 
@@ -272,7 +268,7 @@ def get_breaches():
         cur.close()
         return pe_orgs
     except (Exception, psycopg2.DatabaseError) as error:
-        logging.error("There was a problem with your database query %s", error)
+        LOGGER.error("There was a problem with your database query %s", error)
     finally:
         if conn is not None:
             close(conn)
@@ -301,11 +297,11 @@ def insert_sixgill_credentials(df):
             tuples,
         )
         conn.commit()
-        logging.info(
+        LOGGER.info(
             "Successfully inserted/updated exposed credentials into PE database."
         )
     except (Exception, psycopg2.DatabaseError) as error:
-        logging.info(error)
+        LOGGER.info(error)
         conn.rollback()
     cursor.close()
 
@@ -332,9 +328,9 @@ def insert_sixgill_topCVEs(df):
             tuples,
         )
         conn.commit()
-        logging.info("Successfully inserted/updated top cve data into PE database.")
+        LOGGER.info("Successfully inserted/updated top cve data into PE database.")
     except (Exception, psycopg2.DatabaseError) as error:
-        logging.info(error)
+        LOGGER.info(error)
         conn.rollback()
     cursor.close()
 
@@ -358,14 +354,14 @@ def insert_shodan_data(dataframe, table, thread, org_name, failed):
             tpls,
         )
         conn.commit()
-        logging.info(
+        LOGGER.info(
             "{} Data inserted using execute_values() successfully - {}".format(
                 thread, org_name
             )
         )
     except Exception as e:
-        logging.error("{} failed inserting into {}".format(org_name, table))
-        logging.error("{} {} - {}".format(thread, e, org_name))
+        LOGGER.error("{} failed inserting into {}".format(org_name, table))
+        LOGGER.error("{} {} - {}".format(thread, e, org_name))
         failed.append("{} failed inserting into {}".format(org_name, table))
         conn.rollback()
     cursor.close()

src/pe_source/data/shodan/shodan_search.py

@@ -2,7 +2,6 @@
 
 # Standard Python Libraries
 import datetime
-import logging
 import time
 
 # Third-Party Libraries
@@ -11,23 +10,15 @@
 import shodan
 
 # cisagov Libraries
-from pe_reports import CENTRAL_LOGGING_FILE
+from pe_reports import app
 from pe_source.data.pe_db.db_query import (
     get_data_source_uid,
     get_ips,
     insert_shodan_data,
 )
 
 # Setup logging to central file
-logging.basicConfig(
-    filename=CENTRAL_LOGGING_FILE,
-    filemode="a",
-    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
-    datefmt="%m/%d/%Y %I:%M:%S",
-    level="INFO",
-)
-
-LOGGER = logging.getLogger(__name__)
+LOGGER = app.config["LOGGER"]
 
 
 def run_shodan_thread(api, org_chunk, thread_name):

src/pe_source/data/sixgill/source.py

@@ -1,12 +1,15 @@
 """Scripts for importing Sixgill data into PE Postgres database."""
 
 # Standard Python Libraries
-import logging
+
 
 # Third-Party Libraries
 import pandas as pd
 import requests
 
+# cisagov Libraries
+from pe_reports import app
+
 from .api import (
     alerts_count,
     alerts_list,
@@ -16,14 +19,7 @@
     org_assets,
 )
 
-# Setup logging to central file
-logging.basicConfig(
-    filename="pe_reports_Logging.log",
-    format="%(asctime)-15s %(levelname)s %(message)s",
-    level="INFO",
-)
-
-LOGGER = logging.getLogger(__name__)
+LOGGER = app.config["LOGGER"]
 
 
 def alias_organization(org_id):

src/pe_source/pe_scripts.py

@@ -34,22 +34,13 @@
 from schema import And, Schema, SchemaError, Use
 
 # cisagov Libraries
-from pe_reports import CENTRAL_LOGGING_FILE
+from pe_reports import CENTRAL_LOGGING_FILE, app
 
 from ._version import __version__
 from .cybersixgill import Cybersixgill
 from .shodan import Shodan
 
-# Setup logging to central file
-logging.basicConfig(
-    filename=CENTRAL_LOGGING_FILE,
-    filemode="a",
-    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
-    datefmt="%m/%d/%Y %I:%M:%S",
-    level="INFO",
-)
-
-LOGGER = logging.getLogger(__name__)
+LOGGER = app.config["LOGGER"]
 
 
 def run_pe_script(source, orgs_list, cybersix_methods):