publish 24/25 ncd code changes

CHANGELOG.txt

@@ -28,5 +28,5 @@ data/practice_maps.py - produces distinct pratice codes and practice pmaps
 
 Added three files in root_directory:
 input/CQRS_files/NCD_CQRS_Metadata_Mapping.csv - COntains a mapping table to fix field names
-input/CQRS_files/NCD_proposed_metadata.csv - Contains a table to produce indicator and measure mappings
+input/CQRS_files/NCD_metadata.csv - Contains a table to produce indicator and measure mappings
 output/CQRS_omitted_tracker.csv - Contains omitted practices for certain rules (does not count PCN codes)

README.md

@@ -33,7 +33,7 @@ root
 |   |---Archive
 |   |---CQRS_files
 |       |---NCD_CQRS_Metadata_Mapping.csv
-|       |---NCD_proposed_metadata.csv
+|       |---NCD_metadata.csv
 |
 |---Output
 |   |---Automated Checks

config.toml

@@ -0,0 +1,26 @@
+#NCD publication Parameters 
+
+[Setup]
+test_mode = "false"
+
+[Filepaths]
+root_directory  = "insert path to your root directory here"
+epcn_path       = "insert absolute path to your ePCN file here"
+
+
+[Connections]
+server          = "server name"
+database        = "database name"
+map_server      = "server name for mapping data"
+map_database    = "database name for mapping data"
+
+
+[Indicators]
+removal_indicator_list = []
+
+[Measures]
+bad_measure_list =  []
+
+[Pairs]
+remove_ind_pair.EHCH01 = "Numerator"
+remove_meas_pair.EHCH04 = "Denominator"

environment.yml

@@ -6,4 +6,4 @@ dependencies:
  - numpy == 1.21.5
  - openpyxl == 3.0.10
  - pyodbc == 4.0.34
-
+ - tomli == 2.0.1

ncdes/create_publication.py

@@ -2,15 +2,16 @@
 import pandas as pd
 import os
 
-from .data import sql_connection
-from .data import data_load
-from .data import amender
-from .processing import processing_steps
-from .processing import validation_check as check
+from ncdes.data import sql_connection
+from ncdes.data.data_load import *
+from ncdes.data import amender
+from ncdes.processing import processing_steps
+from ncdes.processing import validation_check as check
 
-from .output import outputs
+from ncdes.output import outputs
 
-from .utils.adhoc_fix import remove_problem_indicators, remove_problem_measures, remove_problem_indicator_measure_pairs
+from ncdes.utils.adhoc_fix import remove_problem_indicators, remove_problem_measures, remove_problem_indicator_measure_pairs
+from ncdes.utils.setup import check_saved_changes
 import warnings
 
 warnings.simplefilter(action="ignore", category=UserWarning)
@@ -31,16 +32,28 @@
 
 
 def main() -> None:
+
+    #check that all changes made are saved and have indeed been updated ready to run - important for test runs
+    check_saved_changes()
+
     print("\n"*3,"Loading config file")
-    config = data_load.load_json_config_file(".\\config.json")
+    config = get_config('config.toml')
+    config_conn = config['Connections']
+    config_fp = config['Filepaths']
+
+    test_run = config['Setup']['test_mode']
+    print("\n"*3,f"Test mode established as: {test_run}")
 
     print("Establishing SQL connection")
-    connection = sql_connection.connect(server=config["server"], database=config["database"])
+    connection = sql_connection.connect(server=config_conn["server"], database=config_conn["database"])
+    root_directory = config_fp["root_directory"]
 
-    root_directory = config["root_directory"]
+    output_directory = outputs.test_run_change_outputs_fldr(test_run, root_directory)
+    print(f"Selecting output folder: {output_directory}")
 
     print("Loading NCDes data")
-    ncdes_raw = data_load.load_csvs_in_directory_as_concat_dataframe(f"{root_directory}\\Input\\Current")
+    ncdes_raw = load_csvs_in_directory_as_concat_dataframe(f"{root_directory}\\Input\\Current")
+    ncdes_raw_archive = ncdes_raw.copy(deep=True)
 
     print("Amending CQRS data")
     ncdes_raw = amender.update_dataframe(ncdes_raw, config)
@@ -49,7 +62,7 @@ def main() -> None:
     ncdes_clean = processing_steps.clean_ncdes(ncdes_raw)
     reporting_period = processing_steps.get_formatted_reporting_end_date_from_ncdes_data(ncdes_clean)
 
-    geo_ccg_sql_str, geo_reg_sql_str, stp_sql_str, prac_sql_str = data_load.get_sql_query_strings(reporting_period)
+    geo_ccg_sql_str, geo_reg_sql_str, stp_sql_str, prac_sql_str = get_sql_query_strings(reporting_period)
 
     print("Loading SQL mapping data")
     geo_ccg_df = pd.read_sql(sql=geo_ccg_sql_str, con=connection)
@@ -61,7 +74,7 @@ def main() -> None:
     geo_ccg_df, geo_reg_df, stp_df = processing_steps.sql_df_cols_to_upper_case(geo_ccg_df, geo_reg_df, stp_df)
 
     print("Loading ePCN data")
-    raw_epcn = data_load.load_epcn_excel_table(epcn_path=config["epcn_path"])
+    raw_epcn = load_epcn_excel_table(epcn_path=config_fp["epcn_path"])
     epcn_df = processing_steps.epcn_transform(raw_epcn)
 
     print("Creating mapping table")
@@ -73,9 +86,6 @@ def main() -> None:
     print("Starting validation checks")
     check.run_all_column_has_expected_values_validations(NCDes_with_geogs, root_directory)
 
-    print("Replacing placeholder indicator and measures")
-    NCDes_with_geogs = processing_steps.replace_placeholders(NCDes_with_geogs)
-
     print("Applying suppression")
     NCDes_suppressed = processing_steps.suppress_output(
         main_table=NCDes_with_geogs,
@@ -89,43 +99,48 @@ def main() -> None:
         main_table_ind_code_col_name='IND_CODE'
     )
 
-    print("Removing problem indicators")
-    NCDes_problem_ind_rem = remove_problem_indicators.remove(NCDes_suppressed, ["NCD015", "NCD026"])
+    print("Removing problem indicators:")
+    print(str(config["Indicators"]["removal_indicator_list"]))
+    NCDes_problem_ind_rem = remove_problem_indicators.remove_indicators(NCDes_suppressed,  removal_indicator_list=config["Indicators"]["removal_indicator_list"])
     print("Removing problem measures")
-    NCDes_problem_meas_rem = remove_problem_measures.remove(NCDes_problem_ind_rem, ["Num Patients in Set"])
-    print("Removing problem indicator measure combos")
-    NCDes_problem_meas_rem = remove_problem_indicator_measure_pairs.remove(NCDes_problem_ind_rem, [("NCDMI198", "Numerator")])
-    NCDes_problem_meas_rem = remove_problem_indicator_measure_pairs.remove(NCDes_problem_meas_rem, [("NCD012", "Denominator")])
+    NCDes_problem_meas_rem = remove_problem_measures.remove_measures(NCDes_problem_ind_rem, bad_measure_list=config["Measures"]["bad_measure_list"])
+
+    print("Removing problem indicator-measure combinations if required")
+    NCDes_problem_meas_rem = remove_problem_indicator_measure_pairs.remove_pairs(NCDes_problem_ind_rem, bad_indicator_measure_list=config["Pairs"]['remove_ind_pair'])
+    NCDes_problem_meas_rem = remove_problem_indicator_measure_pairs.remove_pairs(NCDes_problem_meas_rem, bad_indicator_measure_list=config["Pairs"]['remove_meas_pair'])
 
     print("Joining ruleset ID to copy of output data for ruleset-specific outputs")
     NCDes_with_rulesets = processing_steps.merge_data_with_ruleset_id(NCDes_problem_meas_rem, root_directory)
-
+    
     print("Saving main output")
-    outputs.save_NCDes_main_to_csv(NCDes_problem_meas_rem, root_directory)
+    outputs.save_NCDes_main_to_csv(NCDes_problem_meas_rem, output_directory)
 
     print("Zipping main output")
-    outputs.save_NCDes_main_to_zip(NCDes_problem_meas_rem, root_directory)
-
-    print("Saving excel output")
-    outputs.save_NCDes_main_to_excel(NCDes_problem_meas_rem, root_directory, server=config["server"], database=config["database"])
-
-    print("Saving trend monitor")
-    outputs.save_trendmonitor(NCDes_problem_meas_rem, root_directory)
+    outputs.save_NCDes_main_to_zip(NCDes_problem_meas_rem, output_directory)
 
     print("Saving outputs split by ruleset")
-    outputs.save_NCDes_by_ruleset_to_csvs(NCDes_with_rulesets, root_directory)
+    outputs.save_NCDes_by_ruleset_to_csvs(NCDes_with_rulesets, output_directory)
 
     print("Zipping outputs split by ruleset")
-    outputs.save_NCDes_by_ruleset_to_zip(NCDes_with_rulesets, root_directory)
+    outputs.save_NCDes_by_ruleset_to_zip(NCDes_with_rulesets, output_directory)
+
+    print("Saving trend monitor")
+    if test_run == True:
+        print("Test mode: skipping trend monitor")
+    else:
+        outputs.save_trendmonitor(NCDes_problem_meas_rem, root_directory)
+
+    print("Saving excel output")
+    outputs.save_NCDes_main_to_excel(NCDes_problem_meas_rem, root_directory, server=config_conn["server"], database=config_conn["database"], test_run=test_run)
 
     print("Archiving input")
-    outputs.archive_input_as_csv(ncdes_raw, root_directory)
+    outputs.archive_input_as_csv(ncdes_raw_archive, output_directory)
 
     print("Deleting input files from input folder")
     outputs.remove_files_from_input_folder(path=f"{root_directory}Input\\Current\\")
 
     print("Job complete")
-    outputs.open_outputs(NCDes_problem_meas_rem, root_directory)
+    outputs.open_outputs(NCDes_problem_meas_rem, output_directory)
 
 
 if __name__ == "__main__":

ncdes/data/amender.py

@@ -2,22 +2,23 @@
 from datetime import datetime as datetime2
 import pyodbc as dbc
 import os
-from ..data.practice_maps import *
+from ncdes.data.practice_maps import *
+from ncdes.output.outputs import *
 
 
 def measure_and_indicator_mappings(root: str) -> pd.DataFrame:
     """Gets the measure and indicator mappings from proposed metadata"""
 
-    df = pd.read_csv(f"{root}\\Input\\CQRS_files\\NCD_proposed_metadata.csv")
+    df = pd.read_csv(f"{root}\\Input\\CQRS_files\\NCD_metadata.csv", encoding='unicode_escape')
     df = df.sort_values("INDICATOR_ID", ascending=True)
 
     #Indicator mappings
     ind_dict = df[["INDICATOR_ID","INDICATOR_DESCRIPTION","Ruleset"]]
     ind_dict = ind_dict.drop_duplicates()
     ind_dict["Payment or Management Information (MI)"] = "MI"
 
-    ind_dict.loc[(ind_dict["INDICATOR_ID"] == "NCD003") | (ind_dict["INDICATOR_ID"] == "NCD004")|
-                (ind_dict["INDICATOR_ID"] == "NCD120"), "Payment or Management Information (MI)"] = "Payment"
+    ind_dict.loc[(ind_dict["INDICATOR_ID"] == "HI03") |
+                (ind_dict["INDICATOR_ID"] == "CAN02"), "Payment or Management Information (MI)"] = "Payment"
 
     ind_dict = ind_dict.rename(columns={"INDICATOR_ID": "Indicator ID",
                                         "INDICATOR_DESCRIPTION":"Indicator Description",
@@ -28,9 +29,7 @@ def measure_and_indicator_mappings(root: str) -> pd.DataFrame:
     meas_dict = meas_dict.drop_duplicates()
     meas_dict = meas_dict.rename(columns={"CQRS short code":"MEASURE ID"})
     meas_dict = meas_dict[meas_dict["MEASURE ID"] != "n/a"]
-    meas_dict["MEASURE_DESCRIPTION"] = meas_dict["MEASURE_DESCRIPTION"].replace({"Numerator":"Numerator count for indicator", "Denominator":"Denominator count for indicator"})
-    meas_dict.loc[len(meas_dict)] = ["Management Information", "Management information count for indicator", ""]
-    meas_dict.loc[len(meas_dict)] = ["Num Patients in Set", "Copy of denominator which is sometimes provided by CQRS and needs removing","Exclusion"]    
+    meas_dict["MEASURE_DESCRIPTION"] = meas_dict["MEASURE_DESCRIPTION"].replace({"Numerator":"Numerator count for indicator", "Denominator":"Denominator count for indicator"})  
 
     #Adding numerator fix from ncdes review
     meas_dict.loc[meas_dict["MEASURE_TYPE"] == "Numerator", "MEASURE ID"] = "Numerator"
@@ -41,7 +40,7 @@ def measure_and_indicator_mappings(root: str) -> pd.DataFrame:
 def remove_codes(df: pd.DataFrame, config:dict) -> pd.DataFrame:
     """removes any data for PCN codes and only leaves practice codes"""
 
-    codes = code_map(config["server"], config["database"])
+    codes = code_map(config['Connections']["server"], config['Connections']["database"])
     codes = codes.rename(columns={"CODE":"ORG_CODE"})
     df = codes.merge(df, on="ORG_CODE", how="left")
 
@@ -62,13 +61,13 @@ def update_dataframe(df: pd.DataFrame, config: dict) -> pd.DataFrame:
         return df
 
     #print(os.getcwd())
-    root = config["root_directory"]
+    root = config['Filepaths']["root_directory"]
     field_map = pd.read_csv(f"{root}\\Input\\CQRS_files\\NCD_CQRS_Metadata_Mapping.csv")
 
     #Drop all U codes
     df = remove_codes(df, config)
 
-    prac_sup_df = practice_supplier_map(config["map_server"], config["map_database"])
+    prac_sup_df = practice_supplier_map(config['Connections']["map_server"], config['Connections']["map_database"])
     prac_sup_df = prac_sup_df.rename(columns={"GP_Code":"ORG_CODE"})
 
 
@@ -80,11 +79,14 @@ def update_dataframe(df: pd.DataFrame, config: dict) -> pd.DataFrame:
     df = df.drop(["FIELD_NAME"], axis=1)
     df = df.rename(columns={"SHORT_NAME":"FIELD_NAME"})
 
+    #check if year folder exists, if not create one 
+    check_and_create_folder(f"{root}\\Output\\CQRS_omitted_tracker\\")
+
     #Duplicated rows, logged
     not_last_sub = df[df["LAST_SUBMISSION"] == "N"]
     not_last_sub = not_last_sub[["ORG_CODE","SUBMIT_DATE","SUBMIT_TIME"]].merge(prac_sup_df, on="ORG_CODE", how="left")
     not_last_sub = not_last_sub.fillna("Unknown")
-    root_directory = config["root_directory"]
+    root_directory = config['Filepaths']["root_directory"]
     not_last_sub = not_last_sub.drop_duplicates()
     not_last_sub.to_csv(f"{root_directory}\\Output\\CQRS_omitted_tracker\\CQRS_omits.csv", index=False)
 

ncdes/data/data_load.py

@@ -1,12 +1,35 @@
 import os
 import pandas as pd
-import json
-from ..data.amender import *
+import tomli
+from datetime import datetime
+from pathlib import PurePath
+#import json
+from ncdes.data.amender import *
 
-def load_json_config_file(path):
-    with open(path) as f:
-        file = json.load(f)
-    return file
+
+
+# def load_json_config_file(path):
+#     with open(path) as f:
+#         file = json.load(f)
+#     return file
+
+
+def get_config(loc:str) -> dict:
+    """
+    Function Actions:
+    - Reads the config toml file containing user inputs.
+    - Returns a dictionary of parameters.
+    """   
+
+    filename = PurePath(loc)
+    assert os.path.isfile(filename), "config.toml file could not be found. Please make sure the toml file is saved in the main repo location and the name is 'config.toml' as expected."
+
+    with open(filename, "rb") as f:
+        config_file = tomli.load(f)
+
+    print('Config file read in.')
+
+    return config_file
 
 
 def load_csvs_in_directory_as_concat_dataframe(directory):
@@ -21,7 +44,7 @@ def load_csvs_in_directory_as_concat_dataframe(directory):
 
     holder = []
     for phase in phase_file_names:
-        df = pd.read_csv(directory + "\\" + phase)
+        df = pd.read_csv(directory + "\\" + phase, quotechar='"', delimiter=',', quoting = csv.QUOTE_ALL)
         holder.append(df)
 
     output_df = pd.concat(holder, ignore_index=True)