diff --git a/ncdes/create_publication.py b/ncdes/create_publication.py index 751f144..2ab35b1 100644 --- a/ncdes/create_publication.py +++ b/ncdes/create_publication.py @@ -65,18 +65,19 @@ def main() -> None: print("Replacing placeholder indicator and measures") NCDes_with_geogs = processing_steps.replace_placeholders(NCDes_with_geogs) - print("Suppressing PCA values") - NCDes_with_geogs = processing_steps.suppress_PCA_values( + print("Applying suppression") + NCDes_with_geogs = processing_steps.suppress_output( main_table=NCDes_with_geogs, + root_directory=root_directory, measure_dict_meas_col_name='MEASURE ID', - measure_dict_meas_type_col_name = 'MEASURE_TYPE', + measure_dict_meas_type_col_name='MEASURE_TYPE', + measure_dict_meas_description_col_name='MEASURE_DESCRIPTION', main_table_meas_col_name='MEASURE', main_table_value_col_name='VALUE', main_table_prac_code_col_name='PRACTICE_CODE', - main_table_ind_code_col_name='IND_CODE', - root_directory = root_directory + main_table_ind_code_col_name='IND_CODE' ) - + print("Joining ruleset ID to copy of output data for ruleset-specific outputs") NCDes_with_geogs_and_rulesets = processing_steps.merge_mapped_data_with_ruleset_id(NCDes_with_geogs, root_directory) diff --git a/ncdes/processing/processing_steps.py b/ncdes/processing/processing_steps.py index 6a54a1b..74e0357 100644 --- a/ncdes/processing/processing_steps.py +++ b/ncdes/processing/processing_steps.py @@ -149,182 +149,160 @@ def replace_placeholders(ncdes_table): return ncdes_table +#--------------------------------------------------- Suppression logic start -----------------------------------------------------------------# -# Below functions deal with suppression -def suppress_PCA_values( - main_table, - root_directory, +def split_dataframe( + merged_table, measure_dict_meas_col_name='MEASURE ID', measure_dict_meas_type_col_name = 'MEASURE_TYPE', - main_table_meas_col_name='MEASURE', - main_table_value_col_name='VALUE', - main_table_prac_code_col_name='PRACTICE_CODE', - main_table_ind_code_col_name='IND_CODE', + main_table_ind_code_col_name='IND_CODE' ): """ - This function will suppress sensitive PCA data from the input table - by applying the below rule: - - Denominator rule: If an indicator's denominator value for any given practice is 0 - - AND - - PCA rule: If only one PCA is non-zero - - THEN - - Suppress all PCA values for indicator + Split dataframe according to how many PCAs each indicator has + + Input: + Raw df merged with measure dictionary + + Output: + merged_df_1_PCA: input df containing only indicators/practices with one 1 PCA + merged_df_2_or_more_PCA: input df containing only indicators/practices with 2 or more PCAs + merged_df_0_PCA: input df containing only indicators/practices with 0 PCAs """ - # Load in measure dictionary - measure_dict = data_load.load_indicator_and_measure_data_dictionaries(root_directory)[1] - - # Format measures dictionary - measure_dict_PCA = filter_to_only_contain_PCAs( - input_df=measure_dict, - measure_type_col_name=measure_dict_meas_type_col_name - ) - # Merge in the PCA measure type data with NCDes data so we can identify the PCA rows - main_with_PCA = merge_main_df_and_measures_dict( - main_table = main_table, - measure_dict_PCA = measure_dict_PCA, - main_table_meas_col_name=main_table_meas_col_name, - measure_dict_meas_col_name=measure_dict_meas_col_name - ) - - # Filter the new table to only contain practices/indicator combinations that staisfy the denominator condition - main_denom_condition = filter_main_table_by_denom_condition( - main_table_with_PCA=main_with_PCA, - main_table_meas_col_name=main_table_meas_col_name, - main_table_value_col_name=main_table_value_col_name, - main_table_prac_code_col_name=main_table_prac_code_col_name, - main_table_ind_code_col_name=main_table_ind_code_col_name - ) - # print(main_denom_condition) - # Get rid of all rows from the filtered table that aren't PCAs - main_prepivot = filter_to_only_contain_PCAs( - input_df=main_denom_condition, - measure_type_col_name=measure_dict_meas_type_col_name - ) - - # Pivot data - pivoted = pivot_measures_col_fill_na( - main_table_prepivot=main_prepivot, - main_table_value_col_name=main_table_value_col_name, - main_table_prac_code_col_name=main_table_prac_code_col_name, - main_table_ind_code_col_name=main_table_ind_code_col_name, - main_table_meas_col_name=main_table_meas_col_name - ) + # Create count dict that maps IND to number of PCAs + unique_meas_ind_table = merged_table[[main_table_ind_code_col_name, measure_dict_meas_col_name,measure_dict_meas_type_col_name]].drop_duplicates() + PCA_count_dict = {} + for ind in set(unique_meas_ind_table[main_table_ind_code_col_name]): + ind_sub_df = unique_meas_ind_table[unique_meas_ind_table[main_table_ind_code_col_name] == ind] + PCA_sum = list(ind_sub_df[measure_dict_meas_type_col_name]).count('PCA') + PCA_count_dict[ind] = PCA_sum - # Get problematic practice/indidicators - prac_ind_to_suppress = get_problem_practice_indicator_pairs( - pivoted = pivoted, - main_table_prac_code_col_name=main_table_prac_code_col_name, - main_table_ind_code_col_name=main_table_ind_code_col_name - ) - #print(main_with_PCA) + # Split main table into two sub tables depending on which PCA condition the table meets + one_PCA_inds = [key for key, val in PCA_count_dict.items() if val == 1] + two_plus_PCA_inds = [key for key, val in PCA_count_dict.items() if val > 1] + zero_PCA_inds = [key for key, val in PCA_count_dict.items() if val == 0] - # Complete suppression of prblematic PCAs - main_PCA_suppressed = suppress( - prac_ind_to_suppress=prac_ind_to_suppress, - main_with_PCA=main_with_PCA, - main_table_prac_code_col_name=main_table_prac_code_col_name, - main_table_ind_code_col_name=main_table_ind_code_col_name, - measure_dict_meas_type_col_name=measure_dict_meas_type_col_name - ) + merged_df_1_PCA = merged_table[merged_table[main_table_ind_code_col_name].isin(one_PCA_inds)] + merged_df_2_plus_PCA = merged_table[merged_table[main_table_ind_code_col_name].isin(two_plus_PCA_inds)] + merged_df_0_PCA = merged_table[merged_table[main_table_ind_code_col_name].isin(zero_PCA_inds)] - return main_PCA_suppressed - -def filter_to_only_contain_PCAs( - input_df, - measure_type_col_name='MEASURE_TYPE' + return merged_df_1_PCA, merged_df_2_plus_PCA, merged_df_0_PCA + +def denom_condition_1_PCA( + merged_df_1_PCA, + main_table_meas_col_name='MEASURE', + main_table_value_col_name='VALUE', + main_table_prac_code_col_name='PRACTICE_CODE', + main_table_ind_code_col_name='IND_CODE' ): """ + Denominator condition for indicators/practices with one PCA: denominator must be less that <2 + Input: - input_df: Dataframe we wish to filter - measure_type_col_name: The name of the column that holds the measure type info in the input df + merged_df_1_PCA + Output: - The input dataframe with only rows that have measure type == PCA - """ - return input_df[input_df[measure_type_col_name] == 'PCA'] + Filtered out all indicators from input that don't meet the denominator condition for indicators with one PCA + """ + # Filter out which of these indicators don't meet the denom condition for each practice + denom_condition_met_subset = merged_df_1_PCA[ + (merged_df_1_PCA[main_table_meas_col_name] == 'Denominator') & (merged_df_1_PCA[main_table_value_col_name] < 2) + ] + # Merge in above df with input to ensure that we retain other measures and not just denominators + denom_condition_met_whole = pd.merge( + merged_df_1_PCA.reset_index(), + denom_condition_met_subset[[main_table_prac_code_col_name, main_table_ind_code_col_name]], + on=[main_table_prac_code_col_name, main_table_ind_code_col_name], + how='inner' + ) + + return denom_condition_met_whole -def merge_main_df_and_measures_dict( - main_table, - measure_dict_PCA, - main_table_meas_col_name='MEASURE', - measure_dict_meas_col_name='MEASURE ID' +def suppress_1_PCA( + denom_condition_met_1_PCA, + merged_df_1_PCA, + measure_dict_meas_type_col_name='MEASURE_TYPE', + main_table_value_col_name='VALUE', + main_table_prac_code_col_name='PRACTICE_CODE', + main_table_ind_code_col_name='IND_CODE', + main_table_meas_col_name='MEASURE' ): """ - Input: - main_table: The main input table you are trying to suppress - measures dictionary with only PCA values - main_table_meas_col_name: The column name for the measure column in the main table - measure_dict_meas_col_name: The column name for the measure column in the measure dictionary table - - Returns: - The main table with a new column that identifies which rows are PCA's + Test which indicators have a PCA > 0 in 'denom_condition_met_1_PCA' df and if this condition is met, + suppress the PCAs and their associated denominators """ - # Merge in measure type data with NCDes data - main_table_measure_typed = pd.merge( - main_table, - measure_dict_PCA, - left_on=main_table_meas_col_name, - right_on=measure_dict_meas_col_name, - how='left' - ) + # Drop all non-PCA rows + PCA_filtered = denom_condition_met_1_PCA[denom_condition_met_1_PCA[measure_dict_meas_type_col_name] == 'PCA'] + + # Get a table with only non-zero PCAs + table_for_suppression_a = PCA_filtered[PCA_filtered[main_table_value_col_name] > 0] - return main_table_measure_typed + # Get unique identifier to isolate PCAs that need to be suppressed; list these indicator/PCA combinations and the respective indicator/denominator combinations + ZIP = zip(table_for_suppression_a[main_table_prac_code_col_name], table_for_suppression_a[main_table_ind_code_col_name], table_for_suppression_a[main_table_meas_col_name]) + unique_identifier_a = [] + for val in ZIP: + unique_identifier_a.append(val) + unique_identifier_a.append(list(val[0:2]) + ['Denominator']) + + ## Loop through using unqie identifiers and suppress the PCAs and denominators + PCA_1_out = merged_df_1_PCA.copy() + for ind in unique_identifier_a: + PCA_1_out.loc[(PCA_1_out[main_table_prac_code_col_name] == ind[0]) & (PCA_1_out[main_table_ind_code_col_name] == ind[1]) & (PCA_1_out[main_table_meas_col_name] == ind[2]) , main_table_value_col_name] = '*' + + return PCA_1_out -def filter_main_table_by_denom_condition( - main_table_with_PCA, +def denom_condition_2_plus_PCA( + merged_df_2_plus_PCA, main_table_meas_col_name='MEASURE', main_table_value_col_name='VALUE', main_table_prac_code_col_name='PRACTICE_CODE', main_table_ind_code_col_name='IND_CODE' ): """ - Filters the main table to only contain the practices/indicator rows that - satisfy the denominator condition + Denominator condition for indicators/practices with two or more PCAs: Denominator must be equal to 0 Input: - main_table_with_PCA: main table we are trying to suppress with associated PCA values merged in + merged_df_2_plus_PCA - Return: - The main table now filtered to only contain practices/indicator combinations that staisfy the - denominator condition + Output: + Filtered out all indicators from input that don't meet the denominator condition for indicators with 2 or more PCAs + """ - main_table_filtered_only_denoms = main_table_with_PCA[ - (main_table_with_PCA[main_table_meas_col_name] == 'Denominator') & (main_table_with_PCA[main_table_value_col_name] == 0) + # Filter to only get denominator/practice rows that meet the denominator condition + merged_filtered = merged_df_2_plus_PCA[ + (merged_df_2_plus_PCA[main_table_meas_col_name] == 'Denominator') & (merged_df_2_plus_PCA[main_table_value_col_name] == 0) ] - - main_table_full_filtered = pd.merge( - main_table_with_PCA, - main_table_filtered_only_denoms[[main_table_prac_code_col_name, main_table_ind_code_col_name]], + + # Merge in above df with input to ensure that all items for practice/indicator combinations that meet the denominator criterion are identified + full_table_filtered = pd.merge( + merged_df_2_plus_PCA, + merged_filtered[[main_table_prac_code_col_name, main_table_ind_code_col_name]], on=[main_table_prac_code_col_name, main_table_ind_code_col_name], how='inner' ) - return main_table_full_filtered + return full_table_filtered -def pivot_measures_col_fill_na( - main_table_prepivot, +def pivot_measures_col( + only_PCAs, main_table_value_col_name='VALUE', main_table_prac_code_col_name='PRACTICE_CODE', main_table_ind_code_col_name='IND_CODE', main_table_meas_col_name='MEASURE' ): """ - The test which rows meet the 'PCA rule' + Pivot dataframe in preparation for testing the PCA condition for indicators with 2+ PCAs """ - pivoted_and_filled = pd.pivot_table( - main_table_prepivot, - values = main_table_value_col_name, - index=[main_table_prac_code_col_name, main_table_ind_code_col_name], - columns = main_table_meas_col_name).reset_index().fillna(0) + pivoted = pd.pivot_table( + only_PCAs, + values = main_table_value_col_name, + index=[main_table_prac_code_col_name, main_table_ind_code_col_name], + columns = main_table_meas_col_name + ).reset_index().fillna(0) - return pivoted_and_filled + return pivoted def get_problem_practice_indicator_pairs( pivoted, @@ -332,69 +310,223 @@ def get_problem_practice_indicator_pairs( main_table_ind_code_col_name='IND_CODE' ): """ - Itterates through the pivoted dataframe and checks which rows are complient with the 'PCA rule' - it adds the practice code/indicator code combinations for these probelmatic rows to a list which it - then returns + PCA Condition for indicators with 2+ PCAs: The sum of all PCAs is equal to the value of any one PCA + + Function iterates through the pivoted dataframe and checks which rows are compliant with the + 'PCA Condition for indicators with 2+ PCAs'. It adds the practice code/indicator code combinations + for these rows to a list which it then returns. """ list_of_prac_ind_to_supress = [] - + # Iterate through pivoted dataframe rows where each row represents all the PCAs for a practice/indicator + # pair that meets the denominator condition for i, row in pivoted.iterrows(): + # Get temp row with only PCA cols row_reduced = row[2:] - # If below is true it means theres only one non-zero PCA for that prac/ind combo + # If below is true it means there's only one non-zero PCA for that practice/indicator combination (number of zero count PCAs = number of all PCAs-1) + # and therefore one PCA value = sum of PCA values for that practice/indicator combination if Counter(row_reduced)[0.0] == len(row_reduced)-1: + # Append problem indicators to the list list_of_prac_ind_to_supress.append( (row[main_table_prac_code_col_name], row[main_table_ind_code_col_name]) ) return list_of_prac_ind_to_supress -def suppress( - prac_ind_to_suppress, - main_with_PCA, +def suppress_2_plus_PCA( + list_of_prac_ind_to_supress, + merged_df_2_plus_PCA, main_table_prac_code_col_name, main_table_ind_code_col_name, - measure_dict_meas_type_col_name + measure_dict_meas_type_col_name, + measure_dict_meas_col_name, + measure_dict_meas_description_col_name, + main_table_value_col_name ): + """ + Retrieves indexes to suppress from the 'get_indexes_to_suppress' fn; uses these to locate the + problem values and inserts a * + """ indexes_to_suppress = get_indexes_to_suppress( - prac_ind_to_suppress=prac_ind_to_suppress, - main_with_PCA=main_with_PCA, + prac_ind_to_suppress=list_of_prac_ind_to_supress, + merged_df_2_plus_PCA=merged_df_2_plus_PCA, main_table_prac_code_col_name=main_table_prac_code_col_name, main_table_ind_code_col_name=main_table_ind_code_col_name, measure_dict_meas_type_col_name=measure_dict_meas_type_col_name ) - main_with_PCA.loc[indexes_to_suppress, 'VALUE'] = '*' - - - return main_with_PCA.drop(columns=["MEASURE ID", "MEASURE_DESCRIPTION", "MEASURE_TYPE"]) + merged_df_2_plus_PCA.loc[indexes_to_suppress, main_table_value_col_name] = '*' + return merged_df_2_plus_PCA.drop(columns=[measure_dict_meas_col_name, measure_dict_meas_description_col_name, measure_dict_meas_type_col_name]) def get_indexes_to_suppress( prac_ind_to_suppress, - main_with_PCA, + merged_df_2_plus_PCA, main_table_prac_code_col_name, main_table_ind_code_col_name, measure_dict_meas_type_col_name ): + """ + Gets a list of indexes to suppress from the rows that meet the suppression + condition + """ index_to_suppress = [] for practice, indicator in prac_ind_to_suppress: - - subset_to_suppress = main_with_PCA[(main_with_PCA[main_table_prac_code_col_name] == practice) - & (main_with_PCA[main_table_ind_code_col_name] == indicator) - & (main_with_PCA[measure_dict_meas_type_col_name] == 'PCA')] - + # Retrieves a sub dataframe whose rows meet the conditions for suppression + subset_to_suppress = merged_df_2_plus_PCA[ + (merged_df_2_plus_PCA[main_table_prac_code_col_name] == practice) & + (merged_df_2_plus_PCA[main_table_ind_code_col_name] == indicator) & + (merged_df_2_plus_PCA[measure_dict_meas_type_col_name] == 'PCA') + ] + # Adds the indexes for these rows to a list list_subset_to_suppress = [x for x in subset_to_suppress.index] - index_to_suppress.extend(list_subset_to_suppress) return index_to_suppress +def suppress_output( + main_table, + root_directory, + measure_dict_meas_col_name='MEASURE ID', + measure_dict_meas_type_col_name='MEASURE_TYPE', + measure_dict_meas_description_col_name='MEASURE_DESCRIPTION', + main_table_meas_col_name='MEASURE', + main_table_value_col_name='VALUE', + main_table_prac_code_col_name='PRACTICE_CODE', + main_table_ind_code_col_name='IND_CODE' +): + """ + Applies the following rules to the fully processed dataframe: + + 1. For all fractional indicators: omit exclusion counts from publications (this is already implemented for LDHC, but would be a change for all other publications e.g. QOF, NCDES, INLIQ...) + + 2. For fractional indicators with 1 PCA specified: where denominator <2 and PCA > 0, suppress the PCA and the denominator for that indicator + + 3. For fractional indicators with >1 PCA specified: where denominator = 0 and the sum of all PCAs is equal to the value of any one PCA, suppress all the PCAs for that indicator + + + Function can be broken down into 4 main parts: + + a. Ingestion/pre-processing + b. Dealing with suppression relating to point '2.' above + c. Dealing with suppression realting to point '3.' above + d. Recombining dataframes + """ + # -------------------------------------------------- a -------------------------------------------------------- # + # Load in measure dictionary + measure_dict = data_load.load_indicator_and_measure_data_dictionaries(root_directory)[1] + + + # Merge in measure type data to main table + merged_table = pd.merge( + main_table, + measure_dict, + left_on=main_table_meas_col_name, + right_on=measure_dict_meas_col_name, + how='left' + ) + + # Split dataframe so different suppression rules can be applied to each split + merged_df_1_PCA, merged_df_2_plus_PCA, merged_df_0_PCA = split_dataframe(merged_table) + + # -------------------------------------------------- b -------------------------------------------------------- # + # filter relevant merged df according to denominator condition for indicators with one PCA + denom_condition_met_1_PCA = denom_condition_1_PCA(merged_df_1_PCA) + # Apply suppression to indicators with one PCA that meet the denominator condition and PCA condition + PCA_1_out = suppress_1_PCA( + denom_condition_met_1_PCA, + merged_df_1_PCA, + measure_dict_meas_type_col_name, + main_table_value_col_name, + main_table_prac_code_col_name, + main_table_ind_code_col_name, + main_table_meas_col_name + ) + + + + # -------------------------------------------------- c -------------------------------------------------------- # + # filter relevant merged df according to denominator condition for indicators with 2 or more PCAs + denom_condition_met_2_plus_PCA = denom_condition_2_plus_PCA( + merged_df_2_plus_PCA, + main_table_meas_col_name=main_table_meas_col_name, + main_table_value_col_name=main_table_value_col_name, + main_table_prac_code_col_name=main_table_prac_code_col_name, + main_table_ind_code_col_name=main_table_ind_code_col_name + ) + # Filter the above df to only contain PCA values; i.e. isolate PCAs for practice/indicator combinations meeting denominator suppression criterion + only_PCAs = denom_condition_met_2_plus_PCA[denom_condition_met_2_plus_PCA[measure_dict_meas_type_col_name] == 'PCA'] + + + # Pivot the filtered table so that we can apply summation logic + pivoted = pivot_measures_col( + only_PCAs, + main_table_value_col_name=main_table_value_col_name, + main_table_prac_code_col_name=main_table_prac_code_col_name, + main_table_ind_code_col_name=main_table_ind_code_col_name, + main_table_meas_col_name=main_table_meas_col_name + ) + + # Get list of indicators that meet the PCA condition for indicators with more than one PCA (and meet the above + # denominator condition) + list_of_prac_ind_to_supress = get_problem_practice_indicator_pairs( + pivoted, + main_table_prac_code_col_name=main_table_prac_code_col_name, + main_table_ind_code_col_name=main_table_ind_code_col_name + ) + + # Apply suppression to indicators with 2 or more PCAs that meet the denominator condition and PCA condition + PCA_2_plus_out = suppress_2_plus_PCA( + list_of_prac_ind_to_supress, + merged_df_2_plus_PCA, + main_table_prac_code_col_name, + main_table_ind_code_col_name, + measure_dict_meas_type_col_name, + measure_dict_meas_col_name, + measure_dict_meas_description_col_name, + main_table_value_col_name + ) + + # -------------------------------------------------- d -------------------------------------------------------- # + re_merged_df = pd.concat( + [PCA_1_out, PCA_2_plus_out, merged_df_0_PCA] + ).drop( + columns=[measure_dict_meas_col_name, measure_dict_meas_description_col_name, measure_dict_meas_type_col_name] + ) + # Remerge in measure dictionary data so that we can drop exclusions + re_merged_df_typed = pd.merge( + re_merged_df, + measure_dict, + left_on=main_table_meas_col_name, + right_on=measure_dict_meas_col_name, + how='left' + ) + exclusions_dropped = re_merged_df_typed[re_merged_df_typed[measure_dict_meas_type_col_name] != 'Exclusion'] + # Drop columns from final output + fully_suppressed_df = exclusions_dropped.drop( + columns=[measure_dict_meas_col_name, measure_dict_meas_description_col_name,measure_dict_meas_type_col_name] + ) + # Print how many rows are being suppressed + total_suppressed = len(fully_suppressed_df[fully_suppressed_df.VALUE == '*']) + print(f"Suppressed {total_suppressed} rows") + + return fully_suppressed_df + +#--------------------------------------------------- Suppression logic end -----------------------------------------------------------------# + def merge_mapped_data_with_ruleset_id(ncdes_with_geogs, root_directory): # Load in measure dictionary indicator_dictionary = data_load.load_indicator_and_measure_data_dictionaries(root_directory)[0] # Join ruleset ID onto data - ncdes_with_geogs_and_rulesets = pd.merge(ncdes_with_geogs, indicator_dictionary, left_on = "IND_CODE", right_on = "Indicator ID", how = "left").drop(columns = ["Indicator Description","Payment or Management Information (MI)","Indicator ID"]) + ncdes_with_geogs_and_rulesets = pd.merge( + ncdes_with_geogs, + indicator_dictionary, + left_on = "IND_CODE", + right_on = "Indicator ID", + how = "left" + ).drop( + columns = ["Indicator Description","Payment or Management Information (MI)","Indicator ID"] + ) return ncdes_with_geogs_and_rulesets \ No newline at end of file diff --git a/public_meta_data/ncdes_synthetic_data.zip b/public_meta_data/ncdes_synthetic_data.zip deleted file mode 100644 index 09e60f6..0000000 Binary files a/public_meta_data/ncdes_synthetic_data.zip and /dev/null differ