From 1503e5d69d306d012d7fb2c677a249d44989d6a6 Mon Sep 17 00:00:00 2001 From: Girum Bizuayehu Date: Fri, 19 Sep 2025 10:51:17 +0300 Subject: [PATCH 1/2] Strip and safer conversion of values to float to avoid error see HEA-615 --- pipelines/assets/wealth_characteristic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipelines/assets/wealth_characteristic.py b/pipelines/assets/wealth_characteristic.py index afacff6..9dd6303 100644 --- a/pipelines/assets/wealth_characteristic.py +++ b/pipelines/assets/wealth_characteristic.py @@ -332,10 +332,10 @@ def wealth_characteristic_instances( # If this is the summary, then also save the min and max values if reference_type == WealthGroupCharacteristicValue.CharacteristicReference.SUMMARY: min_value = df.loc[row, df.columns[-2]] - if min_value != "" and float(min_value) < 1: + if min_value and str(min_value).strip() != "" and float(min_value) < 1: min_value = float(min_value) * 100 max_value = df.loc[row, df.columns[-1]] - if max_value != "" and float(max_value) < 1: + if max_value and str(max_value).strip() != "" and float(max_value) < 1: max_value = float(max_value) * 100 wealth_group_characteristic_value["min_value"] = min_value wealth_group_characteristic_value["max_value"] = max_value From eca6ea84f25852003b42e9c3e5230bbbd907ac1d Mon Sep 17 00:00:00 2001 From: Roger Hunwicks Date: Fri, 19 Sep 2025 08:10:23 -0400 Subject: [PATCH 2/2] Better handling converting percentage_of_households - see HEA-615 --- pipelines/assets/wealth_characteristic.py | 46 +++++++++++++++++------ 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/pipelines/assets/wealth_characteristic.py b/pipelines/assets/wealth_characteristic.py index 9dd6303..5a0d12f 100644 --- a/pipelines/assets/wealth_characteristic.py +++ b/pipelines/assets/wealth_characteristic.py @@ -317,26 +317,50 @@ def wealth_characteristic_instances( wealth_group_characteristic_value["reference_type"] = reference_type - # The percentagme of households should be stored as a number between 1 and 100, + # The percentage of households should be stored as a number between 1 and 100, # but may be stored in the BSS (particularly in the summary column) as a # decimal fraction between 0 and 1, so correct those values - if ( - wealth_group_characteristic_value["wealth_characteristic_id"] == "percentage of households" - and value != "" - and float(value) < 1 - ): - value = float(value) * 100 + try: + if ( + wealth_group_characteristic_value["wealth_characteristic_id"] + == "percentage of households" + and str(value).strip() + and float(value) < 1 + ): + value = float(value) * 100 + except Exception as e: + raise ValueError( + "Error in %s converting percentage of households value '%s' to float from 'WB'!%s%s" + % (partition_key, value, column, row) + ) from e wealth_group_characteristic_value["value"] = value # If this is the summary, then also save the min and max values if reference_type == WealthGroupCharacteristicValue.CharacteristicReference.SUMMARY: min_value = df.loc[row, df.columns[-2]] - if min_value and str(min_value).strip() != "" and float(min_value) < 1: - min_value = float(min_value) * 100 max_value = df.loc[row, df.columns[-1]] - if max_value and str(max_value).strip() != "" and float(max_value) < 1: - max_value = float(max_value) * 100 + # Convert min/max percentage of households values from decimal fractions to percentages + if ( + wealth_group_characteristic_value["wealth_characteristic_id"] + == "percentage of households" + ): + try: + if str(min_value).strip() and float(min_value) < 1: + min_value = float(min_value) * 100 + except Exception as e: + raise ValueError( + "Error in %s converting percentage of households value '%s' to float from 'WB'!%s%s" + % (partition_key, min_value, df.columns[-2], row) + ) from e + try: + if str(max_value).strip() and float(max_value) < 1: + max_value = float(max_value) * 100 + except Exception as e: + raise ValueError( + "Error in %s converting percentage of households value '%s' to float from 'WB'!%s%s" + % (partition_key, max_value, df.columns[-1], row) + ) from e wealth_group_characteristic_value["min_value"] = min_value wealth_group_characteristic_value["max_value"] = max_value