Skip to content

Spp9924 nan handling updates #44

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gibbardsteve merged 5 commits into from
Aug 30, 2023
Merged

Spp9924 nan handling updates #44

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
6f0710b
SPP-9924 Cater for NaN values
gibbardsteve Aug 29, 2023
eb3dead
SPP-9924 Add extra tests for UAT
gibbardsteve Aug 29, 2023
0e61178
SPP-9924 Fix isort issues
gibbardsteve Aug 29, 2023
fdf9717
SPP-9924 Fix absolute difference type change
gibbardsteve Aug 30, 2023
488297e
Merge branch 'main' into SPP9924-NaN-Handling-Updates
gibbardsteve Aug 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions sml_small/editing/totals_and_components/totals_and_components.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
"""

import logging.config
import math
import sys
from decimal import Decimal, getcontext
from enum import Enum
Expand Down Expand Up @@ -604,7 +605,7 @@ def check_zero_errors(predictive: float, components_sum: float) -> TccMarker:
:return: tcc_marker, return a value of STOP if zero error is triggered
:rtype: TccMarker
"""
if predictive > 0 and components_sum == 0:
if predictive > 0 and (components_sum == 0 or math.isnan(components_sum)):
tcc_marker = TccMarker.STOP
logger.warning(f"TCCMarker = STOP at line:{sys._getframe().f_back.f_lineno}")
else:
Expand All @@ -629,9 +630,12 @@ def check_sum_components_predictive(
sum of the components as a float.
:rtype: float
"""
absolute_difference = abs(Decimal(str(predictive)) - Decimal(str(components_sum)))
absolute_difference = float(absolute_difference)
return absolute_difference
if not math.isnan(components_sum):
absolute_difference = abs(Decimal(str(predictive)) - Decimal(str(components_sum)))
else:
absolute_difference = abs(Decimal(str(predictive)))

return float(absolute_difference)


def determine_error_detection(
Expand Down Expand Up @@ -869,6 +873,7 @@ def correct_components(
Decimal(str(component.original_value)) / Decimal(str(components_sum))
) * Decimal(str(total))
component.final_value = float(component.final_value)

tcc_marker = TccMarker.COMPONENTS_CORRECTED
return final_total, components, tcc_marker

Expand All @@ -885,7 +890,8 @@ def sum_components(components: List[ComponentPair]) -> float:
"""
total_sum = 0
for component in components:
total_sum += Decimal(str(component.original_value))
if not math.isnan(component.original_value):
total_sum += Decimal(str(component.original_value))

return float(total_sum)

Expand Down
300 changes: 300 additions & 0 deletions tests/editing/totals_and_components/test_totals_and_components.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import math
import random
from decimal import Decimal, getcontext
from typing import List

import pytest

Expand Down Expand Up @@ -2787,6 +2789,269 @@ def test_totals_and_components(
assert (str(exc_info.value)) == str(expected_result)


class TestTotalsAndComponentsUATFails:
@pytest.mark.parametrize(
"identifier, total, components, amend_total, predictive, precision,"
"auxiliary, absolute_difference_threshold, percentage_difference_threshold,"
"expected_result, test_id",
[
# (
# "UAT-ABD-DIFF-6002", # identifier
# 10817, # total
# [9201, 866, 632, 112], # components
# False, # amend_total
# 10817, # predictive
# 28, # precision
# None, # auxiliary
# 11, # absolute_difference_threshold
# None, # percentage_difference_threshold
# (
# "UAT-ABD-DIFF-6002", # identifier
# 6, # absolute_difference
# None, # low_percent_threshold
# None, # high_percent_threshold
# 10817, # final_total
# [9206.10646563685, 866.4806215891222, 632.3507538618074, 112.0621589122204],
# "C", # tcc_marker
# ),
# "a6-6002 - If absolute difference < 11 and amend total is true then we correct the components",
# # Sheet TCC_test_data_case_a7
# # If absolute difference <= absolute difference threshold and > 0 but components sum = 0 or missing,
# # and amend total = TRUE, then do not amend total.
# # TCC marker = S
# ),
(
"UAT-ABD-DIFF-7001", # identifier
9, # total
[(0), (0), (0), (0)], # components
True, # amend_total
9, # predictive
28, # precision
None, # auxiliary
11, # absolute_difference_threshold
None, # percentage_difference_threshold
(
"UAT-ABD-DIFF-7001", # identifier
None, # absolute_difference
None, # low_percent_threshold
None, # high_percent_threshold
9, # final_total
[0, 0, 0, 0], # final_components
"S", # tcc_marker
),
"Test a7-7001 - If absolute difference < 11 and amend total is true then we correct the components",
# Sheet TCC_test_data_case_a7
# If absolute difference <= absolute difference threshold and > 0 but components sum = 0 or missing,
# and amend total = TRUE, then do not amend total.
# TCC marker = S
),
(
"UAT-ABD-DIFF-7002",
4,
[float("NaN"), float("NaN"), float("NaN"), float("NaN")],
True,
4,
28,
None,
11,
None,
(
"UAT-ABD-DIFF-7002",
None,
None,
None,
4,
[float("nan"), float("nan"), float("nan"), float("nan")],
"S",
),
"Test a7-7002 - If absolute difference < 11 and amend total is true then we correct the components",
# Sheet TCC_test_data_case_a7
# If absolute difference <= absolute difference threshold and > 0 but components sum = 0 or missing,
# and amend total = TRUE, then do not amend total.
# TCC marker = S
),
(
"UAT-ABD-DIFF-7003", # identifier
11, # total
[(0), (0), (0), (0)], # components
True, # amend_total
11, # predictive
28, # precision
None, # auxiliary
11, # absolute_difference_threshold
None, # percentage_difference_threshold
(
"UAT-ABD-DIFF-7003", # identifier
None, # absolute_difference
None, # low_percent_threshold
None, # high_percent_threshold
11, # final_total
[0, 0, 0, 0], # final_components
"S", # tcc_marker
),
"Test a7-7003 - If absolute difference < 11 and amend total is true then we correct the components",
# Sheet TCC_test_data_case_a7
# If absolute difference <= absolute difference threshold and > 0 but components sum = 0 or missing,
# and amend total = TRUE, then do not amend total.
# TCC marker = S
),
(
"UAT-ABD-DIFF-14001",
1522,
[(632), (732), float("NaN"), (162)],
False,
1522,
28,
None,
11,
None,
(
"UAT-ABD-DIFF-14001",
4,
None,
None,
1522,
[(630.343381389253), (730.0812581913499), float("NaN"), (161.57536041939713)],
"C",
),
"Test a14-14001 - If absolute difference < 11 and amend total is true then we correct the components",
# Sheet TCC_test_data_case_a7
# If absolute difference <= absolute difference threshold and > 0 but components sum = 0 or missing,
# and amend total = TRUE, then do not amend total.
# TCC marker = S
),
(
"UAT-ABD-DIFF-14002",
10705,
[(9201), (866), (632), float("NaN")],
False,
10705,
28,
None,
11,
None,
(
"UAT-ABD-DIFF-14002",
6,
None,
None,
10705,
[(9206.15992148799), (866.4856528647537), (632.3544256472568), float("NaN")], # 632.3544256472568
"C",
),
"Test a14-14002 - If absolute difference < 11 and amend total is true then we correct the components",
# Sheet TCC_test_data_case_a7
# If absolute difference <= absolute difference threshold and > 0 but components sum = 0 or missing,
# and amend total = TRUE, then do not amend total.
# TCC marker = S
),
(
"UAT-ABD-DIFF-14003",
103,
[(98), float("Nan"), (4), (6)],
False,
103,
28,
None,
11,
None,
(
"UAT-ABD-DIFF-14003",
5,
None,
None,
103,
[(93.46296296296296), float("NaN"), (3.814814814814815), (5.722222222222222)],
"C",
),
"Test a14-14003 - If absolute difference < 11 and amend total is true then we correct the components",
# Sheet TCC_test_data_case_a7
# If absolute difference <= absolute difference threshold and > 0 but components sum = 0 or missing,
# and amend total = TRUE, then do not amend total.
# TCC marker = S
)
],
)
def test_totals_and_components(
self,
capfd,
identifier,
total,
components,
amend_total,
predictive,
precision,
auxiliary,
absolute_difference_threshold,
percentage_difference_threshold,
expected_result,
test_id,
):
if isinstance(expected_result, tuple):
try:
results = totals_and_components(
identifier=identifier,
total=total,
components=components,
amend_total=amend_total,
predictive=predictive,
auxiliary=auxiliary,
absolute_difference_threshold=absolute_difference_threshold,
percentage_difference_threshold=percentage_difference_threshold,
)

# Capture the printed output and remove any leading or trailing whitespace
captured = capfd.readouterr()
printed_output = captured.out.strip()

print(printed_output)

# calculate_weighted(total, components)

assert results.identifier == expected_result[0]
assert results.absolute_difference == expected_result[1]
assert results.low_percent_threshold == expected_result[2]
assert results.high_percent_threshold == expected_result[3]
assert results.final_total == expected_result[4]
assert results.tcc_marker == expected_result[6]

if results.tcc_marker == "T" or results.tcc_marker == "C":
sum_of_components = 0
for component in results.final_components:
print(f"fc - {results.final_components}")
if not math.isnan(component):
sum_of_components += Decimal(str(component))

print(f"Comparison Sum: {sum_of_components}")
sum_of_components = float(sum_of_components)
assert sum_of_components == expected_result[4]

compare_list_of_floats(results.final_components, expected_result[5])

except Exception as e:
pytest.fail(
EXCEPTION_FAIL_MESSAGE.format(
test_id=test_id,
exception_type=type(e).__name__,
exception_msg=str(e.args),
)
)
else:
with pytest.raises(Exception) as exc_info:
totals_and_components(
identifier=identifier,
total=total,
components=components,
amend_total=amend_total,
predictive=predictive,
precision=precision,
auxiliary=auxiliary,
absolute_difference_threshold=absolute_difference_threshold,
percentage_difference_threshold=percentage_difference_threshold,
)
assert (str(exc_info.value)) == str(expected_result)


class TestValidateInput:
@pytest.mark.parametrize(
"identifier, total, components, amend_total, predictive, "
Expand Down Expand Up @@ -3996,3 +4261,38 @@ def test_correct_components(
exception_msg=str(e),
)
)


def compare_list_of_floats(components: List[float], expected_components: List[float]):
for i, (component, expected) in enumerate(zip(components, expected_components)):
if math.isnan(component) and math.isnan(expected):
assert math.isnan(component) and math.isnan(expected), f"Both components at index {i} are NaN"
elif math.isnan(component) or math.isnan(expected):
assert False, f"Component at index {i} is NaN, the other is not component {component}, expected {expected}"
elif component != expected:
assert False, f"Values of components at index {i} do not match: component {component}, expected {expected}"


def calculate_weighted(total: float, components: List[float]):
component_sum = Decimal(str(0))
adjusted_component_sum = Decimal(str(0))
adjusted_components = []
for component in components:
if not math.isnan(component):
component_sum += Decimal(str(component))

print(f"Component sum is: {component_sum}")

for i, component in enumerate(components):
adjusted_components.append((Decimal(component) / component_sum) * Decimal(total))

if not math.isnan(component):
print(f"Adjusted Component {adjusted_components[i]}")
# adjusted_component_sum += element
print(f"convert to float {float(adjusted_components[i])}")

adjusted_component_sum = math.fsum(adjusted_components)
print(adjusted_component_sum)
print(f"adjusted_component_sum: {adjusted_component_sum}")

print(f"convert to float {float(adjusted_component_sum)}")