From e3a73942b84a948e9531c09188608614b9d7a3a5 Mon Sep 17 00:00:00 2001 From: benbenben2 <110821832+benbenben2@users.noreply.github.com> Date: Thu, 25 May 2023 22:05:24 +0200 Subject: [PATCH] feat: bypass data quality error for citrus (#8444) * added feature to ignore energy does not match value computed from other nut * rework get_inherited_property * rework get_inherited_property * apply suggestions for return --- lib/ProductOpener/DataQualityFood.pm | 38 +++++++++++++++++----------- lib/ProductOpener/Tags.pm | 31 +++++++++++++++++++++++ taxonomies/categories.txt | 15 ++++++++--- tests/unit/dataqualityfood.t | 22 ++++++++++++++-- 4 files changed, 86 insertions(+), 20 deletions(-) diff --git a/lib/ProductOpener/DataQualityFood.pm b/lib/ProductOpener/DataQualityFood.pm index 287f3da99ceb2..d1eabe57a2462 100644 --- a/lib/ProductOpener/DataQualityFood.pm +++ b/lib/ProductOpener/DataQualityFood.pm @@ -591,22 +591,30 @@ sub check_nutrition_data_energy_computation ($product_ref) { $computed_energy += $grams * $energy_per_gram; } - # Compare to specified energy value with a tolerance of 30% + an additiontal tolerance of 5 - if ( ($computed_energy < ($specified_energy * 0.7 - 5)) - or ($computed_energy > ($specified_energy * 1.3 + 5))) - { - # we have a quality problem - push @{$product_ref->{data_quality_errors_tags}}, - "en:energy-value-in-$unit-does-not-match-value-computed-from-other-nutrients"; - } + # following error/warning should be ignored for some categories + # for example, lemon juices containing organic acid, it is forbidden to display organic acid in nutrition tables but + # organic acid contributes to the total energy calculation + my $ignore_energy_calculated_error + = get_inherited_property_from_categories_tags($product_ref, "ignore_energy_calculated_error:en"); + + if (not($ignore_energy_calculated_error)) { + # Compare to specified energy value with a tolerance of 30% + an additiontal tolerance of 5 + if ( ($computed_energy < ($specified_energy * 0.7 - 5)) + or ($computed_energy > ($specified_energy * 1.3 + 5))) + { + # we have a quality problem + push @{$product_ref->{data_quality_errors_tags}}, + "en:energy-value-in-$unit-does-not-match-value-computed-from-other-nutrients"; + } - # Compare to specified energy value with a tolerance of 15% + an additiontal tolerance of 5 - if ( ($computed_energy < ($specified_energy * 0.85 - 5)) - or ($computed_energy > ($specified_energy * 1.15 + 5))) - { - # we have a quality warning - push @{$product_ref->{data_quality_warnings_tags}}, - "en:energy-value-in-$unit-may-not-match-value-computed-from-other-nutrients"; + # Compare to specified energy value with a tolerance of 15% + an additiontal tolerance of 5 + if ( ($computed_energy < ($specified_energy * 0.85 - 5)) + or ($computed_energy > ($specified_energy * 1.15 + 5))) + { + # we have a quality warning + push @{$product_ref->{data_quality_warnings_tags}}, + "en:energy-value-in-$unit-may-not-match-value-computed-from-other-nutrients"; + } } $nutriments_ref->{"energy-${unit}_value_computed"} = $computed_energy; diff --git a/lib/ProductOpener/Tags.pm b/lib/ProductOpener/Tags.pm index da0447c0b0508..932b1e16b4337 100644 --- a/lib/ProductOpener/Tags.pm +++ b/lib/ProductOpener/Tags.pm @@ -59,6 +59,7 @@ BEGIN { &get_property &get_property_with_fallbacks &get_inherited_property + &get_inherited_property_from_categories_tags &get_inherited_properties &get_tags_grouped_by_property @@ -381,6 +382,36 @@ sub get_inherited_property ($tagtype, $canon_tagid, $property) { return; } +=head2 get_inherited_property_from_categories_tags ($product_ref, $property) { + +Iterating from the most specific category, try to get a property for a tag by exploring the taxonomy (using parents). + +=head3 Parameters + +=head4 $product_ref - the product reference +=head4 $property - the property - string + +=head3 Return + +The property if found. + +=cut + +sub get_inherited_property_from_categories_tags ($product_ref, $property) { + my $category_match; + + if ((defined $product_ref->{categories_tags}) and (scalar @{$product_ref->{categories_tags}} > 0)) { + + # Start with most specific category first + foreach my $category (reverse @{$product_ref->{categories_tags}}) { + + $category_match = lc(get_inherited_property("categories", $category, $property)); + last if $category_match; + } + } + return $category_match; +} + =head2 get_inherited_properties ($tagtype, $canon_tagid, $properties_names_ref, $fallback_lcs = ["xx", "en"]) { Try to get a set of properties for a tag by exploring the taxonomy (using parents). diff --git a/taxonomies/categories.txt b/taxonomies/categories.txt index 6ab806a4678f9..e07f4990ac181 100644 --- a/taxonomies/categories.txt +++ b/taxonomies/categories.txt @@ -36,6 +36,13 @@ stopwords:nl:bevat,en stopwords:nl_be:bevat,en stopwords:de:und,mit,von +# add following tag to ignore "Energy value in kJ does not correspond to the value calculated from the other nutrients error +# only for categories having nutrients that are not displayed in the nutrition table and contributing to the energy +# for example, lemon juices containing organic acid, it is forbidden to display organic acid in nutrition tables but +# organic acid contributes to the total energy calculation. Only option is yes for this tag +# ignore_energy_calculated_error:en:yes + + en:Artisan products ca:Prouctes artesans de:Handgefertigte Produkte, Artisanale Produkte, Handgemachte Produkte @@ -5631,22 +5638,23 @@ agribalyse_food_code:en:13067 agribalyse_proxy_food_name:en:Lime, pulp, raw agribalyse_proxy_food_name:fr:Citron vert ou Lime, pulpe, cru wikidata:en:Q5361217 +ignore_energy_calculated_error:en:yes -# { "energy-kj_value" => 5, @@ -360,7 +360,25 @@ ok(has_tag($product_ref, 'data_quality', 'en:energy-value-in-kj-does-not-match-v 'energy not matching nutrients') or diag explain $product_ref; -# energy does not match nutrients +# energy does not match nutrients but this alert is ignored for this category +$product_ref = { + categories_tags => ['en:squeezed-lemon-juices'], + nutriments => { + "energy-kj_value" => 5, + "carbohydrates_value" => 10, + "fat_value" => 20, + "proteins_value" => 30, + "fiber_value" => 2, + } +}; +ProductOpener::DataQuality::check_quality($product_ref); +is($product_ref->{nutriments}{"energy-kj_value_computed"}, 1436); +ok( + !has_tag($product_ref, 'data_quality', 'en:energy-value-in-kj-does-not-match-value-computed-from-other-nutrients'), + 'energy not matching nutrients but category possesses ignore_energy_calculated_error:en:yes tag' +) or diag explain $product_ref; + +# energy matches nutrients $product_ref = { nutriments => { "energy-kj_value" => 1435,