Skip to content

Commit

Permalink
feat: bypass data quality error for citrus (#8444)
Browse files Browse the repository at this point in the history 
* added feature to ignore energy does not match value computed from other nut

* rework get_inherited_property

* rework get_inherited_property

* apply suggestions for return
  • Loading branch information
@benbenben2
benbenben2 committed May 25, 2023
1 parent 7b97896 commit e3a7394
Show file tree
Hide file tree
Showing 4 changed files with 86 additions and 20 deletions.
38 changes: 23 additions & 15 deletions lib/ProductOpener/DataQualityFood.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -591,22 +591,30 @@ sub check_nutrition_data_energy_computation ($product_ref) {
$computed_energy += $grams * $energy_per_gram;
}

# Compare to specified energy value with a tolerance of 30% + an additiontal tolerance of 5
if ( ($computed_energy < ($specified_energy * 0.7 - 5))
or ($computed_energy > ($specified_energy * 1.3 + 5)))
{
# we have a quality problem
push @{$product_ref->{data_quality_errors_tags}},
"en:energy-value-in-$unit-does-not-match-value-computed-from-other-nutrients";
}
# following error/warning should be ignored for some categories
# for example, lemon juices containing organic acid, it is forbidden to display organic acid in nutrition tables but
# organic acid contributes to the total energy calculation
my $ignore_energy_calculated_error
= get_inherited_property_from_categories_tags($product_ref, "ignore_energy_calculated_error:en");

if (not($ignore_energy_calculated_error)) {
# Compare to specified energy value with a tolerance of 30% + an additiontal tolerance of 5
if ( ($computed_energy < ($specified_energy * 0.7 - 5))
or ($computed_energy > ($specified_energy * 1.3 + 5)))
{
# we have a quality problem
push @{$product_ref->{data_quality_errors_tags}},
"en:energy-value-in-$unit-does-not-match-value-computed-from-other-nutrients";
}

# Compare to specified energy value with a tolerance of 15% + an additiontal tolerance of 5
if ( ($computed_energy < ($specified_energy * 0.85 - 5))
or ($computed_energy > ($specified_energy * 1.15 + 5)))
{
# we have a quality warning
push @{$product_ref->{data_quality_warnings_tags}},
"en:energy-value-in-$unit-may-not-match-value-computed-from-other-nutrients";
# Compare to specified energy value with a tolerance of 15% + an additiontal tolerance of 5
if ( ($computed_energy < ($specified_energy * 0.85 - 5))
or ($computed_energy > ($specified_energy * 1.15 + 5)))
{
# we have a quality warning
push @{$product_ref->{data_quality_warnings_tags}},
"en:energy-value-in-$unit-may-not-match-value-computed-from-other-nutrients";
}
}

$nutriments_ref->{"energy-${unit}_value_computed"} = $computed_energy;
Expand Down
31 changes: 31 additions & 0 deletions lib/ProductOpener/Tags.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ BEGIN {
&get_property
&get_property_with_fallbacks
&get_inherited_property
&get_inherited_property_from_categories_tags
&get_inherited_properties
&get_tags_grouped_by_property
Expand Down Expand Up @@ -381,6 +382,36 @@ sub get_inherited_property ($tagtype, $canon_tagid, $property) {
return;
}

=head2 get_inherited_property_from_categories_tags ($product_ref, $property) {
Iterating from the most specific category, try to get a property for a tag by exploring the taxonomy (using parents).
=head3 Parameters
=head4 $product_ref - the product reference
=head4 $property - the property - string
=head3 Return
The property if found.
=cut

sub get_inherited_property_from_categories_tags ($product_ref, $property) {
my $category_match;

if ((defined $product_ref->{categories_tags}) and (scalar @{$product_ref->{categories_tags}} > 0)) {

# Start with most specific category first
foreach my $category (reverse @{$product_ref->{categories_tags}}) {

$category_match = lc(get_inherited_property("categories", $category, $property));
last if $category_match;
}
}
return $category_match;
}

=head2 get_inherited_properties ($tagtype, $canon_tagid, $properties_names_ref, $fallback_lcs = ["xx", "en"]) {
Try to get a set of properties for a tag by exploring the taxonomy (using parents).
Expand Down
15 changes: 12 additions & 3 deletions taxonomies/categories.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,13 @@ stopwords:nl:bevat,en
stopwords:nl_be:bevat,en
stopwords:de:und,mit,von

# add following tag to ignore "Energy value in kJ does not correspond to the value calculated from the other nutrients error
# only for categories having nutrients that are not displayed in the nutrition table and contributing to the energy
# for example, lemon juices containing organic acid, it is forbidden to display organic acid in nutrition tables but
# organic acid contributes to the total energy calculation. Only option is yes for this tag
# ignore_energy_calculated_error:en:yes


en:Artisan products
ca:Prouctes artesans
de:Handgefertigte Produkte, Artisanale Produkte, Handgemachte Produkte
Expand Down Expand Up @@ -5631,22 +5638,23 @@ agribalyse_food_code:en:13067
agribalyse_proxy_food_name:en:Lime, pulp, raw
agribalyse_proxy_food_name:fr:Citron vert ou Lime, pulpe, cru
wikidata:en:Q5361217
ignore_energy_calculated_error:en:yes

#<en:Lime juice
#<en:Lime juices
#en:Home-made lime juice
#fr:Jus de citron vert maison
#ciqual_food_code:en:2030
#ciqual_food_name:en:Lime juice, home-made
#ciqual_food_name:fr:Jus de citron vert, maison

<en:Lime juice
<en:Lime juices
en:Squeezed lime juices, Pure lime juices
fr:Jus de citron vert pur jus
ciqual_food_code:en:2031
ciqual_food_name:en:Lime juice, pure juice
ciqual_food_name:fr:Jus de citron vert, pur jus

<en:Lime juice
<en:Lime juices
en:Lime juices from concentrate
fr:Jus de citron vert à partir d'un concentré
nl:limoensappen uit concentraat
Expand All @@ -5669,6 +5677,7 @@ pt:Sumos de limão
ru:Лимонный сок
zh:柠檬汁
wikidata:en:Q1375049
ignore_energy_calculated_error:en:yes

<en:Lemon juice
en:Home-made lemon juice
Expand Down
22 changes: 20 additions & 2 deletions tests/unit/dataqualityfood.t
View file
Original file line number Diff line number Diff line change
Expand Up @@ -344,7 +344,7 @@ ok(has_tag($product_ref, 'data_quality', 'en:sum-of-ingredients-with-unspecified
ok(has_tag($product_ref, 'data_quality', 'en:sum-of-ingredients-with-specified-percent-greater-than-100'))
or diag explain $product_ref;

# energy matches nutrients
# energy does not match nutrients
$product_ref = {
nutriments => {
"energy-kj_value" => 5,
Expand All @@ -360,7 +360,25 @@ ok(has_tag($product_ref, 'data_quality', 'en:energy-value-in-kj-does-not-match-v
'energy not matching nutrients')
or diag explain $product_ref;

# energy does not match nutrients
# energy does not match nutrients but this alert is ignored for this category
$product_ref = {
categories_tags => ['en:squeezed-lemon-juices'],
nutriments => {
"energy-kj_value" => 5,
"carbohydrates_value" => 10,
"fat_value" => 20,
"proteins_value" => 30,
"fiber_value" => 2,
}
};
ProductOpener::DataQuality::check_quality($product_ref);
is($product_ref->{nutriments}{"energy-kj_value_computed"}, 1436);
ok(
!has_tag($product_ref, 'data_quality', 'en:energy-value-in-kj-does-not-match-value-computed-from-other-nutrients'),
'energy not matching nutrients but category possesses ignore_energy_calculated_error:en:yes tag'
) or diag explain $product_ref;

# energy matches nutrients
$product_ref = {
nutriments => {
"energy-kj_value" => 1435,
Expand Down

0 comments on commit e3a7394

Please sign in to comment.