fix: Nutri-Score for olive oils with unrecognized ingredients (#9247)

lib/ProductOpener/DataQualityFood.pm

@@ -594,7 +594,7 @@ sub check_nutrition_data_energy_computation ($product_ref) {
 			# following error/warning should be ignored for some categories
 			# for example, lemon juices containing organic acid, it is forbidden to display organic acid in nutrition tables but
 			# organic acid contributes to the total energy calculation
-			my $ignore_energy_calculated_error
+			my ($ignore_energy_calculated_error, $category_id)
 				= get_inherited_property_from_categories_tags($product_ref, "ignore_energy_calculated_error:en");
 
 			if (not((defined $ignore_energy_calculated_error) and ($ignore_energy_calculated_error eq 'yes'))) {
@@ -920,7 +920,7 @@ sub check_nutrition_data ($product_ref) {
 		}
 
 		# some categories have expected nutriscore grade - push data quality error if calculated nutriscore grade differs from expected nutriscore grade or if it is not calculated
-		my $expected_nutriscore_grade
+		my ($expected_nutriscore_grade, $category_id)
 			= get_inherited_property_from_categories_tags($product_ref, "expected_nutriscore_grade:en");
 
 		# we expect single letter a, b, c, d, e for nutriscore grade in the taxonomy. Case insensitive (/i).
@@ -940,7 +940,8 @@ sub check_nutrition_data ($product_ref) {
 
 		# some categories have an expected ingredient - push data quality error if ingredient differs from expected ingredient
 		# note: we currently support only 1 expected ingredient
-		my $expected_ingredients = get_inherited_property_from_categories_tags($product_ref, "expected_ingredients:en");
+		my ($expected_ingredients, $category_id)
+			= get_inherited_property_from_categories_tags($product_ref, "expected_ingredients:en");
 
 		if ((defined $expected_ingredients)) {
 			$expected_ingredients = canonicalize_taxonomy_tag("en", "ingredients", $expected_ingredients);

lib/ProductOpener/Food.pm

@@ -1000,10 +1000,10 @@ sub is_fat_oil_nuts_seeds_for_nutrition_score ($product_ref) {
 		return 1;
 	}
 	else {
-		my $hs_heading = get_inherited_property_from_categories_tags($product_ref, "wco_hs_heading:en");
+		my ($hs_heading, $category_id) = get_inherited_property_from_categories_tags($product_ref, "wco_hs_heading:en");
 
 		if (defined $hs_heading) {
-			my $hs_code = get_inherited_property_from_categories_tags($product_ref, "wco_hs_code:en");
+			my ($hs_code, $category_id) = get_inherited_property_from_categories_tags($product_ref, "wco_hs_code:en");
 
 			if (
 				($hs_heading eq "08.01") or ($hs_heading eq "08.02")    # nuts
@@ -1060,7 +1060,7 @@ similar products, and meat extracts and juices)
 sub is_red_meat_product_for_nutrition_score ($product_ref) {
 
 	# Use the category HS code if all the corresponding products are considered red meat
-	my $hs_heading = get_inherited_property_from_categories_tags($product_ref, "wco_hs_heading:en");
+	my ($hs_heading, $category_id) = get_inherited_property_from_categories_tags($product_ref, "wco_hs_heading:en");
 
 	if (defined $hs_heading) {
 
@@ -1195,14 +1195,21 @@ sub compute_nutriscore_2021_fruits_vegetables_nuts_colza_walnut_olive_oil ($prod
 
 	my $fruits = undef;
 
-	# If the product is in a category that has no unprocessed fruits/vegetables/nuts, return 0
-	my $nutriscore_without_unprocessed_fruits_vegetables_legumes
+	# Check if we have a category override:
+	# - if the product is in a category that has no unprocessed fruits/vegetables/nuts (e.g. crisps), return 0
+	# - if the product is in category that has only ingredients that are consired fruits/vegetables/nuts (e.g. olive oil), return 100
+	my ($nutriscore_category_override_for_fruits_vegetables_legumes, $category_id)
 		= get_inherited_property_from_categories_tags($product_ref,
-		"nutriscore_without_unprocessed_fruits_vegetables_legumes:en");
-	if (    (defined $nutriscore_without_unprocessed_fruits_vegetables_legumes)
-		and ($nutriscore_without_unprocessed_fruits_vegetables_legumes eq "yes"))
-	{
-		return 0;
+		"nutriscore_category_override_for_fruits_vegetables_legumes:en");
+	if (defined $nutriscore_category_override_for_fruits_vegetables_legumes) {
+
+		# We are close to certain that those category overrides (either 0 or 100) are correct,
+		# so we do not add a nutrition_score_warning_fruits_vegetables_legumes_from_category warning
+		add_tag($product_ref, "misc", "en:nutrition-fruits-vegetables-nuts-from-category");
+		my $category = $category_id;
+		$category =~ s/:/-/;
+		add_tag($product_ref, "misc", "en:nutrition-fruits-vegetables-nuts-from-category-$category");
+		return $nutriscore_category_override_for_fruits_vegetables_legumes;
 	}
 
 	if (defined $product_ref->{nutriments}{"fruits-vegetables-nuts-dried" . $prepared . "_100g"}) {
@@ -1277,7 +1284,7 @@ sub compute_nutriscore_2021_fruits_vegetables_nuts_colza_walnut_olive_oil ($prod
 
 # estimates by category of products. not exact values. For the Nutri-Score, it's important to distinguish only between the thresholds: 40, 60 and 80
 # first entries match first, so we put potatoes before vegetables
-my @fruits_vegetables_legumes_by_category_sorted = (
+my @fruits_vegetables_legumes_by_category_if_no_ingredients_specified_sorted = (
 	["en:potatoes", 0],
 	["en:sweet-potatoes", 0],
 	["en:fruit-juices", 100],
@@ -1289,10 +1296,6 @@ my @fruits_vegetables_legumes_by_category_sorted = (
 	["en:canned-fruits", 90],
 	["en:frozen-fruits", 90],
 	["en:jams", 50],
-	# for products in the fat/oil/nuts/seeds category
-	["en:avocado-oils", 100],
-	["en:olive-oils", 100],
-
 );
 
 =head2 compute_nutriscore_2023_fruits_vegetables_legumes($product_ref, $prepared)
@@ -1313,14 +1316,21 @@ Differences with the 2021 version:
 
 sub compute_nutriscore_2023_fruits_vegetables_legumes ($product_ref, $prepared) {
 
-	# If the product is in a category that has no unprocessed fruits/vegetables/nuts, return 0
-	my $nutriscore_without_unprocessed_fruits_vegetables_legumes
+	# Check if we have a category override:
+	# - if the product is in a category that has no unprocessed fruits/vegetables/nuts (e.g. crisps), return 0
+	# - if the product is in category that has only ingredients that are consired fruits/vegetables/nuts (e.g. olive oil), return 100
+	my ($nutriscore_category_override_for_fruits_vegetables_legumes, $category_id)
 		= get_inherited_property_from_categories_tags($product_ref,
-		"nutriscore_without_unprocessed_fruits_vegetables_legumes:en");
-	if (    (defined $nutriscore_without_unprocessed_fruits_vegetables_legumes)
-		and ($nutriscore_without_unprocessed_fruits_vegetables_legumes eq "yes"))
-	{
-		return 0;
+		"nutriscore_category_override_for_fruits_vegetables_legumes:en");
+	if (defined $nutriscore_category_override_for_fruits_vegetables_legumes) {
+		# We are close to certain that those category overrides (either 0 or 100) are correct,
+		# so we do not add a nutrition_score_warning_fruits_vegetables_legumes_from_category warning
+		add_tag($product_ref, "misc", "en:nutrition-fruits-vegetables-legumes-from-category");
+		my $category = $category_id;
+		$category =~ s/:/-/;
+		add_tag($product_ref, "misc", "en:nutrition-fruits-vegetables-legumes-from-category-$category");
+		return $nutriscore_category_override_for_fruits_vegetables_legumes
+			+ 0;    # Add 0 to make the property value a number
 	}
 
 	my $fruits_vegetables_legumes = deep_get($product_ref, "nutriments",
@@ -1335,7 +1345,7 @@ sub compute_nutriscore_2023_fruits_vegetables_legumes ($product_ref, $prepared)
 	}
 	# if we do not have ingredients, try to use the product category
 	else {
-		foreach my $category_ref (@fruits_vegetables_legumes_by_category_sorted) {
+		foreach my $category_ref (@fruits_vegetables_legumes_by_category_if_no_ingredients_specified_sorted) {
 
 			my $category_id = $category_ref->[0];
 			if (has_tag($product_ref, "categories", $category_id)) {

lib/ProductOpener/Tags.pm

@@ -409,15 +409,15 @@ sub get_property_from_tags ($tagtype, $tags_ref, $property) {
 	if (defined $tags_ref) {
 		foreach my $tagid (@$tags_ref) {
 			$value = get_property($tagtype, $tagid, $property);
-			last if $value;
+			last if defined $value;
 		}
 	}
 	return $value;
 }
 
 =head2 get_inherited_property_from_tags ($tagtype, $tags_ref, $property)
 
-Return the value of an inherited property for the first tag of a list that has this property.
+Return the value of an inherited property for the first tag of a list that has this property, and the corresponding matching tag.
 
 =head3 Parameters
 
@@ -427,18 +427,28 @@ Return the value of an inherited property for the first tag of a list that has t
 
 =head4 $property
 
+=head3 Return values
+
+=head4 $property_value
+
+=head4 $matching_tagid
+
 =cut
 
 sub get_inherited_property_from_tags ($tagtype, $tags_ref, $property) {
 
 	my $value;
+	my $matching_tagid;
 	if (defined $tags_ref) {
 		foreach my $tagid (@$tags_ref) {
 			$value = get_inherited_property($tagtype, $tagid, $property);
-			last if $value;
+			if (defined $value) {
+				$matching_tagid = $tagid;
+				last;
+			}
 		}
 	}
-	return $value;
+	return ($value, $matching_tagid);
 }
 
 =head2 get_matching_regexp_property_from_tags ($tagtype, $tags_ref, $property, $regexp)
@@ -483,18 +493,25 @@ Iterating from the most specific category, try to get a property for a tag by ex
 
 =head3 Return
 
-The property if found.
+=head4 $property_value
+
+The property value if found.
+
+=head4 $matching_category_id
+
+The matching category id if we found a property value.
 
 =cut
 
 sub get_inherited_property_from_categories_tags ($product_ref, $property) {
 
 	if (defined $product_ref->{categories_tags}) {
 		# We reverse the list of categories in order to have the most specific categories first
-		return get_inherited_property_from_tags("categories", [reverse @{$product_ref->{categories_tags}}], $property);
+		return (
+			get_inherited_property_from_tags("categories", [reverse @{$product_ref->{categories_tags}}], $property));
 	}
 
-	return;
+	return (undef, undef);
 }
 
 =head2 get_inherited_properties ($tagtype, $canon_tagid, $properties_names_ref, $fallback_lcs = ["xx", "en"]) {

taxonomies/categories.txt

@@ -36,14 +36,15 @@ stopwords:nl:bevat,en
 stopwords:nl_be:bevat,en
 stopwords:de:und,mit,von
 
+### Properties for categories entries
+
 # add following tag for category having always same nutriscore grade
 # only 1 letter is allowed
 # expected_nutriscore_grade:en:c
 # add following tag for category having always same ingredient
 # only 1 ingredient tag is allowed (use "en:olive-oil" and not "en:Olive oil")
 # expected_ingredients:en: en:olive-oil
 
-
 # add following tag to ignore "Energy value in kJ does not correspond to the value calculated from the other nutrients error
 # only for categories having nutrients that are not displayed in the nutrition table and contributing to the energy
 # for example, lemon juices containing organic acid, it is forbidden to display organic acid in nutrition tables but
@@ -59,6 +60,11 @@ stopwords:de:und,mit,von
 # wco_hs_code:en: 1204.00
 # are from https://www.wcoomd.org/en/topics/nomenclature/instrument-and-tools/hs-nomenclature-2022-edition/hs-nomenclature-2022-edition.aspx
 
+# For categories for which we are certain there are no fruits/vegetables/legumes (as defined for Nutri-Score)
+# we put nutriscore_category_override_for_fruits_vegetables_legumes:en: 0
+# And for categories where we are certain there is 100% fruits/vegetables/legumes
+# we put nutriscore_category_override_for_fruits_vegetables_legumes:en: 100
+
 en:Artisan products
 ca:Prouctes artesans
 de:Handgefertigte Produkte, Artisanale Produkte, Handgemachte Produkte
@@ -35242,7 +35248,7 @@ wikidata:en:Q1163138
 ciqual_food_code:en:31027
 ciqual_food_name:en:Candied fruits
 ciqual_food_name:fr:Fruit confit
-nutriscore_without_unprocessed_fruits_vegetables_legumes:en:yes
+nutriscore_category_override_for_fruits_vegetables_legumes:en: 0
 
 <en:Candied fruit
 en:Candied lemon
@@ -39923,7 +39929,7 @@ pl:Chipsy, Czipsy
 ru:Чипсы
 th:มันฝรั่งทอด
 zh:薯片
-nutriscore_without_unprocessed_fruits_vegetables_legumes:en:yes
+nutriscore_category_override_for_fruits_vegetables_legumes:en: 0
 
 <en:Crisps
 en:Chickpea crisps
@@ -53578,6 +53584,8 @@ agribalyse_food_code:en:17100
 ciqual_food_code:en:17100
 ciqual_food_name:en:Avocado oil
 ciqual_food_name:fr:Huile d'avocat
+# oils from fruits/vegetables/legumes for products in the fats/oils category are counted as fruits/vegetables/legumes for the Nutri-Score 2023
+nutriscore_category_override_for_fruits_vegetables_legumes:en: 100
 
 <en:Fruit and fruit seed oils
 en:Coconut oils
@@ -53914,6 +53922,8 @@ wikidata:en:Q93165
 agribalyse_proxy_food_code:en:17270
 agribalyse_proxy_food_name:en:Olive oil, extra virgin
 agribalyse_proxy_food_name:fr:Huile d'olive vierge extra
+# oils from fruits/vegetables/legumes for products in the fats/oils category are counted as fruits/vegetables/legumes for the Nutri-Score 2023
+nutriscore_category_override_for_fruits_vegetables_legumes:en: 100
 
 <en:Olive oils
 en:Virgin olive oils
@@ -55955,7 +55965,7 @@ pt:Farinha
 ru:Мука
 tr:Un
 zh:面粉
-nutriscore_without_unprocessed_fruits_vegetables_legumes:en:yes
+nutriscore_category_override_for_fruits_vegetables_legumes:en: 0
 
 <en:Flours
 fr:Farines de souchet
@@ -57047,7 +57057,7 @@ oqali_family:en: fr:Aperitifs a croquer - Pop corn
 agribalyse_proxy_food_code:en:9230
 agribalyse_proxy_food_name:en:Pop-corn or oil popped maize, salted
 agribalyse_proxy_food_name:fr:Pop-corn ou Maïs éclaté, à l'huile, salé
-nutriscore_without_unprocessed_fruits_vegetables_legumes:en:yes
+nutriscore_category_override_for_fruits_vegetables_legumes:en: 0
 
 <en:Popcorn
 en:Plain popcorn, Unsalted pop-corn, Unsalted air-popped maize
@@ -99020,7 +99030,7 @@ ciqual_food_name:en:Tofu, plain
 ciqual_food_name:fr:Tofu, nature
 nova:en:3
 description:en:Tofu is a food prepared by coagulating soy milk and then pressing the resulting curds into solid white blocks of varying softness. Tofu has very little flavor or smell of its own. Consequently, tofu can be used in both savory or sweet dishes, acting as a bland background for presenting the flavors of the other ingredients used. In order to flavor the tofu it is often marinated in soy sauce, chillis, sesame oil, etc.
-nutriscore_without_unprocessed_fruits_vegetables_legumes:en:yes
+nutriscore_category_override_for_fruits_vegetables_legumes:en: 0
 
 <en:Tofu
 en:Plain tofu

tests/unit/dataqualityfood.t

@@ -17,10 +17,12 @@ sub check_quality_and_test_product_has_quality_tag($$$$) {
 	my $yesno = shift;
 	ProductOpener::DataQuality::check_quality($product_ref);
 	if ($yesno) {
-		ok(has_tag($product_ref, 'data_quality', $tag), $reason) or diag explain $product_ref;
+		ok(has_tag($product_ref, 'data_quality', $tag), $reason)
+			or diag explain {tag => $tag, yesno => $yesno, product => $product_ref};
 	}
 	else {
-		ok(!has_tag($product_ref, 'data_quality', $tag), $reason) or diag explain $product_ref;
+		ok(!has_tag($product_ref, 'data_quality', $tag), $reason)
+			or diag explain {tag => $tag, yesno => $yesno, product => $product_ref};
 	}
 
 	return;

tests/unit/expected_test_results/nutriscore/en-avocado-oil.json

@@ -100,9 +100,10 @@
    "minerals_tags" : [],
    "misc_tags" : [
       "en:nutriscore-computed",
-      "en:nutrition-fruits-vegetables-nuts-estimate-from-ingredients",
-      "en:nutrition-all-nutriscore-values-known",
-      "en:nutrition-fruits-vegetables-legumes-estimate-from-ingredients",
+      "en:nutrition-fruits-vegetables-nuts-from-category",
+      "en:nutrition-fruits-vegetables-nuts-from-category-en-avocado-oils",
+      "en:nutrition-fruits-vegetables-legumes-from-category",
+      "en:nutrition-fruits-vegetables-legumes-from-category-en-avocado-oils",
       "en:nutriscore-2021-different-from-2023",
       "en:nutriscore-2021-worse-than-2023",
       "en:nutriscore-2021-c-2023-b"
@@ -139,7 +140,7 @@
             "fiber" : 0,
             "fiber_points" : 0,
             "fiber_value" : 0,
-            "fruits_vegetables_nuts_colza_walnut_olive_oils" : 100,
+            "fruits_vegetables_nuts_colza_walnut_olive_oils" : "100",
             "fruits_vegetables_nuts_colza_walnut_olive_oils_points" : 5,
             "fruits_vegetables_nuts_colza_walnut_olive_oils_value" : 100,
             "is_beverage" : 0,
@@ -230,7 +231,7 @@
       "fiber" : 0,
       "fiber_points" : 0,
       "fiber_value" : 0,
-      "fruits_vegetables_nuts_colza_walnut_olive_oils" : 100,
+      "fruits_vegetables_nuts_colza_walnut_olive_oils" : "100",
       "fruits_vegetables_nuts_colza_walnut_olive_oils_points" : 5,
       "fruits_vegetables_nuts_colza_walnut_olive_oils_value" : 100,
       "grade" : "c",
@@ -271,10 +272,6 @@
    ],
    "nutrition_score_beverage" : 0,
    "nutrition_score_debug" : "",
-   "nutrition_score_warning_fruits_vegetables_legumes_estimate_from_ingredients" : 1,
-   "nutrition_score_warning_fruits_vegetables_legumes_estimate_from_ingredients_value" : 100,
-   "nutrition_score_warning_fruits_vegetables_nuts_estimate_from_ingredients" : 1,
-   "nutrition_score_warning_fruits_vegetables_nuts_estimate_from_ingredients_value" : 100,
    "other_nutritional_substances_tags" : [],
    "pnns_groups_1" : "Fat and sauces",
    "pnns_groups_1_tags" : [