fix: better count of fruits/vegetables/legumes for Nutri-Score (#9102)

lib/ProductOpener/Food.pm

@@ -1120,6 +1120,16 @@ sub compute_nutriscore_2021_fruits_vegetables_nuts_colza_walnut_olive_oil ($prod
 
 	my $fruits = undef;
 
+	# If the product is in a category that has no unprocessed fruits/vegetables/nuts, return 0
+	my $nutriscore_without_unprocessed_fruits_vegetables_legumes
+		= get_inherited_property_from_categories_tags($product_ref,
+		"nutriscore_without_unprocessed_fruits_vegetables_legumes:en");
+	if (    (defined $nutriscore_without_unprocessed_fruits_vegetables_legumes)
+		and ($nutriscore_without_unprocessed_fruits_vegetables_legumes eq "yes"))
+	{
+		return 0;
+	}
+
 	if (defined $product_ref->{nutriments}{"fruits-vegetables-nuts-dried" . $prepared . "_100g"}) {
 		$fruits = 2 * $product_ref->{nutriments}{"fruits-vegetables-nuts-dried" . $prepared . "_100g"};
 		add_tag($product_ref, "misc", "en:nutrition-fruits-vegetables-nuts-dried");
@@ -1224,6 +1234,16 @@ Differences with the 2021 version:
 
 sub compute_nutriscore_2023_fruits_vegetables_legumes ($product_ref, $prepared) {
 
+	# If the product is in a category that has no unprocessed fruits/vegetables/nuts, return 0
+	my $nutriscore_without_unprocessed_fruits_vegetables_legumes
+		= get_inherited_property_from_categories_tags($product_ref,
+		"nutriscore_without_unprocessed_fruits_vegetables_legumes:en");
+	if (    (defined $nutriscore_without_unprocessed_fruits_vegetables_legumes)
+		and ($nutriscore_without_unprocessed_fruits_vegetables_legumes eq "yes"))
+	{
+		return 0;
+	}
+
 	my $fruits_vegetables_legumes = deep_get($product_ref, "nutriments",
 		"fruits-vegetables-legumes-estimate-from-ingredients" . $prepared . "_100g");
 

lib/ProductOpener/Ingredients.pm

@@ -6542,6 +6542,10 @@ sub detect_allergens_from_text ($product_ref) {
 
 Recursive function to compute the percentage of ingredients that match a specific function.
 
+The match function takes 2 arguments:
+- ingredient id
+- processing (comma separated list of ingredients_processing taxonomy entries)
+
 Used to compute % of fruits and vegetables, % of milk etc. which is needed by some algorithm
 like the Nutri-Score.
 
@@ -6552,7 +6556,7 @@ sub add_ingredients_matching_function ($ingredients_ref, $match_function_ref) {
 	my $count = 0;
 
 	foreach my $ingredient_ref (@{$ingredients_ref}) {
-		my $match = $match_function_ref->($ingredient_ref->{id});
+		my $match = $match_function_ref->($ingredient_ref->{id}, $ingredient_ref->{processing});
 		if ($match) {
 			if (defined $ingredient_ref->{percent}) {
 				$count += $ingredient_ref->{percent};
@@ -6642,19 +6646,46 @@ sub estimate_ingredients_matching_function ($product_ref, $match_function_ref, $
 	return $count;
 }
 
-=head2 is_fruits_vegetables_nuts_olive_walnut_rapeseed_oils ( $ingredient_id )
+=head2 is_fruits_vegetables_nuts_olive_walnut_rapeseed_oils ( $ingredient_id, $processing = undef )
 
 Determine if an ingredient should be counted as "fruits, vegetables, nuts, olive / walnut / rapeseed oils"
 in Nutriscore 2021 algorithm.
 
+- we use the nutriscore_fruits_vegetables_nuts:en property to identify qualifying ingredients
+- we check that the parent of those ingredients is not a flour
+- we check that the ingredient does not have a processing like en:powder
+
+NUTRI-SCORE FREQUENTLY ASKED QUESTIONS - UPDATED 27/09/2022:
+
+"However, fruits, vegetables and pulses that are subject to further processing (e.g. concentrated fruit juice
+sugars, powders, freeze-drying, candied fruits, fruits in stick form, flours leading to loss of water) do not
+count. As an example, corn in the form of popcorn or soy proteins cannot be considered as vegetables.
+Regarding the frying process, fried vegetables which are thick and only partially dehydrated by the process
+can be taken into account, whereas crisps which are thin and completely dehydrated are excluded."
+
 =cut
 
-sub is_fruits_vegetables_nuts_olive_walnut_rapeseed_oils ($ingredient_id) {
+my $further_processing_regexp = 'en:candied|en:flour|en:freeze-dried|en:powder';
+
+sub is_fruits_vegetables_nuts_olive_walnut_rapeseed_oils ($ingredient_id, $processing = undef) {
 
 	my $nutriscore_fruits_vegetables_nuts
 		= get_inherited_property("ingredients", $ingredient_id, "nutriscore_fruits_vegetables_nuts:en");
 
-	return (((defined $nutriscore_fruits_vegetables_nuts) and ($nutriscore_fruits_vegetables_nuts eq "yes")) or 0);
+	# Check that the ingredient is not further processed
+	my $is_a_further_processed_ingredient = is_a("ingredients", $ingredient_id, "en:flour");
+
+	my $further_processed = ((defined $processing) and ($processing =~ /\b($further_processing_regexp)\b/));
+
+	return (
+		(
+					(defined $nutriscore_fruits_vegetables_nuts)
+				and ($nutriscore_fruits_vegetables_nuts eq "yes")
+				and (not $is_a_further_processed_ingredient)
+				and (not $further_processed)
+		)
+			or 0
+	);
 }
 
 =head2 estimate_nutriscore_2021_fruits_vegetables_nuts_percent_from_ingredients ( product_ref )
@@ -6678,11 +6709,15 @@ sub estimate_nutriscore_2021_fruits_vegetables_nuts_percent_from_ingredients ($p
 
 }
 
-=head2 is_fruits_vegetables_legumes ( $ingredient_id )
+=head2 is_fruits_vegetables_legumes ( $ingredient_id, $processing = undef )
 
 Determine if an ingredient should be counted as "fruits, vegetables, legumes"
 in Nutriscore 2023 algorithm.
 
+- we use the eurocode_2_group_1:en and eurocode_2_group_2:en  property to identify qualifying ingredients
+- we check that the parent of those ingredients is not a flour
+- we check that the ingredient does not have a processing like en:powder
+
 1.2.2. Ingredients contributing to the "Fruit, vegetables and legumes" component
 
 The list of ingredients qualifying for the "Fruit, vegetables and legumes" component has been revised
@@ -6712,6 +6747,16 @@ o 9.60 (Fruit mixtures).
 Pulses groups
 o 7.10 (Pulses).
 
+--
+
+NUTRI-SCORE FREQUENTLY ASKED QUESTIONS - UPDATED 27/09/2022:
+
+"However, fruits, vegetables and pulses that are subject to further processing (e.g. concentrated fruit juice
+sugars, powders, freeze-drying, candied fruits, fruits in stick form, flours leading to loss of water) do not
+count. As an example, corn in the form of popcorn or soy proteins cannot be considered as vegetables.
+Regarding the frying process, fried vegetables which are thick and only partially dehydrated by the process
+can be taken into account, whereas crisps which are thin and completely dehydrated are excluded."
+
 =cut
 
 my %fruits_vegetables_legumes_eurocodes = (
@@ -6737,19 +6782,28 @@ my %fruits_vegetables_legumes_eurocodes = (
 	"7.10" => 1,
 );
 
-sub is_fruits_vegetables_legumes ($ingredient_id) {
+sub is_fruits_vegetables_legumes ($ingredient_id, $processing = undef) {
 
 	my $eurocode_2_group_1 = get_inherited_property("ingredients", $ingredient_id, "eurocode_2_group_1:en");
 	my $eurocode_2_group_2 = get_inherited_property("ingredients", $ingredient_id, "eurocode_2_group_2:en");
 
+	# Check that the ingredient is not further processed
+	my $is_a_further_processed_ingredient = is_a("ingredients", $ingredient_id, "en:flour");
+
+	my $further_processed = ((defined $processing) and ($processing =~ /\b($further_processing_regexp)\b/));
+
 	return (
 		(
-			# All fruits groups
-			# TODO: check that we don't have entries under en:fruits that are in fact not listed in Eurocode 9 "Fruits and fruit products"
-			((defined $eurocode_2_group_1) and ($eurocode_2_group_1 eq "9"))
-				# Vegetables and legumes
-				or ((defined $eurocode_2_group_2)
-				and (exists $fruits_vegetables_legumes_eurocodes{$eurocode_2_group_2}))
+			(
+				# All fruits groups
+				# TODO: check that we don't have entries under en:fruits that are in fact not listed in Eurocode 9 "Fruits and fruit products"
+				((defined $eurocode_2_group_1) and ($eurocode_2_group_1 eq "9"))
+					# Vegetables and legumes
+					or ((defined $eurocode_2_group_2)
+					and (exists $fruits_vegetables_legumes_eurocodes{$eurocode_2_group_2}))
+			)
+				and (not $is_a_further_processed_ingredient)
+				and (not $further_processed)
 		)
 			or 0
 	);
@@ -6773,13 +6827,13 @@ sub estimate_nutriscore_2023_fruits_vegetables_legumes_percent_from_ingredients
 	);
 }
 
-=head2 is_milk ( $ingredient_id )
+=head2 is_milk ( $ingredient_id, $processing = undef )
 
 Determine if an ingredient should be counted as milk in Nutriscore 2021 algorithm
 
 =cut
 
-sub is_milk ($ingredient_id) {
+sub is_milk ($ingredient_id, $processing = undef) {
 
 	return is_a("ingredients", $ingredient_id, "en:milk");
 }
@@ -6804,7 +6858,7 @@ Determine if an ingredient should be counted as red meat in Nutriscore 2023 algo
 
 =cut
 
-sub is_red_meat ($ingredient_id) {
+sub is_red_meat ($ingredient_id, $ingredient_processing = undef) {
 
 	my $red_meat_property = get_inherited_property("ingredients", $ingredient_id, "nutriscore_red_meat:en");
 	if ((defined $red_meat_property) and ($red_meat_property eq "yes")) {

taxonomies/categories.txt

@@ -35186,6 +35186,7 @@ wikidata:en:Q1163138
 ciqual_food_code:en:31027
 ciqual_food_name:en:Candied fruits
 ciqual_food_name:fr:Fruit confit
+nutriscore_without_unprocessed_fruits_vegetables_legumes:en:yes
 
 <en:Candied fruit
 en:Candied lemon
@@ -39866,6 +39867,7 @@ pl:Chipsy, Czipsy
 ru:Чипсы
 th:มันฝรั่งทอด
 zh:薯片
+nutriscore_without_unprocessed_fruits_vegetables_legumes:en:yes
 
 <en:Crisps
 en:Chickpea crisps
@@ -55811,6 +55813,7 @@ pt:Farinha
 ru:Мука
 tr:Un
 zh:面粉
+nutriscore_without_unprocessed_fruits_vegetables_legumes:en:yes
 
 <en:Flours
 fr:Farines de souchet
@@ -56902,6 +56905,7 @@ oqali_family:en: fr:Aperitifs a croquer - Pop corn
 agribalyse_proxy_food_code:en:9230
 agribalyse_proxy_food_name:en:Pop-corn or oil popped maize, salted
 agribalyse_proxy_food_name:fr:Pop-corn ou Maïs éclaté, à l'huile, salé
+nutriscore_without_unprocessed_fruits_vegetables_legumes:en:yes
 
 <en:Popcorn
 en:Plain popcorn, Unsalted pop-corn, Unsalted air-popped maize
@@ -98836,7 +98840,7 @@ ciqual_food_name:en:Tofu, plain
 ciqual_food_name:fr:Tofu, nature
 nova:en:3
 description:en:Tofu is a food prepared by coagulating soy milk and then pressing the resulting curds into solid white blocks of varying softness. Tofu has very little flavor or smell of its own. Consequently, tofu can be used in both savory or sweet dishes, acting as a bland background for presenting the flavors of the other ingredients used. In order to flavor the tofu it is often marinated in soy sauce, chillis, sesame oil, etc.
-
+nutriscore_without_unprocessed_fruits_vegetables_legumes:en:yes
 
 <en:Tofu
 en:Plain tofu

taxonomies/ingredients.txt

@@ -46382,7 +46382,8 @@ de:Gemüsepulver
 fr:légumes en poudre
 
 <en:vegetable
-fr:légumes frais
+en:fresh vegetables
+fr:légumes frais, légume frais
 nl:verse groenten
 
 
@@ -55732,6 +55733,7 @@ wikidata:en:Q159753
 
 # description:en:A LEGUME is a plant in the family Fabaceae (or Leguminosae), or the fruit or seed of such a plant (also called a pulse).
 # Legumes are grown agriculturally, primarily for human consumption. Grain legumes include beans, lentils, lupins, peas, and peanuts.
+<en:vegetable
 en:legume
 af:Peulgewas
 an:Legumbre

tests/integration/expected_test_results/convert_and_import_excel_file/test/products/3270190128403.json

@@ -648,6 +648,7 @@
    "ingredients_from_palm_oil_tags" : [],
    "ingredients_hierarchy" : [
       "en:green-lentils",
+      "en:vegetable",
       "en:legume",
       "en:pulse",
       "en:lentils"
@@ -664,6 +665,7 @@
    "ingredients_percent_analysis" : 1,
    "ingredients_tags" : [
       "en:green-lentils",
+      "en:vegetable",
       "en:legume",
       "en:pulse",
       "en:lentils"
@@ -682,7 +684,7 @@
    "ingredients_without_ciqual_codes" : [],
    "ingredients_without_ciqual_codes_n" : 0,
    "interface_version_created" : "import_csv_file - version 2019/09/17",
-   "known_ingredients_n" : 4,
+   "known_ingredients_n" : 5,
    "labels" : "Organic, EU Organic, FR-BIO-01, AB Agriculture Biologique, Agriculture France",
    "labels_hierarchy" : [
       "en:organic",
@@ -786,8 +788,8 @@
       "energy_value" : 1409,
       "fruits-vegetables-legumes-estimate-from-ingredients_100g" : 100,
       "fruits-vegetables-legumes-estimate-from-ingredients_serving" : 100,
-      "fruits-vegetables-nuts-estimate-from-ingredients_100g" : 0,
-      "fruits-vegetables-nuts-estimate-from-ingredients_serving" : 0,
+      "fruits-vegetables-nuts-estimate-from-ingredients_100g" : 100,
+      "fruits-vegetables-nuts-estimate-from-ingredients_serving" : 100,
       "nova-group" : 1,
       "nova-group_100g" : 1,
       "nova-group_serving" : 1
@@ -845,7 +847,7 @@
          "data" : {
             "energy" : 1409,
             "fiber" : 0,
-            "fruits_vegetables_nuts_colza_walnut_olive_oils" : 0,
+            "fruits_vegetables_nuts_colza_walnut_olive_oils" : 100,
             "is_beverage" : 0,
             "is_cheese" : 0,
             "is_fat" : 0,
@@ -908,7 +910,7 @@
    "nutrition_score_warning_fruits_vegetables_legumes_estimate_from_ingredients" : 1,
    "nutrition_score_warning_fruits_vegetables_legumes_estimate_from_ingredients_value" : 100,
    "nutrition_score_warning_fruits_vegetables_nuts_estimate_from_ingredients" : 1,
-   "nutrition_score_warning_fruits_vegetables_nuts_estimate_from_ingredients_value" : 0,
+   "nutrition_score_warning_fruits_vegetables_nuts_estimate_from_ingredients_value" : 100,
    "nutrition_score_warning_no_fiber" : 1,
    "origins" : "France",
    "origins_hierarchy" : [

tests/integration/expected_test_results/export_more_fields/export_more_fields.csv

@@ -11,7 +11,7 @@ code	producer_product_id	producer_version_id	lc	product_name_de	product_name_en
 
 Ingredients list (English):
 
-Canola Oil, Water, Garbanzo Beans, Lupin (bean), Mustard Seeds, Grape Vinegar, Lemon Juice, Salt, White Sugar, White Pepper, Garlic Powder, EDTA."			Altramuces, Mostaza	en:lupin,en:mustard			100g	100g			582	kcal	58	g	3	g	9.3	g	1	g	0	g	1	g	640	mg	256	mg	582	kcal	39	g	3	g																																													en:dressings-and-sauces	en:fats-and-sauces,en:dressings-and-sauces		unknown	c	10	d	34	1	11054	-5	-15	1	0																App - yuka, Apps	0
+Canola Oil, Water, Garbanzo Beans, Lupin (bean), Mustard Seeds, Grape Vinegar, Lemon Juice, Salt, White Sugar, White Pepper, Garlic Powder, EDTA."			Altramuces, Mostaza	en:lupin,en:mustard			100g	100g			582	kcal	58	g	3	g	9.3	g	1	g	0	g	1	g	640	mg	256	mg	582	kcal	39	g	3	g																																													en:dressings-and-sauces	en:fats-and-sauces,en:dressings-and-sauces		unknown	c	9	d	34	1	11054	-5	-15	1	0																App - yuka, Apps	0
 5410803950689			fr				Sauce algérienne					Sauce émulsionnée froide piquante aux oignons rôtis		500 ml		Plastique, Bouteille ou Flacon, Flacon	en:plastic,en:bottle-or-vial,en:vial	Halal maïza	halal-maiza		Condiments, Sauces, Sauces algériennes, Epicerie	en:condiments,en:sauces,en:algerian-sauces,fr:epicerie	Halal	en:halal	France	en:france																		100g	100g			521	kcal	49.4	g	4.2	g	17.1	g	12.8	g			1.9	g	2.1	g	0.84	g	521	kcal																																																	en:dressings-and-sauces	en:fats-and-sauces,en:dressings-and-sauces		unknown	e	21	unknown		1		-5	-15	1	0																App - yuka, Apps	0
 27096765			fr				Lait crème							5 x 40 g	40 g	Plastique, Métal, Métaux recyclables, Aluminium	en:plastic,en:metal,en:recyclable-metals,en:aluminium	Château	chateau		Snacks, Snacks sucrés, Cacao et dérivés, Biscuits et gâteaux, Gâteaux, Chocolats, Chocolats au lait, Gâteaux au chocolat	en:snacks,en:sweet-snacks,en:cocoa-and-its-products,en:biscuits-and-cakes,en:cakes,en:chocolates,en:milk-chocolates,en:chocolate-cakes	Agriculture durable, Certifié UTZ	en:sustainable-farming,en:utz-certified	Belgique, France	en:belgium,en:france										Sugar, cocoa butter, cream powder (milk) (15.9%), cocoa paste, whole milk powder (5.5%), skim milk powder, lactose, emulsifier: lecithin (soy), vanilla extract		Sucre, beurre de cacao, crème en poudre (lait) (15,9%), pâte de cacao, lait entier en poudre (5,5%), lait écrémé en poudre, lactose, émulsifiant : lécithine (soja), extrait de vanille		Lait, Soja	en:milk,en:soybeans	"Fruits à coque, Arachides"	en:nuts,en:peanuts	100g	100g			563	kcal	35	g	22	g	52	g	51	g			8	g	0.24	g	0.096	g	563	kcal																																																	en:chocolate-products	en:sugary-snacks,en:chocolate-products	4	en:4-ultra-processed-food-and-drink-products	e	28	d	37	1	23585	-5	-15	1	10																App - yuka, Apps, App - InFood	0
 3270160503070			fr				Oignons rouges émincés surgelés					Oignons surgelés		450 g		Plastique, Surgelé	en:plastic,en:frozen	Picard	picard		"Aliments et boissons à base de végétaux, Aliments d'origine végétale, Condiments, Aliments à base de fruits et de légumes, Plantes condimentaires, Surgelés, Légumes et dérivés, Aliments à base de plantes surgelés, Légumes surgelés, Epicerie"	en:plant-based-foods-and-beverages,en:plant-based-foods,en:condiments,en:fruits-and-vegetables-based-foods,en:culinary-plants,en:frozen-foods,en:vegetables-based-foods,en:frozen-plant-based-foods,en:frozen-vegetables,fr:epicerie			France	en:france	Picard	picard			France, Bénélux			EMB 62863C	emb-62863c			Oignons rouges 100%						100g	100g			41	kcal	0.2	g	0.1	g	7.9	g	5.6	g	1.4	g	1.2	g	0.01	g	0.004	g	41	kcal																																																	en:vegetables	en:fruits-and-vegetables,en:vegetables	1	en:1-unprocessed-or-minimally-processed-foods	a	-5	unknown		1		-5	-15	1	0																App - Yuka, Apps	0

tests/integration/expected_test_results/export_more_fields/rows/7804659650035.json

@@ -70,7 +70,7 @@
    "off:nova_groups" : "",
    "off:nova_groups_tags" : "unknown",
    "off:nutriscore_grade" : "c",
-   "off:nutriscore_score" : "10",
+   "off:nutriscore_score" : "9",
    "origin_fr" : "",
    "origins" : "",
    "origins_tags" : "",

tests/unit/expected_test_results/ingredients_contents/individual_ingredients.json

@@ -59,7 +59,7 @@
       "ingredient" : "legumes",
       "ingredient_id" : "en:legume",
       "is_fruits_vegetables_legumes" : 0,
-      "is_fruits_vegetables_nuts_olive_walnut_rapeseed_oils" : 0,
+      "is_fruits_vegetables_nuts_olive_walnut_rapeseed_oils" : "1",
       "is_milk" : 0
    },
    {