fix: parsing of dots in packagins / recycling instructions (#7948)

* fix: parsing of dots in packagins / recycling instructions + better handling of commas
openfoodfacts · Jan 6, 2023 · 1b2e690 · 1b2e690
1 parent 6062126
commit 1b2e690
Show file tree

Hide file tree

Showing 6 changed files with 101 additions and 3 deletions.
diff --git a/lib/ProductOpener/Packaging.pm b/lib/ProductOpener/Packaging.pm
@@ -159,6 +159,9 @@ sub parse_packaging_component_data_from_text_phrase ($text, $text_language) {
 		$text = $';
 	}
 
+	# We might have escaped dots and commas inside numbers from analyze_and_combine_packaging_data()
+	$text =~ s/(\d)\\(\.|\,)(\d)/$1$2$3/g;
+
 	# Also try to match the canonicalized form so that we can match the extended synonyms that are only available in canonicalized form
 	my $textid = get_string_id_for_lang($text_language, $text);
 
@@ -793,7 +796,13 @@ sub analyze_and_combine_packaging_data ($product_ref, $response_ref) {
 	# Packaging text field (populated by OCR of the packaging image and/or contributors or producers)
 	if (defined $product_ref->{packaging_text}) {
 
-		my @packaging_text_entries = split(/,|;|\n/, $product_ref->{packaging_text});
+		# Separate phrases by matching:
+		# . , ; and newlines
+		# but we want to keep commas and dots that are inside numbers (3.40 or 1,5)
+		# so we escape them first
+		my $packaging_text = $product_ref->{packaging_text};
+		$packaging_text =~ s/(\d)(\.|,)(\d)/$1\\$2$3/g;
+		my @packaging_text_entries = split(/(?<!\\)\.|(?<!\\),|;|\n/, $packaging_text);
 		push(@phrases, @packaging_text_entries);
 		$number_of_packaging_text_entries = scalar @packaging_text_entries;
 	}

diff --git a/tests/unit/expected_test_results/packaging/fr-comma-inside-a-number.json b/tests/unit/expected_test_results/packaging/fr-comma-inside-a-number.json
@@ -0,0 +1,20 @@
+{
+   "lc" : "fr",
+   "misc_tags" : [
+      "en:packagings-not-complete",
+      "en:packagings-not-empty-but-not-complete",
+      "en:packagings-not-empty"
+   ],
+   "packaging_text" : "6 bouteilles en plastique transparent PET de 1,5 L à recycler",
+   "packagings" : [
+      {
+         "material" : "en:pet-polyethylene-terephthalate",
+         "number_of_units" : 6,
+         "quantity_per_unit" : "1,5 l",
+         "quantity_per_unit_unit" : "l",
+         "quantity_per_unit_value" : 1.5,
+         "recycling" : "en:recycle",
+         "shape" : "en:bottle"
+      }
+   ]
+}
diff --git a/tests/unit/expected_test_results/packaging/fr-comma-without-space.json b/tests/unit/expected_test_results/packaging/fr-comma-without-space.json
@@ -0,0 +1,21 @@
+{
+   "lc" : "fr",
+   "misc_tags" : [
+      "en:packagings-not-complete",
+      "en:packagings-not-empty-but-not-complete",
+      "en:packagings-not-empty"
+   ],
+   "packaging_text" : "1 boîte en métal,4 bouteilles (plastique).",
+   "packagings" : [
+      {
+         "material" : "en:metal",
+         "number_of_units" : 1,
+         "shape" : "en:box"
+      },
+      {
+         "material" : "en:plastic",
+         "number_of_units" : 4,
+         "shape" : "en:bottle"
+      }
+   ]
+}
diff --git a/tests/unit/expected_test_results/packaging/fr-dot-to-separate-components.json b/tests/unit/expected_test_results/packaging/fr-dot-to-separate-components.json
@@ -0,0 +1,21 @@
+{
+   "lc" : "fr",
+   "misc_tags" : [
+      "en:packagings-not-complete",
+      "en:packagings-not-empty-but-not-complete",
+      "en:packagings-not-empty"
+   ],
+   "packaging_text" : "Film plastique à jeter. Étui carton à recycler.",
+   "packagings" : [
+      {
+         "material" : "en:plastic",
+         "recycling" : "en:discard",
+         "shape" : "en:film"
+      },
+      {
+         "material" : "en:cardboard",
+         "recycling" : "en:recycle",
+         "shape" : "en:sleeve"
+      }
+   ]
+}
diff --git a/tests/unit/expected_test_results/packaging/packaging_text_fr_multiple_semi_colon.json b/tests/unit/expected_test_results/packaging/packaging_text_fr_multiple_semi_colon.json
@@ -5,7 +5,7 @@
       "en:packagings-not-empty-but-not-complete",
       "en:packagings-not-empty"
    ],
-   "packaging_text" : "barquette en plastique à jeter;film plastique à jeter; boîte en carton à recycler",
+   "packaging_text" : "barquette en plastique à jeter; film plastique à jeter; boîte en carton à recycler",
    "packagings" : [
       {
          "material" : "en:plastic",

diff --git a/tests/unit/packaging.t b/tests/unit/packaging.t
@@ -86,7 +86,7 @@ boîte en carton à recycler"
 		'packaging_text_fr_multiple_semi_colon',
 		{
 			lc => "fr",
-			packaging_text => "barquette en plastique à jeter;film plastique à jeter; boîte en carton à recycler"
+			packaging_text => "barquette en plastique à jeter; film plastique à jeter; boîte en carton à recycler"
 		}
 	],
 	[
@@ -517,6 +517,33 @@ boîte en carton à recycler"
 		}
 	],
 
+	# dots were not parsed correctly
+	[
+		'fr-dot-to-separate-components',
+		{
+			lc => "fr",
+			packaging_text => "Film plastique à jeter. Étui carton à recycler.",
+		}
+	],
+
+	# comma inside a number: don't split
+	[
+		'fr-comma-inside-a-number',
+		{
+			lc => "fr",
+			packaging_text => "6 bouteilles en plastique transparent PET de 1,5 L à recycler",
+		}
+	],
+
+	# comma without spaces, not in a number: split
+	[
+		'fr-comma-without-space',
+		{
+			lc => "fr",
+			packaging_text => "1 boîte en métal,4 bouteilles (plastique).",
+		}
+	],
+
 );
 
 my $json = JSON->new->allow_nonref->canonical;