feat: Refactor CSV exports, add tests, export Eco-Score fields (#6444)

* new misc facet values for Eco-Score missing packaging warning

* more ecoscore missing data facets

* update tests

* refactor on demand csv/xlsx export on website - wip

* export off derived fields

* export Eco-Score data

* prefix off computed fields by off:, add _tags fields

* fix filehandle

* add tests for CSV exports #6424

* Update labeler.yml

* start mongodb

* update tests

* remove condition on hardcoded domain name

* feat: add missing_key_data flag to ecoscore

* export more ecoscore fields

* fix tests

* Update lib/ProductOpener/Export.pm

Co-authored-by: Alex Garel <alex@garel.org>

* Update t/export.t

Co-authored-by: Alex Garel <alex@garel.org>

* suggestions from code review

* create Test.pm module

* create Test.pm module

Co-authored-by: Pierre Slamich <pierre@openfoodfacts.org>
Co-authored-by: Alex Garel <alex@garel.org>

.github/labeler.yml

@@ -89,6 +89,7 @@ ecoscore:
 
 Export:
 - lib/ProductOpener/Export.pm
+- t/expected_test_results/export/**/*
 
 Food.pm:
 - lib/ProductOpener/Food.pm

.github/workflows/pull_request.yml

@@ -42,6 +42,11 @@ jobs:
     - uses: actions/setup-node@v3
       with:
         node-version: '12.x'
+
+    - name: Start MongoDB
+      uses: supercharge/mongodb-github-action@1.7.0
+      with:
+        mongodb-version: 4.4
 
     - run: npm install
 

cpanfile

@@ -54,6 +54,7 @@ requires 'Modern::Perl', '>= 1.20200211'; # libmodern-perl-perl has 1.20170117/1
 requires 'Data::Dumper::AutoEncode'; # deps: libmodule-build-pluggable-perl, libclass-accessor-lite-perl
 requires 'XML::Rules';
 requires 'Text::CSV', '>= 2.01, < 3.0'; # libtext-csv-perl has 1.95/1.99 vs 2.00.
+requires 'Text::CSV_XS';
 requires 'Text::Fuzzy';
 requires 'Spreadsheet::CSV'; # deps: libspreadsheet-parseexcel-perl
 requires 'File::chmod::Recursive'; # deps: libfile-chmod-perl

lib/ProductOpener/Config_off.pm

@@ -646,10 +646,6 @@ $options{categories_exempted_from_nutrient_levels} = [qw(
 	no_nutriments
 	additives_n
 	additives
-	ingredients_from_palm_oil_n
-	ingredients_from_palm_oil
-	ingredients_that_may_be_from_palm_oil_n
-	ingredients_that_may_be_from_palm_oil
 	nutriscore_score
 	nutriscore_grade
 	nova_group
@@ -658,11 +654,14 @@ $options{categories_exempted_from_nutrient_levels} = [qw(
 	food_groups
 	states
 	brand_owner
-	ecoscore_score_fr
-	ecoscore_grade_fr
+	ecoscore_score
+	ecoscore_grade
+	ecoscore_data.agribalyse.code
+	ecoscore_data.adjustments.packaging.value
 );
 
-
+# List of fields that can be imported on the producers platform
+# and that are also exported from the producers platform to the public platform
 $options{import_export_fields_groups} = [
 	[   "identification",
 		[   "code",                      "producer_product_id",
@@ -710,6 +709,25 @@ $options{import_export_fields_groups} = [
 	],
 ];
 
+# Secondary fields that are computed by OFF from primary data
+# Those fields are only exported, they are not imported.
+$options{off_export_fields_groups} = [
+	[   "off",
+		[
+			"nutriscore_grade",
+			"nutriscore_score",
+			"ecoscore_grade",
+			"ecoscore_score",
+			"ecoscore_data.agribalyse.code",
+			"ecoscore_data.adjustments.origins_of_ingredients.value",
+			"ecoscore_data.adjustments.packaging.value",
+			"ecoscore_data.adjustments.packaging.non_recyclable_and_non_biodegradable_materials",
+			"ecoscore_data.adjustments.production_system.value",
+			"ecoscore_data.adjustments.threatened_species.value",
+		]
+	],
+];
+
 # Used to generate the list of possible product attributes, which is
 # used to display the possible choices for user preferences
 $options{attribute_groups} = [