fix: obsolete support in update_all_products.pl and export status fix

lib/ProductOpener/Import.pm

@@ -2824,14 +2824,12 @@ sub update_export_status_for_csv_file ($args_ref) {
 	my $columns_ref = $csv->getline($io);
 	$csv->column_names(@{deduped_colnames($columns_ref)});
 
-	my $products_collection = get_products_collection();
-
 	while (my $imported_product_ref = $csv->getline_hr($io)) {
 
 		$i++;
 
 		# By default, use the orgid passed in the arguments
-		# it may be overrode later on a per product basis
+		# it may be overridden later on a per product basis
 		my $org_id = $args_ref->{org_id};
 
 		# The option import_owner is used when exporting from the producers database to the public database
@@ -2880,9 +2878,12 @@ sub update_export_status_for_csv_file ($args_ref) {
 			$product_ref->{states} = join(',', @{$product_ref->{states_tags}});
 			compute_field_tags($product_ref, $product_ref->{lc}, "states");
 
+			# Update the product without creating a new revision
 			my $path = product_path($product_ref);
 			store("$data_root/products/$path/product.sto", $product_ref);
 			$product_ref->{code} = $product_ref->{code} . '';
+			# Use the obsolete collection if the product is obsolete
+			my $products_collection = get_products_collection({obsolete => $product_ref->{obsolete}});
 			$products_collection->replace_one({"_id" => $product_ref->{_id}}, $product_ref, {upsert => 1});
 		}
 	}

scripts/update_all_products.pl

@@ -136,6 +136,8 @@
 my $prefix_packaging_tags_with_language = '';
 my $fix_non_string_ids = '';
 my $assign_ciqual_codes = '';
+my $obsolete = 0;
+my $fix_obsolete;
 
 my $query_ref = {};    # filters for mongodb query
 
@@ -190,6 +192,8 @@
 	"compute-main-countries" => \$compute_main_countries,
 	"prefix-packaging-tags-with-language" => \$prefix_packaging_tags_with_language,
 	"assign-ciqual-codes" => \$assign_ciqual_codes,
+	"obsolete" => \$obsolete,
+	"fix-obsolete" => \$fix_obsolete,
 ) or die("Error in command line arguments:\n\n$usage");
 
 use Data::Dumper;
@@ -261,7 +265,8 @@
 	and (not $count)
 	and (not $just_print_codes)
 	and (not $prefix_packaging_tags_with_language)
-	and (not $assign_ciqual_codes))
+	and (not $assign_ciqual_codes)
+	and (not $fix_obsolete))
 {
 	die("Missing fields to update or --count option:\n$usage");
 }
@@ -352,7 +357,15 @@
 print STDERR "MongoDB query:\n" . Dumper($query_ref);
 
 my $socket_timeout_ms = 2 * 60000;    # 2 mins, instead of 30s default, to not die as easily if mongodb is busy.
-my $products_collection = get_products_collection({timeout => $socket_timeout_ms});
+
+# Collection that will be used to iterate products
+my $products_collection = get_products_collection({obsolete => $obsolete, timeout => $socket_timeout_ms});
+
+# Collections for saving current / obsolete products
+my %products_collections = (
+	current => get_products_collection({timeout => $socket_timeout_ms}),
+	obsolete => get_products_collection({obsolete => $obsolete, timeout => $socket_timeout_ms}),
+);
 
 my $products_count = "";
 
@@ -379,6 +392,7 @@
 my $m = 0;    # number of products with a new version created
 
 my $fix_rev_not_incremented_fixed = 0;
+my $fix_obsolete_fixed = 0;
 
 # Used to get stats on fields deleted by an user
 my %deleted_fields = ();
@@ -1345,7 +1359,25 @@
 				# make sure that code is saved as a string, otherwise mongodb saves it as number, and leading 0s are removed
 				$product_ref->{_id} .= '';
 				$product_ref->{code} .= '';
-				$products_collection->replace_one({"_id" => $product_ref->{_id}}, $product_ref, {upsert => 1});
+				my $collection = "current";
+				if ($product_ref->{obsolete}) {
+					$collection = "obsolete";
+				}
+				$products_collections{$collection}
+					->replace_one({"_id" => $product_ref->{_id}}, $product_ref, {upsert => 1});
+
+				# If the obsolete flag of the product does not
+				# correspond to the collection we are iterating over
+				# delete the product from the collection
+				if (
+					$fix_obsolete
+					and (  ($obsolete and not $product_ref->{obsolete})
+						or ((not $obsolete) and $product_ref->{obsolete}))
+					)
+				{
+					$products_collection->delete_one({"_id" => $product_ref->{_id}});
+					$fix_obsolete_fixed++;
+				}
 			}
 		}
 
@@ -1395,6 +1427,10 @@
 
 print "$n products updated (pretend: $pretend) - $m new versions created\n";
 
+if ($fix_obsolete_fixed) {
+	print "$fix_obsolete_fixed removed from wrong collection (obsolete or current)\n";
+}
+
 if ($fix_rev_not_incremented_fixed) {
 	print "$fix_rev_not_incremented_fixed rev fixed\n";
 }