fix: mongodb _id must be a string (#7255)

* fix: mongodb _id must be a string #4667

* find and repair products with _id stored as a number

* print mongodb command

* save _id as string

lib/ProductOpener/Products.pm

@@ -1018,7 +1018,7 @@ sub store_product($user_id, $product_ref, $comment) {
 				return $products_collection->delete_one({"_id" => $product_ref->{_id}});
 			});
 
-			$product_ref->{_id} = $product_ref->{code};
+			$product_ref->{_id} = $product_ref->{code} . ''; # treat id as string;
 
 		}
 		else {
@@ -1132,8 +1132,10 @@ sub store_product($user_id, $product_ref, $comment) {
 	# index for full text search
 	index_product($product_ref);
 
-	# make sure that code is saved as a string, otherwise mongodb saves it as number, and leading 0s are removed
-	$product_ref->{code} = $product_ref->{code} . '';
+	# make sure that the _id and code are saved as a string, otherwise mongodb may save them as numbers
+	# for _id , it makes them possibly non unique, and for code, we would lose leading 0s
+	$product_ref->{_id} .= '';
+	$product_ref->{code} .= '';
 
 	# make sure we have numbers, perl can convert numbers to string depending on the last operation done...
 	$product_ref->{last_modified_t} += 0;

scripts/update_all_products.pl

@@ -133,6 +133,7 @@
 my $fix_nutrition_data = '';
 my $compute_main_countries = '';
 my $prefix_packaging_tags_with_language = '';
+my $fix_non_string_ids = '';
 
 my $query_ref = {};    # filters for mongodb query
 
@@ -163,6 +164,7 @@
 			"fix-missing-lc" => \$fix_missing_lc,
 			"fix-zulu-lang" => \$fix_zulu_lang,
 			"fix-rev-not-incremented" => \$fix_rev_not_incremented,
+			"fix-non-string-ids" => \$fix_non_string_ids,
 			"user-id=s" => \$User_id,
 			"comment=s" => \$comment,
 			"run-ocr" => \$run_ocr,
@@ -224,6 +226,7 @@
 	and (not $run_ocr) and (not $autorotate)
 	and (not $fix_missing_lc) and (not $fix_serving_size_mg_to_ml) and (not $fix_zulu_lang) and (not $fix_rev_not_incremented) and (not $fix_yuka_salt)
 	and (not $fix_spanish_ingredientes) and (not $fix_nutrition_data_per)  and (not $fix_nutrition_data)
+	and (not $fix_non_string_ids)
 	and (not $compute_sort_key)
 	and (not $remove_team) and (not $remove_label) and (not $remove_nutrient)
 	and (not $mark_as_obsolete_since_date) and (not $compute_main_countries)
@@ -296,6 +299,11 @@
 	}
 }
 
+# Query products that have the _id field stored as a number
+if ($fix_non_string_ids) {
+	$query_ref->{_id} = { '$type' => "long"};
+}
+
 # On the producers platform, require --query owners_tags to be set, or the --all-owners field to be set.
 
 if ((defined $server_options{private_products}) and ($server_options{private_products})) {
@@ -1201,10 +1209,11 @@
 				}
 
 				# Store data to mongodb
-				# Make sure product code is saved as string and not a number
+				# Make sure product _id and code are saved as string and not a number
 				# see bug #1077 - https://github.com/openfoodfacts/openfoodfacts-server/issues/1077
 				# make sure that code is saved as a string, otherwise mongodb saves it as number, and leading 0s are removed
-				$product_ref->{code} = $product_ref->{code} . '';
+				$product_ref->{_id} .= '';
+				$product_ref->{code} .= '';
 				$products_collection->replace_one({"_id" => $product_ref->{_id}}, $product_ref, { upsert => 1 });
 			}
 		}
@@ -1266,4 +1275,11 @@
 	}
 }
 
+if ($fix_non_string_ids) {
+	print STDERR "\nproducts stored in MongoDB with a non string _id have been reloaded from .sto files (if the products still exist) and stored with a string _id.\n";
+	print STDERR "products with non string ids can now be deleted from MongoDB with this command:"
+	. 'db.products.remove({_id : { $type : "long" }})' . "\n";
+}
+
+
 exit(0);