Add a string encoding selector to the product importer (#36819)

woocommerce · Feb 24, 2023 · af7c3f3 · af7c3f3
2 parents 7f5b65b + 6cd66c5
commit af7c3f3
Show file tree

Hide file tree

Showing 7 changed files with 111 additions and 43 deletions.
diff --git a/plugins/woocommerce/changelog/add-encoding-selector-to-product-importer b/plugins/woocommerce/changelog/add-encoding-selector-to-product-importer
@@ -0,0 +1,4 @@
+Significance: minor
+Type: add
+
+Add an encoding selector to the product importer
diff --git a/plugins/woocommerce/client/legacy/js/admin/wc-product-import.js b/plugins/woocommerce/client/legacy/js/admin/wc-product-import.js
@@ -5,14 +5,15 @@
 	 * productImportForm handles the import process.
 	 */
 	var productImportForm = function( $form ) {
-		this.$form           = $form;
-		this.xhr             = false;
-		this.mapping         = wc_product_import_params.mapping;
-		this.position        = 0;
-		this.file            = wc_product_import_params.file;
-		this.update_existing = wc_product_import_params.update_existing;
-		this.delimiter       = wc_product_import_params.delimiter;
-		this.security        = wc_product_import_params.import_nonce;
+		this.$form              = $form;
+		this.xhr                = false;
+		this.mapping            = wc_product_import_params.mapping;
+		this.position           = 0;
+		this.file               = wc_product_import_params.file;
+		this.update_existing    = wc_product_import_params.update_existing;
+		this.delimiter          = wc_product_import_params.delimiter;
+		this.security           = wc_product_import_params.import_nonce;
+		this.character_encoding = wc_product_import_params.character_encoding;
 
 		// Number of import successes/failures.
 		this.imported = 0;
@@ -39,13 +40,14 @@
 			type: 'POST',
 			url: ajaxurl,
 			data: {
-				action          : 'woocommerce_do_ajax_product_import',
-				position        : $this.position,
-				mapping         : $this.mapping,
-				file            : $this.file,
-				update_existing : $this.update_existing,
-				delimiter       : $this.delimiter,
-				security        : $this.security
+				action            : 'woocommerce_do_ajax_product_import',
+				position          : $this.position,
+				mapping           : $this.mapping,
+				file              : $this.file,
+				update_existing   : $this.update_existing,
+				delimiter         : $this.delimiter,
+				security          : $this.security,
+				character_encoding: $this.character_encoding
 			},
 			dataType: 'json',
 			success: function( response ) {

diff --git a/plugins/woocommerce/includes/admin/class-wc-admin-importers.php b/plugins/woocommerce/includes/admin/class-wc-admin-importers.php
@@ -147,11 +147,13 @@ public function tax_rates_importer() {
 	public function post_importer_compatibility() {
 		global $wpdb;
 
-		if ( empty( $_POST['import_id'] ) || ! class_exists( 'WXR_Parser' ) ) { // PHPCS: input var ok, CSRF ok.
+		// phpcs:ignore WordPress.Security.NonceVerification.Missing
+		if ( empty( $_POST['import_id'] ) || ! class_exists( 'WXR_Parser' ) ) {
 			return;
 		}
 
-		$id          = absint( $_POST['import_id'] ); // PHPCS: input var ok.
+		// phpcs:ignore WordPress.Security.NonceVerification.Missing
+		$id          = absint( $_POST['import_id'] );
 		$file        = get_attached_file( $id );
 		$parser      = new WXR_Parser();
 		$import_data = $parser->parse( $file );
@@ -216,12 +218,21 @@ public function do_ajax_product_import() {
 
 		$file   = wc_clean( wp_unslash( $_POST['file'] ) ); // PHPCS: input var ok.
 		$params = array(
-			'delimiter'       => ! empty( $_POST['delimiter'] ) ? wc_clean( wp_unslash( $_POST['delimiter'] ) ) : ',', // PHPCS: input var ok.
-			'start_pos'       => isset( $_POST['position'] ) ? absint( $_POST['position'] ) : 0, // PHPCS: input var ok.
-			'mapping'         => isset( $_POST['mapping'] ) ? (array) wc_clean( wp_unslash( $_POST['mapping'] ) ) : array(), // PHPCS: input var ok.
-			'update_existing' => isset( $_POST['update_existing'] ) ? (bool) $_POST['update_existing'] : false, // PHPCS: input var ok.
-			'lines'           => apply_filters( 'woocommerce_product_import_batch_size', 30 ),
-			'parse'           => true,
+			'delimiter'          => ! empty( $_POST['delimiter'] ) ? wc_clean( wp_unslash( $_POST['delimiter'] ) ) : ',', // PHPCS: input var ok.
+			'start_pos'          => isset( $_POST['position'] ) ? absint( $_POST['position'] ) : 0, // PHPCS: input var ok.
+			'mapping'            => isset( $_POST['mapping'] ) ? (array) wc_clean( wp_unslash( $_POST['mapping'] ) ) : array(), // PHPCS: input var ok.
+			'update_existing'    => isset( $_POST['update_existing'] ) ? (bool) $_POST['update_existing'] : false, // PHPCS: input var ok.
+			'character_encoding' => isset( $_POST['character_encoding'] ) ? wc_clean( wp_unslash( $_POST['character_encoding'] ) ) : '',
+
+			/**
+			 * Batch size for the product import process.
+			 *
+			 * @param int $size Batch size.
+			 *
+			 * @since
+			 */
+			'lines'              => apply_filters( 'woocommerce_product_import_batch_size', 30 ),
+			'parse'              => true,
 		);
 
 		// Log failures.

diff --git a/plugins/woocommerce/includes/admin/importers/class-wc-product-csv-importer-controller.php b/plugins/woocommerce/includes/admin/importers/class-wc-product-csv-importer-controller.php
@@ -72,6 +72,13 @@ class WC_Product_CSV_Importer_Controller {
 	 */
 	protected $update_existing = false;
 
+	/**
+	 * The character encoding to use to interpret the input file, or empty string for autodetect.
+	 *
+	 * @var string
+	 */
+	protected $character_encoding = 'UTF-8';
+
 	/**
 	 * Get importer instance.
 	 *
@@ -141,11 +148,12 @@ public function __construct() {
 		$this->steps = apply_filters( 'woocommerce_product_csv_importer_steps', $default_steps );
 
 		// phpcs:disable WordPress.Security.NonceVerification.Recommended
-		$this->step            = isset( $_REQUEST['step'] ) ? sanitize_key( $_REQUEST['step'] ) : current( array_keys( $this->steps ) );
-		$this->file            = isset( $_REQUEST['file'] ) ? wc_clean( wp_unslash( $_REQUEST['file'] ) ) : '';
-		$this->update_existing = isset( $_REQUEST['update_existing'] ) ? (bool) $_REQUEST['update_existing'] : false;
-		$this->delimiter       = ! empty( $_REQUEST['delimiter'] ) ? wc_clean( wp_unslash( $_REQUEST['delimiter'] ) ) : ',';
-		$this->map_preferences = isset( $_REQUEST['map_preferences'] ) ? (bool) $_REQUEST['map_preferences'] : false;
+		$this->step               = isset( $_REQUEST['step'] ) ? sanitize_key( $_REQUEST['step'] ) : current( array_keys( $this->steps ) );
+		$this->file               = isset( $_REQUEST['file'] ) ? wc_clean( wp_unslash( $_REQUEST['file'] ) ) : '';
+		$this->update_existing    = isset( $_REQUEST['update_existing'] ) ? (bool) $_REQUEST['update_existing'] : false;
+		$this->delimiter          = ! empty( $_REQUEST['delimiter'] ) ? wc_clean( wp_unslash( $_REQUEST['delimiter'] ) ) : ',';
+		$this->map_preferences    = isset( $_REQUEST['map_preferences'] ) ? (bool) $_REQUEST['map_preferences'] : false;
+		$this->character_encoding = isset( $_REQUEST['character_encoding'] ) ? wc_clean( wp_unslash( $_REQUEST['character_encoding'] ) ) : 'UTF-8';
 		// phpcs:enable
 
 		// Import mappings for CSV data.
@@ -182,12 +190,13 @@ public function get_next_step_link( $step = '' ) {
 		}
 
 		$params = array(
-			'step'            => $keys[ $step_index + 1 ],
-			'file'            => str_replace( DIRECTORY_SEPARATOR, '/', $this->file ),
-			'delimiter'       => $this->delimiter,
-			'update_existing' => $this->update_existing,
-			'map_preferences' => $this->map_preferences,
-			'_wpnonce'        => wp_create_nonce( 'woocommerce-csv-importer' ), // wp_nonce_url() escapes & to &amp; breaking redirects.
+			'step'               => $keys[ $step_index + 1 ],
+			'file'               => str_replace( DIRECTORY_SEPARATOR, '/', $this->file ),
+			'delimiter'          => $this->delimiter,
+			'update_existing'    => $this->update_existing,
+			'map_preferences'    => $this->map_preferences,
+			'character_encoding' => $this->character_encoding,
+			'_wpnonce'           => wp_create_nonce( 'woocommerce-csv-importer' ), // wp_nonce_url() escapes & to &amp; breaking redirects.
 		);
 
 		return add_query_arg( $params );
@@ -367,8 +376,9 @@ public function handle_upload() {
 	protected function mapping_form() {
 		check_admin_referer( 'woocommerce-csv-importer' );
 		$args = array(
-			'lines'     => 1,
-			'delimiter' => $this->delimiter,
+			'lines'              => 1,
+			'delimiter'          => $this->delimiter,
+			'character_encoding' => $this->character_encoding,
 		);
 
 		$importer     = self::get_importer( $this->file, $args );
@@ -430,14 +440,15 @@ public function import() {
 			'wc-product-import',
 			'wc_product_import_params',
 			array(
-				'import_nonce'    => wp_create_nonce( 'wc-product-import' ),
-				'mapping'         => array(
+				'import_nonce'       => wp_create_nonce( 'wc-product-import' ),
+				'mapping'            => array(
 					'from' => $mapping_from,
 					'to'   => $mapping_to,
 				),
-				'file'            => $this->file,
-				'update_existing' => $this->update_existing,
-				'delimiter'       => $this->delimiter,
+				'file'               => $this->file,
+				'update_existing'    => $this->update_existing,
+				'delimiter'          => $this->delimiter,
+				'character_encoding' => $this->character_encoding,
 			)
 		);
 		wp_enqueue_script( 'wc-product-import' );

diff --git a/plugins/woocommerce/includes/admin/importers/views/html-csv-import-mapping.php b/plugins/woocommerce/includes/admin/importers/views/html-csv-import-mapping.php
@@ -60,6 +60,9 @@
 		<input type="hidden" name="file" value="<?php echo esc_attr( $this->file ); ?>" />
 		<input type="hidden" name="delimiter" value="<?php echo esc_attr( $this->delimiter ); ?>" />
 		<input type="hidden" name="update_existing" value="<?php echo (int) $this->update_existing; ?>" />
+		<?php if ( $args['character_encoding'] ) { ?>
+			<input type="hidden" name="character_encoding" value="<?php echo esc_html( $args['character_encoding'] ); ?>" />
+		<?php } ?>
 		<?php wp_nonce_field( 'woocommerce-csv-importer' ); ?>
 	</div>
 </form>
diff --git a/plugins/woocommerce/includes/admin/importers/views/html-product-csv-import-form.php b/plugins/woocommerce/includes/admin/importers/views/html-product-csv-import-form.php
@@ -78,6 +78,20 @@
 					<th><label><?php esc_html_e( 'Use previous column mapping preferences?', 'woocommerce' ); ?></label><br/></th>
 					<td><input type="checkbox" id="woocommerce-importer-map-preferences" name="map_preferences" value="1" /></td>
 				</tr>
+				<tr class="woocommerce-importer-advanced hidden">
+					<th><label><?php esc_html_e( 'Character encoding of the file', 'woocommerce' ); ?></label><br/></th>
+					<td><select id="woocommerce-importer-character-encoding" name="character_encoding">
+							<option value="" selected><?php esc_html_e( 'Autodetect', 'woocommerce' ); ?></option>
+							<?php
+							$encodings = mb_list_encodings();
+							sort( $encodings, SORT_NATURAL );
+							foreach ( $encodings as $encoding ) {
+								echo '<option>' . esc_html( $encoding ) . '</option>';
+							}
+							?>
+						</select>
+					</td>
+				</tr>
 			</tbody>
 		</table>
 	</section>

diff --git a/plugins/woocommerce/includes/import/class-wc-product-csv-importer.php b/plugins/woocommerce/includes/import/class-wc-product-csv-importer.php
@@ -6,6 +6,8 @@
  * @version 3.1.0
  */
 
+use Automattic\WooCommerce\Utilities\ArrayUtil;
+
 if ( ! defined( 'ABSPATH' ) ) {
 	exit;
 }
@@ -66,6 +68,17 @@ public function __construct( $file, $params = array() ) {
 		$this->read_file();
 	}
 
+	/**
+	 * Convert a string from the input encoding to UTF-8.
+	 *
+	 * @param string $value The string to convert.
+	 * @return string The converted string.
+	 */
+	private function adjust_character_encoding( $value ) {
+		$encoding = $this->params['character_encoding'];
+		return 'UTF-8' === $encoding ? $value : mb_convert_encoding( $value, 'UTF-8', $encoding );
+	}
+
 	/**
 	 * Read file.
 	 */
@@ -77,7 +90,11 @@ protected function read_file() {
 		$handle = fopen( $this->file, 'r' ); // @codingStandardsIgnoreLine.
 
 		if ( false !== $handle ) {
-			$this->raw_keys = version_compare( PHP_VERSION, '5.3', '>=' ) ? array_map( 'trim', fgetcsv( $handle, 0, $this->params['delimiter'], $this->params['enclosure'], $this->params['escape'] ) ) : array_map( 'trim', fgetcsv( $handle, 0, $this->params['delimiter'], $this->params['enclosure'] ) ); // @codingStandardsIgnoreLine
+			$this->raw_keys = array_map( 'trim', fgetcsv( $handle, 0, $this->params['delimiter'], $this->params['enclosure'], $this->params['escape'] ) ); // @codingStandardsIgnoreLine
+
+			if ( ArrayUtil::is_truthy( $this->params, 'character_encoding' ) ) {
+				$this->raw_keys = array_map( array( $this, 'adjust_character_encoding' ), $this->raw_keys );
+			}
 
 			// Remove line breaks in keys, to avoid mismatch mapping of keys.
 			$this->raw_keys = wc_clean( wp_unslash( $this->raw_keys ) );
@@ -92,9 +109,13 @@ protected function read_file() {
 			}
 
 			while ( 1 ) {
-				$row = version_compare( PHP_VERSION, '5.3', '>=' ) ? fgetcsv( $handle, 0, $this->params['delimiter'], $this->params['enclosure'], $this->params['escape'] ) : fgetcsv( $handle, 0, $this->params['delimiter'], $this->params['enclosure'] ); // @codingStandardsIgnoreLine
+				$row = fgetcsv( $handle, 0, $this->params['delimiter'], $this->params['enclosure'], $this->params['escape'] ); // @codingStandardsIgnoreLine
 
 				if ( false !== $row ) {
+					if ( ArrayUtil::is_truthy( $this->params, 'character_encoding' ) ) {
+						$row = array_map( array( $this, 'adjust_character_encoding' ), $row );
+					}
+
 					$this->raw_data[]                                 = $row;
 					$this->file_positions[ count( $this->raw_data ) ] = ftell( $handle );
 
@@ -1005,6 +1026,8 @@ protected function set_parsed_data() {
 			 *
 			 * @param array $parsed_data Parsed data.
 			 * @param WC_Product_Importer $importer Importer instance.
+			 *
+			 * @since
 			 */
 			$this->parsed_data[] = apply_filters( 'woocommerce_product_importer_parsed_data', $this->expand_data( $data ), $this );
 		}