From 84aae09cccfc862d5a33b914e61d58bcfe0d6171 Mon Sep 17 00:00:00 2001 From: joce Date: Sun, 15 Mar 2015 01:38:35 +0100 Subject: [PATCH 1/5] [-] BO : Speed up search indexing & reduce memory usage (PSCSX-5039) --- classes/Search.php | 87 +++++++++++++++------------------------------- 1 file changed, 28 insertions(+), 59 deletions(-) diff --git a/classes/Search.php b/classes/Search.php index dc4785f68a87d..b4c06d9b82681 100644 --- a/classes/Search.php +++ b/classes/Search.php @@ -221,7 +221,7 @@ public static function find($id_lang, $expr, $page_number = 1, $page_size = 1, $ return ($ajax ? array() : array('total' => 0, 'result' => array())); $score = ''; - if (count($score_array)) + if (is_array($score_array) && !empty($score_array)) $score = ',( SELECT SUM(weight) FROM '._DB_PREFIX_.'search_word sw @@ -488,26 +488,20 @@ public static function indexation($full = false, $id_product = false) } else { - // Do it even if you already know the product id in order to be sure that it exists and it needs to be indexed - $products = $db->executeS(' - SELECT p.id_product - FROM '._DB_PREFIX_.'product p + $db->execute('DELETE si FROM `'._DB_PREFIX_.'search_index` si + INNER JOIN `'._DB_PREFIX_.'product` p ON (p.id_product = si.id_product) '.Shop::addSqlAssociation('product', 'p').' - INNER JOIN '._DB_PREFIX_.'product_lang pl ON pl.`id_shop` = product_shop.`id_shop` - WHERE product_shop.visibility IN ("both", "search") + WHERE product_shop.`visibility` IN ("both", "search") AND product_shop.`active` = 1 - AND '.($id_product ? 'p.id_product = '.(int)$id_product : 'product_shop.indexed = 0') - ); - - $ids = array(); - if ($products) - foreach ($products as $product) - $ids[] = (int)$product['id_product']; - if (count($ids)) - { - $db->execute('DELETE FROM '._DB_PREFIX_.'search_index WHERE id_product IN ('.implode(',', $ids).')'); - ObjectModel::updateMultishopTable('Product', array('indexed' => 0), 'a.id_product IN ('.implode(',', $ids).')'); - } + AND '.($id_product ? 'p.`id_product` = '.(int)$id_product : 'product_shop.`indexed` = 0')); + + $db->execute('UPDATE `'._DB_PREFIX_.'search_index` si + INNER JOIN `'._DB_PREFIX_.'product` p ON (p.id_product = si.id_product) + '.Shop::addSqlAssociation('product', 'p').' + SET p.`indexed` = 0, product_shop.`indexed` = 0 + WHERE product_shop.`visibility` IN ("both", "search") + AND product_shop.`active` = 1 + AND '.($id_product ? 'p.`id_product` = '.(int)$id_product : 'product_shop.`indexed` = 0')); } // Every fields are weighted according to the configuration in the backend @@ -534,18 +528,6 @@ public static function indexation($full = false, $id_product = false) $count_words = 0; $query_array3 = array(); - // Every indexed words are cached into a PHP array - $word_ids = $db->executeS(' - SELECT id_word, word, id_lang, id_shop - FROM '._DB_PREFIX_.'search_word', false); - $word_ids_by_word = array(); - while ($word_id = $db->nextRow($word_ids)) - { - if (!isset($word_ids_by_word[$word_id['id_shop']][$word_id['id_lang']])) - $word_ids_by_word[$word_id['id_shop']][$word_id['id_lang']] = array(); - $word_ids_by_word[$word_id['id_shop']][$word_id['id_lang']]['_'.$word_id['word']] = (int)$word_id['id_word']; - } - // Retrieve the number of languages $total_languages = count(Language::getLanguages(false)); @@ -583,49 +565,35 @@ public static function indexation($full = false, $id_product = false) } // If we find words that need to be indexed, they're added to the word table in the database - if (count($product_array)) + if (is_array($product_array) && !empty($product_array)) { $query_array = $query_array2 = array(); foreach ($product_array as $word => $weight) - if ($weight && !isset($word_ids_by_word['_'.$word])) + if ($weight) { $query_array[$word] = '('.(int)$product['id_lang'].', '.(int)$product['id_shop'].', \''.pSQL($word).'\')'; $query_array2[] = '\''.pSQL($word).'\''; - $word_ids_by_word[$product['id_shop']][$product['id_lang']]['_'.$word] = 0; } - if ($query_array2) - { - $existing_words = $db->executeS(' - SELECT DISTINCT word FROM '._DB_PREFIX_.'search_word - WHERE word IN ('.implode(',', $query_array2).') - AND id_lang = '.(int)$product['id_lang'].' - AND id_shop = '.(int)$product['id_shop']); - - foreach ($existing_words as $data) - unset($query_array[Tools::replaceAccentedChars($data['word'])]); - } - - if (count($query_array)) + if (is_array($query_array) && !empty($query_array)) { // The words are inserted... $db->execute(' INSERT IGNORE INTO '._DB_PREFIX_.'search_word (id_lang, id_shop, word) VALUES '.implode(',', $query_array)); } - if (count($query_array2)) + $word_ids_by_word = array(); + if (is_array($query_array2) && !empty($query_array2)) { - // ...then their IDs are retrieved and added to the cache + // ...then their IDs are retrieved $added_words = $db->executeS(' SELECT sw.id_word, sw.word FROM '._DB_PREFIX_.'search_word sw WHERE sw.word IN ('.implode(',', $query_array2).') AND sw.id_lang = '.(int)$product['id_lang'].' - AND sw.id_shop = '.(int)$product['id_shop'].' - LIMIT '.count($query_array2)); - // replace accents from the retrieved words so that words without accents or with differents accents can still be linked + AND sw.id_shop = '.(int)$product['id_shop']); foreach ($added_words as $word_id) - $word_ids_by_word[$product['id_shop']][$product['id_lang']]['_'.Tools::replaceAccentedChars($word_id['word'])] = (int)$word_id['id_word']; + $word_ids_by_word['_'.$word_id['word']] = (int)$word_id['id_word']; } } @@ -633,12 +601,13 @@ public static function indexation($full = false, $id_product = false) { if (!$weight) continue; - if (!isset($word_ids_by_word[$product['id_shop']][$product['id_lang']]['_'.$word])) + if (!isset($word_ids_by_word['_'.$word])) continue; - if (!$word_ids_by_word[$product['id_shop']][$product['id_lang']]['_'.$word]) + $id_word = $word_ids_by_word['_'.$word]; + if (!$id_word) continue; $query_array3[] = '('.(int)$product['id_product'].','. - (int)$word_ids_by_word[$product['id_shop']][$product['id_lang']]['_'.$word].','.(int)$weight.')'; + (int)$id_word.','.(int)$weight.')'; // Force save every 200 words in order to avoid overloading MySQL if (++$count_words % 200 == 0) Search::saveIndex($query_array3); @@ -657,7 +626,7 @@ public static function indexation($full = false, $id_product = false) public static function removeProductsSearchIndex($products) { - if (count($products)) + if (is_array($products) && !empty($products)) { Db::getInstance()->execute('DELETE FROM '._DB_PREFIX_.'search_index WHERE id_product IN ('.implode(',', array_map('intval', $products)).')'); ObjectModel::updateMultishopTable('Product', array('indexed' => 0), 'a.id_product IN ('.implode(',', array_map('intval', $products)).')'); @@ -666,14 +635,14 @@ public static function removeProductsSearchIndex($products) protected static function setProductsAsIndexed(&$products) { - if (count($products)) + if (is_array($products) && !empty($products)) ObjectModel::updateMultishopTable('Product', array('indexed' => 1), 'a.id_product IN ('.implode(',', $products).')'); } /** $queryArray3 is automatically emptied in order to be reused immediatly */ protected static function saveIndex(&$queryArray3) { - if (count($queryArray3)) + if (is_array($queryArray3) && !empty($queryArray3)) Db::getInstance()->execute( 'INSERT INTO '._DB_PREFIX_.'search_index (id_product, id_word, weight) VALUES '.implode(',', $queryArray3).' From ec832202f64981c8fe5864bbac40f3d81949052c Mon Sep 17 00:00:00 2001 From: joce Date: Sun, 15 Mar 2015 02:06:08 +0100 Subject: [PATCH 2/5] // post review fixes --- classes/Search.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/classes/Search.php b/classes/Search.php index b4c06d9b82681..8af9cc3d79ea9 100644 --- a/classes/Search.php +++ b/classes/Search.php @@ -495,8 +495,7 @@ public static function indexation($full = false, $id_product = false) AND product_shop.`active` = 1 AND '.($id_product ? 'p.`id_product` = '.(int)$id_product : 'product_shop.`indexed` = 0')); - $db->execute('UPDATE `'._DB_PREFIX_.'search_index` si - INNER JOIN `'._DB_PREFIX_.'product` p ON (p.id_product = si.id_product) + $db->execute('UPDATE `'._DB_PREFIX_.'product` p '.Shop::addSqlAssociation('product', 'p').' SET p.`indexed` = 0, product_shop.`indexed` = 0 WHERE product_shop.`visibility` IN ("both", "search") From d226da634a308c9efb01ce75beb5edd16a95f141 Mon Sep 17 00:00:00 2001 From: joce Date: Sun, 15 Mar 2015 02:08:54 +0100 Subject: [PATCH 3/5] // post review fixes --- classes/Search.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/Search.php b/classes/Search.php index 8af9cc3d79ea9..b349b77325b30 100644 --- a/classes/Search.php +++ b/classes/Search.php @@ -590,7 +590,7 @@ public static function indexation($full = false, $id_product = false) FROM '._DB_PREFIX_.'search_word sw WHERE sw.word IN ('.implode(',', $query_array2).') AND sw.id_lang = '.(int)$product['id_lang'].' - AND sw.id_shop = '.(int)$product['id_shop']); + AND sw.id_shop = '.(int)$product['id_shop'], true, false); foreach ($added_words as $word_id) $word_ids_by_word['_'.$word_id['word']] = (int)$word_id['id_word']; } From 1fb2e8c7313e0c23788cc760ba68ce92064d5ce1 Mon Sep 17 00:00:00 2001 From: joce Date: Sun, 15 Mar 2015 16:33:49 +0100 Subject: [PATCH 4/5] [-] BO : Fix attribute indexing (PSCSX-5028). Adapted from PR #2651 --- classes/Search.php | 141 +++++++++++++++++++++------- install-dev/data/db_structure.sql | 3 +- install-dev/upgrade/sql/1.6.1.0.sql | 4 +- 3 files changed, 112 insertions(+), 36 deletions(-) diff --git a/classes/Search.php b/classes/Search.php index b349b77325b30..828fe5b1806d6 100644 --- a/classes/Search.php +++ b/classes/Search.php @@ -397,12 +397,58 @@ public static function getFeatures($db, $id_product, $id_lang) return $features; } + /** + * @param $weight_array + * @return string + */ + protected static function getSQLProductAttributeFields(&$weight_array) + { + $sql = ''; + if (is_array($weight_array)) + foreach ($weight_array as $key => $weight) + if ((int)$weight) + switch ($key) + { + case 'pa_reference': + $sql .= ', pa.reference AS pa_reference'; + break; + case 'pa_supplier_reference': + $sql .= ', pa.supplier_reference AS pa_supplier_reference'; + break; + case 'pa_ean13': + $sql .= ', pa.ean13 AS pa_ean13'; + break; + case 'pa_upc': + $sql .= ', pa.upc AS pa_upc'; + break; + } + return $sql; + } + protected static function getProductsToIndex($total_languages, $id_product = false, $limit = 50, $weight_array = array()) { - // Adjust the limit to get only "whole" products, in every languages (and at least one) - $max_possibilities = $total_languages * count(Shop::getShops(true)); - $limit = max($max_possibilities, floor($limit / $max_possibilities) * $max_possibilities); + $ids = null; + if (!$id_product) + { + // Limit products for each step but be sure that each attribute is taken into account + $sql = 'SELECT p.id_product FROM '._DB_PREFIX_.'product p + '.Shop::addSqlAssociation('product', 'p', true, null, true).' + WHERE product_shop.`indexed` = 0 + AND product_shop.`visibility` IN ("both", "search") + AND product_shop.`active` = 1 + ORDER BY product_shop.`id_product` ASC + LIMIT '.(int)$limit; + + + $res = Db::getInstance()->executeS($sql, false); + while($row = Db::getInstance()->nextRow($res)) + { + $ids[] = $row['id_product']; + } + + } + // Now get every attribute in every language $sql = 'SELECT p.id_product, pl.id_lang, pl.id_shop, l.iso_code'; if (is_array($weight_array)) @@ -416,27 +462,15 @@ protected static function getProductsToIndex($total_languages, $id_product = fal case 'reference': $sql .= ', p.reference'; break; - case 'pa_reference': - $sql .= ', pa.reference AS pa_reference'; - break; case 'supplier_reference': $sql .= ', p.supplier_reference'; break; - case 'pa_supplier_reference': - $sql .= ', pa.supplier_reference AS pa_supplier_reference'; - break; case 'ean13': $sql .= ', p.ean13'; break; - case 'pa_ean13': - $sql .= ', pa.ean13 AS pa_ean13'; - break; case 'upc': $sql .= ', p.upc'; break; - case 'pa_upc': - $sql .= ', pa.upc AS pa_upc'; - break; case 'description_short': $sql .= ', pl.description_short'; break; @@ -452,8 +486,6 @@ protected static function getProductsToIndex($total_languages, $id_product = fal } $sql .= ' FROM '._DB_PREFIX_.'product p - LEFT JOIN '._DB_PREFIX_.'product_attribute pa - ON pa.id_product = p.id_product LEFT JOIN '._DB_PREFIX_.'product_lang pl ON p.id_product = pl.id_product '.Shop::addSqlAssociation('product', 'p', true, null, true).' @@ -466,13 +498,52 @@ protected static function getProductsToIndex($total_languages, $id_product = fal WHERE product_shop.indexed = 0 AND product_shop.visibility IN ("both", "search") '.($id_product ? 'AND p.id_product = '.(int)$id_product : '').' + '.($ids ? 'AND p.id_product IN ('.implode(',', array_map('intval', $ids)).')' : '').' AND product_shop.`active` = 1 - AND pl.`id_shop` = product_shop.`id_shop` - LIMIT '.(int)$limit; + AND pl.`id_shop` = product_shop.`id_shop`'; return Db::getInstance()->executeS($sql); } + /** + * @param $db + * @param $id_product + * @param $sql_attribute + * @return mixed + */ + protected static function getAttributesFields($db, $id_product, $sql_attribute) + { + return $db->executeS('SELECT id_product '.$sql_attribute.' FROM '. + _DB_PREFIX_.'product_attribute pa WHERE pa.id_product = '.(int)$id_product); + } + + /** + * @param $product_array + * @param $weight_array + * @param $key + * @param $value + * @param $id_lang + * @param $iso_code + */ + protected static function fillProductArray(&$product_array, $weight_array, $key, $value, $id_lang, $iso_code) + { + if (strncmp($key, 'id_', 3) && isset($weight_array[$key])) + { + $words = explode(' ', Search::sanitize($value, (int)$id_lang, true, $iso_code)); + foreach ($words as $word) + if (!empty($word)) + { + $word = Tools::substr($word, 0, PS_SEARCH_MAX_WORD_LENGTH); + // Remove accents + $word = Tools::replaceAccentedChars($word); + + if (!isset($product_array[$word])) + $product_array[$word] = 0; + $product_array[$word] += $weight_array[$key]; + } + } + } + public static function indexation($full = false, $id_product = false) { $db = Db::getInstance(); @@ -530,6 +601,7 @@ public static function indexation($full = false, $id_product = false) // Retrieve the number of languages $total_languages = count(Language::getLanguages(false)); + $sql_attribute = Search::getSQLProductAttributeFields($weight_array); // Products are processed 50 by 50 in order to avoid overloading MySQL while (($products = Search::getProductsToIndex($total_languages, $id_product, 50, $weight_array)) && (count($products) > 0)) { @@ -543,25 +615,26 @@ public static function indexation($full = false, $id_product = false) $product['attributes'] = Search::getAttributes($db, (int)$product['id_product'], (int)$product['id_lang']); if ((int)$weight_array['features']) $product['features'] = Search::getFeatures($db, (int)$product['id_product'], (int)$product['id_lang']); + if ($sql_attribute) + { + $attribute_fields = Search::getAttributesFields($db, (int)$product['id_product'], $sql_attribute); + if ($attribute_fields) + $product['attributes_fields'] = $attribute_fields; + } // Data must be cleaned of html, bad characters, spaces and anything, then if the resulting words are long enough, they're added to the array $product_array = array(); foreach ($product as $key => $value) - if (strncmp($key, 'id_', 3) && isset($weight_array[$key])) + { + if ($key == 'attributes_fields') { - $words = explode(' ', Search::sanitize($value, (int)$product['id_lang'], true, $product['iso_code'])); - foreach ($words as $word) - if (!empty($word)) - { - $word = Tools::substr($word, 0, PS_SEARCH_MAX_WORD_LENGTH); - // Remove accents - $word = Tools::replaceAccentedChars($word); - - if (!isset($product_array[$word])) - $product_array[$word] = 0; - $product_array[$word] += $weight_array[$key]; - } + foreach($value as $pa_array) + foreach($pa_array as $pa_key => $pa_value) + Search::fillProductArray($product_array, $weight_array, $pa_key, $pa_value, $product['id_lang'], $product['iso_code']); } + else + Search::fillProductArray($product_array, $weight_array, $key, $value, $product['id_lang'], $product['iso_code']); + } // If we find words that need to be indexed, they're added to the word table in the database if (is_array($product_array) && !empty($product_array)) @@ -612,9 +685,9 @@ public static function indexation($full = false, $id_product = false) Search::saveIndex($query_array3); } - if (!in_array($product['id_product'], $products_array)) - $products_array[] = (int)$product['id_product']; + $products_array[] = (int)$product['id_product']; } + $products_array = array_unique($products_array); Search::setProductsAsIndexed($products_array); // One last save is done at the end in order to save what's left diff --git a/install-dev/data/db_structure.sql b/install-dev/data/db_structure.sql index 06db3f2381b6b..e9dd3af2a174b 100644 --- a/install-dev/data/db_structure.sql +++ b/install-dev/data/db_structure.sql @@ -1512,7 +1512,8 @@ CREATE TABLE IF NOT EXISTS `PREFIX_product_shop` ( `pack_stock_type` int(11) unsigned DEFAULT '3' NOT NULL, PRIMARY KEY (`id_product`, `id_shop`), KEY `id_category_default` (`id_category_default`), - KEY `date_add` (`date_add` , `active` , `visibility`) + KEY `date_add` (`date_add` , `active` , `visibility`), + KEY `indexed` (`indexed`, `active`, `id_product`) ) ENGINE=ENGINE_TYPE DEFAULT CHARSET=utf8 COLLATION; CREATE TABLE `PREFIX_product_attribute` ( diff --git a/install-dev/upgrade/sql/1.6.1.0.sql b/install-dev/upgrade/sql/1.6.1.0.sql index 24543af1e4bce..8844b22876f3f 100644 --- a/install-dev/upgrade/sql/1.6.1.0.sql +++ b/install-dev/upgrade/sql/1.6.1.0.sql @@ -162,4 +162,6 @@ ALTER TABLE `PREFIX_customer` DROP KEY `id_shop`; ALTER TABLE `PREFIX_customer` ADD KEY `id_shop` (`id_shop`, `date_add`); ALTER TABLE `PREFIX_cart` DROP KEY `id_shop`; -ALTER TABLE `PREFIX_cart` ADD KEY `id_shop_2` (`id_shop`,`date_upd`), ADD KEY `id_shop` (`id_shop`,`date_add`); \ No newline at end of file +ALTER TABLE `PREFIX_cart` ADD KEY `id_shop_2` (`id_shop`,`date_upd`), ADD KEY `id_shop` (`id_shop`,`date_add`); + +ALTER TABLE `PREFIX_product_shop` ADD KEY `indexed` (`indexed`, `active`, `id_product`); \ No newline at end of file From 5f186c1324368e45fefd61e7b5073018cc8d2cd1 Mon Sep 17 00:00:00 2001 From: joce Date: Fri, 20 Mar 2015 10:37:28 +0100 Subject: [PATCH 5/5] // post review fixes --- classes/Search.php | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/classes/Search.php b/classes/Search.php index 828fe5b1806d6..694d79c35cd47 100644 --- a/classes/Search.php +++ b/classes/Search.php @@ -441,11 +441,8 @@ protected static function getProductsToIndex($total_languages, $id_product = fal $res = Db::getInstance()->executeS($sql, false); - while($row = Db::getInstance()->nextRow($res)) - { + while ($row = Db::getInstance()->nextRow($res)) $ids[] = $row['id_product']; - } - } // Now get every attribute in every language