From 0ee2dbc4a66fda7dfc8aa9da69b5cd5d0f80e33b Mon Sep 17 00:00:00 2001 From: Pierre RAMBAUD Date: Thu, 10 Jan 2019 12:20:05 +0100 Subject: [PATCH 1/4] Robots.txt with multilang/multishop Add the correct path to Disallow directories when using a multishop or a multilang instance. --- classes/Tools.php | 131 ++++++++++++------- controllers/admin/AdminShopUrlController.php | 1 + 2 files changed, 85 insertions(+), 47 deletions(-) diff --git a/classes/Tools.php b/classes/Tools.php index 75f8a735fd203..0b3c7684271dd 100644 --- a/classes/Tools.php +++ b/classes/Tools.php @@ -2321,6 +2321,44 @@ public static function getMediaServer($filename) return Tools::usingSecureMode() ? Tools::getShopDomainSsl() : Tools::getShopDomain(); } + /** + * Get domains information + * + * @return array + */ + public static function getDomains() + { + $domains = []; + foreach (ShopUrl::getShopUrls() as $shop_url) { + /** @var ShopUrl $shop_url */ + if (!isset($domains[$shop_url->domain])) { + $domains[$shop_url->domain] = []; + } + + $domains[$shop_url->domain][] = [ + 'physical' => $shop_url->physical_uri, + 'virtual' => $shop_url->virtual_uri, + 'id_shop' => $shop_url->id_shop, + ]; + + if ($shop_url->domain == $shop_url->domain_ssl) { + continue; + } + + if (!isset($domains[$shop_url->domain_ssl])) { + $domains[$shop_url->domain_ssl] = []; + } + + $domains[$shop_url->domain_ssl][] = [ + 'physical' => $shop_url->physical_uri, + 'virtual' => $shop_url->virtual_uri, + 'id_shop' => $shop_url->id_shop, + ]; + } + + return $domains; + } + public static function generateHtaccess($path = null, $rewrite_settings = null, $cache_control = null, $specific = '', $disable_multiviews = null, $medias = false, $disable_modsec = null) { if (defined('_PS_IN_TEST_') @@ -2370,33 +2408,7 @@ public static function generateHtaccess($path = null, $rewrite_settings = null, fwrite($write_fd, trim($specific_before) . "\n\n"); } - $domains = array(); - foreach (ShopUrl::getShopUrls() as $shop_url) { - /** @var ShopUrl $shop_url */ - if (!isset($domains[$shop_url->domain])) { - $domains[$shop_url->domain] = array(); - } - - $domains[$shop_url->domain][] = array( - 'physical' => $shop_url->physical_uri, - 'virtual' => $shop_url->virtual_uri, - 'id_shop' => $shop_url->id_shop, - ); - - if ($shop_url->domain == $shop_url->domain_ssl) { - continue; - } - - if (!isset($domains[$shop_url->domain_ssl])) { - $domains[$shop_url->domain_ssl] = array(); - } - - $domains[$shop_url->domain_ssl][] = array( - 'physical' => $shop_url->physical_uri, - 'virtual' => $shop_url->virtual_uri, - 'id_shop' => $shop_url->id_shop, - ); - } + $domains = self::getDomains(); // Write data in .htaccess file fwrite($write_fd, "# ~~start~~ Do not remove this comment, Prestashop will keep automatically the code outside this comment when .htaccess will be generated again\n"); @@ -2437,7 +2449,7 @@ public static function generateHtaccess($path = null, $rewrite_settings = null, foreach ($medias as $media) { foreach ($media as $media_url) { if ($media_url) { - $media_domains .= 'RewriteCond %{HTTP_HOST} ^' . $media_url . '$ [OR]' . "\n"; + $media_domains .= 'RewriteCond %{HTTP_HOST} ^' . $media_url . '$ [OR]' . PHP_EOL; } } } @@ -2447,13 +2459,12 @@ public static function generateHtaccess($path = null, $rewrite_settings = null, } foreach ($domains as $domain => $list_uri) { - $physicals = array(); foreach ($list_uri as $uri) { fwrite($write_fd, PHP_EOL . PHP_EOL . '#Domain: ' . $domain . PHP_EOL); if (Shop::isFeatureActive()) { - fwrite($write_fd, 'RewriteCond %{HTTP_HOST} ^' . $domain . '$' . "\n"); + fwrite($write_fd, 'RewriteCond %{HTTP_HOST} ^' . $domain . '$' . PHP_EOL); } - fwrite($write_fd, 'RewriteRule . - [E=REWRITEBASE:' . $uri['physical'] . ']' . "\n"); + fwrite($write_fd, 'RewriteRule . - [E=REWRITEBASE:' . $uri['physical'] . ']' . PHP_EOL); // Webservice fwrite($write_fd, 'RewriteRule ^api$ api/ [L]' . "\n\n"); @@ -2463,7 +2474,7 @@ public static function generateHtaccess($path = null, $rewrite_settings = null, $rewrite_settings = (int) Configuration::get('PS_REWRITING_SETTINGS', null, null, (int) $uri['id_shop']); } - $domain_rewrite_cond = 'RewriteCond %{HTTP_HOST} ^' . $domain . '$' . "\n"; + $domain_rewrite_cond = 'RewriteCond %{HTTP_HOST} ^' . $domain . '$' . PHP_EOL; // Rewrite virtual multishop uri if ($uri['virtual']) { if (!$rewrite_settings) { @@ -2486,10 +2497,10 @@ public static function generateHtaccess($path = null, $rewrite_settings = null, if (Configuration::get('PS_LEGACY_IMAGES')) { fwrite($write_fd, $media_domains); fwrite($write_fd, $domain_rewrite_cond); - fwrite($write_fd, 'RewriteRule ^([a-z0-9]+)\-([a-z0-9]+)(\-[_a-zA-Z0-9-]*)(-[0-9]+)?/.+\.jpg$ %{ENV:REWRITEBASE}img/p/$1-$2$3$4.jpg [L]' . "\n"); + fwrite($write_fd, 'RewriteRule ^([a-z0-9]+)\-([a-z0-9]+)(\-[_a-zA-Z0-9-]*)(-[0-9]+)?/.+\.jpg$ %{ENV:REWRITEBASE}img/p/$1-$2$3$4.jpg [L]' . PHP_EOL); fwrite($write_fd, $media_domains); fwrite($write_fd, $domain_rewrite_cond); - fwrite($write_fd, 'RewriteRule ^([0-9]+)\-([0-9]+)(-[0-9]+)?/.+\.jpg$ %{ENV:REWRITEBASE}img/p/$1-$2$3.jpg [L]' . "\n"); + fwrite($write_fd, 'RewriteRule ^([0-9]+)\-([0-9]+)(-[0-9]+)?/.+\.jpg$ %{ENV:REWRITEBASE}img/p/$1-$2$3.jpg [L]' . PHP_EOL); } // Rewrite product images < 100 millions @@ -2506,17 +2517,17 @@ public static function generateHtaccess($path = null, $rewrite_settings = null, } fwrite($write_fd, $media_domains); fwrite($write_fd, $domain_rewrite_cond); - fwrite($write_fd, 'RewriteRule ^c/([0-9]+)(\-[\.*_a-zA-Z0-9-]*)(-[0-9]+)?/.+\.jpg$ %{ENV:REWRITEBASE}img/c/$1$2$3.jpg [L]' . "\n"); + fwrite($write_fd, 'RewriteRule ^c/([0-9]+)(\-[\.*_a-zA-Z0-9-]*)(-[0-9]+)?/.+\.jpg$ %{ENV:REWRITEBASE}img/c/$1$2$3.jpg [L]' . PHP_EOL); fwrite($write_fd, $media_domains); fwrite($write_fd, $domain_rewrite_cond); - fwrite($write_fd, 'RewriteRule ^c/([a-zA-Z_-]+)(-[0-9]+)?/.+\.jpg$ %{ENV:REWRITEBASE}img/c/$1$2.jpg [L]' . "\n"); + fwrite($write_fd, 'RewriteRule ^c/([a-zA-Z_-]+)(-[0-9]+)?/.+\.jpg$ %{ENV:REWRITEBASE}img/c/$1$2.jpg [L]' . PHP_EOL); } fwrite($write_fd, "# AlphaImageLoader for IE and fancybox\n"); if (Shop::isFeatureActive()) { fwrite($write_fd, $domain_rewrite_cond); } - fwrite($write_fd, 'RewriteRule ^images_ie/?([^/]+)\.(jpe?g|png|gif)$ js/jquery/plugins/fancybox/images/$1.$2 [L]' . "\n"); + fwrite($write_fd, 'RewriteRule ^images_ie/?([^/]+)\.(jpe?g|png|gif)$ js/jquery/plugins/fancybox/images/$1.$2 [L]' . PHP_EOL); } // Redirections to dispatcher if ($rewrite_settings) { @@ -2615,6 +2626,7 @@ public static function generateRobotsFile($executeHook = false) } $robots_content = static::getRobotsContent(); + $languagesIsoIds = Language::getIsoIds(); if (true === $executeHook) { Hook::exec('actionAdminMetaBeforeWriteRobotsFile', array( @@ -2639,7 +2651,7 @@ public static function generateRobotsFile($executeHook = false) if (count($robots_content['Allow'])) { fwrite($write_fd, "# Allow Directives\n"); foreach ($robots_content['Allow'] as $allow) { - fwrite($write_fd, 'Allow: ' . $allow . "\n"); + fwrite($write_fd, 'Allow: ' . $allow . PHP_EOL); } } @@ -2647,28 +2659,53 @@ public static function generateRobotsFile($executeHook = false) if (count($robots_content['GB'])) { fwrite($write_fd, "# Private pages\n"); foreach ($robots_content['GB'] as $gb) { - fwrite($write_fd, 'Disallow: /*' . $gb . "\n"); + fwrite($write_fd, 'Disallow: /*' . $gb . PHP_EOL); } } // Directories if (count($robots_content['Directories'])) { - fwrite($write_fd, "# Directories\n"); - foreach ($robots_content['Directories'] as $dir) { - fwrite($write_fd, 'Disallow: /' . $dir . "\n"); + foreach (self::getDomains() as $domain => $uriList) { + fwrite( + $write_fd, + sprintf( + '# Directories for %s%s', + $domain, + PHP_EOL + ) + ); + foreach ($uriList as $uri) { + foreach ($robots_content['Directories'] as $dir) { + fwrite($write_fd, 'Disallow: ' . $uri['physical'] . $dir . PHP_EOL); + } + } + if (!empty($languagesIsoIds)) { + foreach ($languagesIsoIds as $language) { + foreach ($robots_content['Directories'] as $dir) { + fwrite( + $write_fd, + sprintf( + 'Disallow: /%s/%s%s', + $language['iso_code'], + $dir, + PHP_EOL + ) + ); + } + } + } } } // Files if (count($robots_content['Files'])) { - $language_ids = Language::getIDs(); fwrite($write_fd, "# Files\n"); foreach ($robots_content['Files'] as $iso_code => $files) { foreach ($files as $file) { - if (!empty($language_ids) && count($language_ids) > 1) { - fwrite($write_fd, 'Disallow: /*' . $iso_code . '/' . $file . "\n"); + if (!empty($languagesIsoIds) && count($languagesIsoIds) > 1) { + fwrite($write_fd, 'Disallow: /*' . $iso_code . '/' . $file . PHP_EOL); } else { - fwrite($write_fd, 'Disallow: /' . $file . "\n"); + fwrite($write_fd, 'Disallow: /' . $file . PHP_EOL); } } } @@ -2685,7 +2722,7 @@ public static function generateRobotsFile($executeHook = false) fwrite($write_fd, "# Sitemap\n"); $sitemap_filename = basename($sitemap_file); fwrite($write_fd, 'Sitemap: ' . (Configuration::get('PS_SSL_ENABLED') ? 'https://' : 'http://') . $_SERVER['SERVER_NAME'] - . __PS_BASE_URI__ . $sitemap_filename . "\n"); + . __PS_BASE_URI__ . $sitemap_filename . PHP_EOL); } if (true === $executeHook) { diff --git a/controllers/admin/AdminShopUrlController.php b/controllers/admin/AdminShopUrlController.php index e7dcff75a1e57..306b1305ee31b 100644 --- a/controllers/admin/AdminShopUrlController.php +++ b/controllers/admin/AdminShopUrlController.php @@ -466,6 +466,7 @@ public function processSave() $return = parent::processSave(); if (!$this->errors) { Tools::generateHtaccess(); + Tools::generateRobotsFile(); Tools::clearSmartyCache(); Media::clearCache(); } From 9928837087d244b835629265ec5dfd4a3e1615eb Mon Sep 17 00:00:00 2001 From: Pierre RAMBAUD Date: Thu, 10 Jan 2019 12:44:08 +0100 Subject: [PATCH 2/4] Add missing comments --- classes/Tools.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/classes/Tools.php b/classes/Tools.php index 0b3c7684271dd..79cb5aa703e6f 100644 --- a/classes/Tools.php +++ b/classes/Tools.php @@ -2674,11 +2674,14 @@ public static function generateRobotsFile($executeHook = false) PHP_EOL ) ); + // Disallow multishop directories foreach ($uriList as $uri) { foreach ($robots_content['Directories'] as $dir) { fwrite($write_fd, 'Disallow: ' . $uri['physical'] . $dir . PHP_EOL); } } + + // Disallow multilang directories if (!empty($languagesIsoIds)) { foreach ($languagesIsoIds as $language) { foreach ($robots_content['Directories'] as $dir) { From 13f9ed7b08c917b6b0ac639d31bb11ac1d5a32b8 Mon Sep 17 00:00:00 2001 From: Pierre RAMBAUD Date: Thu, 10 Jan 2019 14:17:44 +0100 Subject: [PATCH 3/4] Useless count --- classes/Tools.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/Tools.php b/classes/Tools.php index 79cb5aa703e6f..d8d41c08f73c7 100644 --- a/classes/Tools.php +++ b/classes/Tools.php @@ -2705,7 +2705,7 @@ public static function generateRobotsFile($executeHook = false) fwrite($write_fd, "# Files\n"); foreach ($robots_content['Files'] as $iso_code => $files) { foreach ($files as $file) { - if (!empty($languagesIsoIds) && count($languagesIsoIds) > 1) { + if (!empty($languagesIsoIds)) { fwrite($write_fd, 'Disallow: /*' . $iso_code . '/' . $file . PHP_EOL); } else { fwrite($write_fd, 'Disallow: /' . $file . PHP_EOL); From 91c6617ea12984cbe80cfa24a516d520210654e5 Mon Sep 17 00:00:00 2001 From: Pierre RAMBAUD Date: Tue, 15 Jan 2019 12:01:49 +0100 Subject: [PATCH 4/4] Add example for the getDomains method --- classes/Tools.php | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/classes/Tools.php b/classes/Tools.php index d8d41c08f73c7..fcf1ab9696636 100644 --- a/classes/Tools.php +++ b/classes/Tools.php @@ -2322,7 +2322,15 @@ public static function getMediaServer($filename) } /** - * Get domains information + * Get domains information with physical and virtual paths + * + * e.g: [ + * prestashop.localhost => [ + * physical => "/", + * virtual => "", + * id_shop => "1", + * ] + * ] * * @return array */