Skip to content
Permalink
Browse files

EZP-29139: RegenerateUrlAliasesCommand should keep historized Url Ali…

…ases (#2423)

* EZP-29138: Implemented ezplatform:urls:regenerate-aliases command

* Deprecated the ezplatform:regenerate:legacy_storage_url_aliases command.

* [PAPI] Implemented LocationService::loadAllLocations API

* [PAPI] Implemented ContentService::loadContentListByContentInfo API

* [PAPI] Implemented URLAliasService cleanup APIs

New UrlAliasService APIs:
  * refreshSystemUrlAliasesForLocation
  * deleteCorruptedUrlAliases

* [SPI] Backported Content\Handler::loadContentInfoList

* [SPI] Backported Content\Handler::loadContentList

* Improved URLAlias Gateway loadPathData to throw API BadStateException

* [Tests] Implemented integration tests for new APIs

* [Installer] Removed corrupted URL alias entry from cleandata.sql
  • Loading branch information...
alongosz committed Sep 17, 2018
1 parent eacd1f0 commit 2a3684893a2714a8c5c07ea1f4f5bffa3fb2f94c
Showing with 2,414 additions and 93 deletions.
  1. +0 −1 data/cleandata.sql
  2. +12 −2 eZ/Bundle/EzPublishCoreBundle/ApiLoader/RepositoryFactory.php
  3. +242 −0 eZ/Bundle/EzPublishCoreBundle/Command/RegenerateUrlAliasesCommand.php
  4. +10 −0 eZ/Bundle/EzPublishCoreBundle/Resources/config/commands.yml
  5. +1 −0 eZ/Bundle/EzPublishCoreBundle/Resources/config/papi.yml
  6. +3 −0 eZ/Bundle/EzPublishCoreBundle/Resources/config/services.yml
  7. +14 −1 eZ/Bundle/EzPublishMigrationBundle/Command/LegacyStorage/RegenerateUrlAliasesCommand.php
  8. +17 −0 eZ/Publish/API/Repository/ContentService.php
  9. +19 −0 eZ/Publish/API/Repository/LocationService.php
  10. +40 −0 eZ/Publish/API/Repository/Tests/BaseTest.php
  11. +28 −0 eZ/Publish/API/Repository/Tests/Common/SlugConverter.php
  12. +32 −0 eZ/Publish/API/Repository/Tests/ContentServiceTest.php
  13. +3 −0 eZ/Publish/API/Repository/Tests/SetupFactory/Legacy.php
  14. +3 −0 eZ/Publish/API/Repository/Tests/SetupFactory/LegacyElasticsearch.php
  15. +468 −0 eZ/Publish/API/Repository/Tests/URLAliasServiceTest.php
  16. +16 −0 eZ/Publish/API/Repository/URLAliasService.php
  17. +5 −0 eZ/Publish/API/Repository/Values/Content/Language.php
  18. +12 −0 eZ/Publish/Core/Persistence/Cache/ContentHandler.php
  19. +27 −0 eZ/Publish/Core/Persistence/Cache/LocationHandler.php
  20. +40 −1 eZ/Publish/Core/Persistence/Cache/UrlAliasHandler.php
  21. +22 −0 eZ/Publish/Core/Persistence/Legacy/Content/Gateway.php
  22. +141 −54 eZ/Publish/Core/Persistence/Legacy/Content/Gateway/DoctrineDatabase.php
  23. +25 −0 eZ/Publish/Core/Persistence/Legacy/Content/Gateway/ExceptionConversion.php
  24. +102 −1 eZ/Publish/Core/Persistence/Legacy/Content/Handler.php
  25. +20 −0 eZ/Publish/Core/Persistence/Legacy/Content/Language/MaskGenerator.php
  26. +19 −0 eZ/Publish/Core/Persistence/Legacy/Content/Location/Gateway.php
  27. +73 −0 eZ/Publish/Core/Persistence/Legacy/Content/Location/Gateway/DoctrineDatabase.php
  28. +37 −0 eZ/Publish/Core/Persistence/Legacy/Content/Location/Gateway/ExceptionConversion.php
  29. +25 −0 eZ/Publish/Core/Persistence/Legacy/Content/Location/Handler.php
  30. +3 −1 eZ/Publish/Core/Persistence/Legacy/Content/Mapper.php
  31. +32 −0 eZ/Publish/Core/Persistence/Legacy/Content/UrlAlias/Gateway.php
  32. +261 −12 eZ/Publish/Core/Persistence/Legacy/Content/UrlAlias/Gateway/DoctrineDatabase.php
  33. +58 −0 eZ/Publish/Core/Persistence/Legacy/Content/UrlAlias/Gateway/ExceptionConversion.php
  34. +75 −1 eZ/Publish/Core/Persistence/Legacy/Content/UrlAlias/Handler.php
  35. +3 −2 eZ/Publish/Core/Persistence/Legacy/Tests/Content/UrlAlias/Gateway/DoctrineDatabaseTest.php
  36. +11 −2 eZ/Publish/Core/Persistence/Legacy/Tests/Content/UrlAlias/UrlAliasHandlerTest.php
  37. +3 −0 eZ/Publish/Core/Persistence/Legacy/Tests/TestCase.php
  38. +20 −3 eZ/Publish/Core/REST/Client/ContentService.php
  39. +18 −1 eZ/Publish/Core/REST/Client/LocationService.php
  40. +22 −1 eZ/Publish/Core/REST/Client/URLAliasService.php
  41. +52 −0 eZ/Publish/Core/Repository/ContentService.php
  42. +25 −5 eZ/Publish/Core/Repository/Helper/DomainMapper.php
  43. +82 −1 eZ/Publish/Core/Repository/LocationService.php
  44. +17 −2 eZ/Publish/Core/Repository/Repository.php
  45. +10 −0 eZ/Publish/Core/Repository/Tests/Service/Mock/UrlAliasTest.php
  46. +95 −2 eZ/Publish/Core/Repository/URLAliasService.php
  47. +27 −0 eZ/Publish/Core/SignalSlot/ContentService.php
  48. +25 −0 eZ/Publish/Core/SignalSlot/LocationService.php
  49. +33 −0 eZ/Publish/Core/SignalSlot/URLAliasService.php
  50. +1 −0 eZ/Publish/Core/settings/storage_engines/legacy/content.yml
  51. +1 −0 eZ/Publish/Core/settings/storage_engines/legacy/url_alias.yml
  52. +7 −0 eZ/Publish/Core/settings/tests/common.yml
  53. +9 −0 eZ/Publish/Core/settings/tests/override.yml
  54. +35 −0 eZ/Publish/SPI/Persistence/Content/Handler.php
  55. +17 −0 eZ/Publish/SPI/Persistence/Content/Location/Handler.php
  56. +16 −0 eZ/Publish/SPI/Persistence/Content/UrlAlias/Handler.php
@@ -1928,7 +1928,6 @@ INSERT INTO `ezurlalias_ml` (`action`, `action_type`, `alias_redirects`, `id`, `
INSERT INTO `ezurlalias_ml` (`action`, `action_type`, `alias_redirects`, `id`, `is_alias`, `is_original`, `lang_mask`, `link`, `parent`, `text`, `text_md5`) VALUES ('eznode:51','eznode',1,18,0,1,3,18,9,'Images','59b514174bffe4ae402b3d63aad79fe0');
INSERT INTO `ezurlalias_ml` (`action`, `action_type`, `alias_redirects`, `id`, `is_alias`, `is_original`, `lang_mask`, `link`, `parent`, `text`, `text_md5`) VALUES ('eznode:45','eznode',1,12,0,1,3,12,10,'Anonymous-User','ccb62ebca03a31272430bc414bd5cd5b');
INSERT INTO `ezurlalias_ml` (`action`, `action_type`, `alias_redirects`, `id`, `is_alias`, `is_original`, `lang_mask`, `link`, `parent`, `text`, `text_md5`) VALUES ('eznode:45','eznode',1,31,0,0,1,12,11,'anonymous_user','c593ec85293ecb0e02d50d4c5c6c20eb');
INSERT INTO `ezurlalias_ml` (`action`, `action_type`, `alias_redirects`, `id`, `is_alias`, `is_original`, `lang_mask`, `link`, `parent`, `text`, `text_md5`) VALUES ('nop:','nop',1,15,0,0,1,15,14,'images','59b514174bffe4ae402b3d63aad79fe0');
INSERT INTO `ezurlalias_ml` (`action`, `action_type`, `alias_redirects`, `id`, `is_alias`, `is_original`, `lang_mask`, `link`, `parent`, `text`, `text_md5`) VALUES ('eznode:53','eznode',1,34,0,0,1,20,17,'multimedia','2e5bc8831f7ae6a29530e7f1bbf2de9c');
INSERT INTO `ezurlalias_ml` (`action`, `action_type`, `alias_redirects`, `id`, `is_alias`, `is_original`, `lang_mask`, `link`, `parent`, `text`, `text_md5`) VALUES ('eznode:52','eznode',1,33,0,0,1,19,17,'files','45b963397aa40d4a0063e0d85e4fe7a1');
INSERT INTO `ezurlalias_ml` (`action`, `action_type`, `alias_redirects`, `id`, `is_alias`, `is_original`, `lang_mask`, `link`, `parent`, `text`, `text_md5`) VALUES ('eznode:51','eznode',1,32,0,0,1,18,17,'images','59b514174bffe4ae402b3d63aad79fe0');
@@ -19,6 +19,8 @@
use eZ\Publish\Core\Base\Container\ApiLoader\FieldTypeCollectionFactory;
use eZ\Publish\Core\Base\Container\ApiLoader\FieldTypeNameableCollectionFactory;
use eZ\Publish\Core\Base\Exceptions\InvalidArgumentException;
use Psr\Log\LoggerInterface;
use Psr\Log\NullLogger;
use Symfony\Component\DependencyInjection\ContainerAwareInterface;
use Symfony\Component\DependencyInjection\ContainerAwareTrait;
@@ -62,18 +64,25 @@ class RepositoryFactory implements ContainerAwareInterface
*/
private $policyMap;
/**
* @var \Psr\Log\LoggerInterface
*/
private $logger;
public function __construct(
ConfigResolverInterface $configResolver,
FieldTypeCollectionFactory $fieldTypeCollectionFactory,
FieldTypeNameableCollectionFactory $fieldTypeNameableCollectionFactory,
$repositoryClass,
array $policyMap
array $policyMap,
LoggerInterface $logger = null
) {
$this->configResolver = $configResolver;
$this->fieldTypeCollectionFactory = $fieldTypeCollectionFactory;
$this->fieldTypeNameableCollectionFactory = $fieldTypeNameableCollectionFactory;
$this->repositoryClass = $repositoryClass;
$this->policyMap = $policyMap;
$this->logger = null !== $logger ? $logger : new NullLogger();
}
/**
@@ -112,7 +121,8 @@ public function buildRepository(
'languages' => $this->configResolver->getParameter('languages'),
'content' => ['default_version_archive_limit' => $config['options']['default_version_archive_limit']],
),
new UserReference($this->configResolver->getParameter('anonymous_user_id'))
new UserReference($this->configResolver->getParameter('anonymous_user_id')),
$this->logger
);
return $repository;
@@ -0,0 +1,242 @@
<?php
/**
* @copyright Copyright (C) eZ Systems AS. All rights reserved.
* @license For full copyright and license information view LICENSE file distributed with this source code.
*/
namespace eZ\Bundle\EzPublishCoreBundle\Command;
use Exception;
use eZ\Publish\API\Repository\Repository;
use eZ\Publish\API\Repository\Values\Content\Language;
use eZ\Publish\API\Repository\Values\Content\Location;
use eZ\Publish\Core\MVC\Symfony\Cache\GatewayCachePurger;
use Psr\Log\LoggerInterface;
use Psr\Log\NullLogger;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Helper\ProgressBar;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Question\ConfirmationQuestion;
/**
* The ezplatform:urls:regenerate-aliases Symfony command implementation.
* Recreates system URL aliases for all existing Locations and cleanups corrupted URL alias nodes.
*/
class RegenerateUrlAliasesCommand extends Command
{
const DEFAULT_ITERATION_COUNT = 1000;
const BEFORE_RUNNING_HINTS = <<<EOT
<error>Before you continue:</error>
- Make sure to back up your database.
- Take installation offline, during the script execution the database should not be modified.
- Run this command without memory limit, i.e. processing of 300k Locations can take up to 1 GB of RAM.
- Run this command in production environment using <info>--env=prod</info>
EOT;
/**
* @var \eZ\Publish\API\Repository\Repository
*/
private $repository;
/**
* @var \Psr\Log\LoggerInterface
*/
private $logger;
/**
* @var \eZ\Publish\Core\MVC\Symfony\Cache\GatewayCachePurger
*/
private $cachePurger;
/**
* @param \eZ\Publish\API\Repository\Repository $repository
* @param \eZ\Publish\Core\MVC\Symfony\Cache\GatewayCachePurger $cachePurger
* @param \Psr\Log\LoggerInterface $logger
*/
public function __construct(
Repository $repository,
GatewayCachePurger $cachePurger,
LoggerInterface $logger = null
) {
parent::__construct();
$this->repository = $repository;
$this->cachePurger = $cachePurger;
$this->logger = null !== $logger ? $logger : new NullLogger();
}
/**
* {@inheritdoc}
*/
protected function configure()
{
$beforeRunningHints = self::BEFORE_RUNNING_HINTS;
$this
->setName('ezplatform:urls:regenerate-aliases')
->setDescription(
'Regenerates Location URL aliases (autogenerated) and cleans up custom Location ' .
'and global URL aliases stored in the Legacy Storage Engine'
)
->addOption(
'iteration-count',
'c',
InputOption::VALUE_OPTIONAL,
'Number of Locations fetched into memory and processed at once',
self::DEFAULT_ITERATION_COUNT
)
->setHelp(
<<<EOT
{$beforeRunningHints}
The command <info>%command.name%</info> regenerates URL aliases for Locations and cleans up
corrupted URL aliases (pointing to non-existent Locations).
Existing aliases are archived (will redirect to the new ones).
Note: This script can potentially run for a very long time.
Due to performance issues the command does not send any Signals.
EOT
);
}
/**
* Regenerate URL aliases.
*
* {@inheritdoc}
*/
protected function execute(InputInterface $input, OutputInterface $output)
{
$iterationCount = (int)$input->getOption('iteration-count');
$locationsCount = $this->repository->sudo(
function (Repository $repository) {
return $repository->getLocationService()->getAllLocationsCount();
}
);
$helper = $this->getHelper('question');
$question = new ConfirmationQuestion(
sprintf(
"<info>Found %d Locations.</info>\n%s\n<info>Do you want to proceed? [y/N] </info>",
$locationsCount,
self::BEFORE_RUNNING_HINTS
),
false
);
if (!$helper->ask($input, $output, $question)) {
return;
}
$output->writeln('<info>Cleaning up corrupted URL aliases...</info>');
$corruptedAliasesCount = $this->repository->sudo(
function (Repository $repository) {
return $repository->getURLAliasService()->deleteCorruptedUrlAliases();
}
);
$output->writeln("<info>Done. Deleted {$corruptedAliasesCount} entries.</info>");
$output->writeln('Regenerating System URL aliases...');
$progressBar = $this->getProgressBar($locationsCount, $output);
$progressBar->start();
for ($offset = 0; $offset <= $locationsCount; $offset += $iterationCount) {
gc_disable();
$locations = $this->repository->sudo(
function (Repository $repository) use ($offset, $iterationCount) {
return $repository->getLocationService()->loadAllLocations($offset, $iterationCount);
}
);
$this->processLocations($locations, $progressBar);
gc_enable();
}
$progressBar->finish();
$output->writeln('');
$output->writeln('Purging HTTP cache...');
$this->cachePurger->purgeAll();
$output->writeln('<info>Done.</info>');
}
/**
* Return configured progress bar helper.
*
* @param int $maxSteps
* @param \Symfony\Component\Console\Output\OutputInterface $output
*
* @return \Symfony\Component\Console\Helper\ProgressBar
*/
protected function getProgressBar($maxSteps, OutputInterface $output)
{
$progressBar = new ProgressBar($output, $maxSteps);
$progressBar->setFormat(
' %current%/%max% [%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s% %memory:6s%'
);
return $progressBar;
}
/**
* Process single results page of fetched Locations.
*
* @param \eZ\Publish\API\Repository\Values\Content\Location[] $locations
* @param \Symfony\Component\Console\Helper\ProgressBar $progressBar
*/
private function processLocations(array $locations, ProgressBar $progressBar)
{
$contentList = $this->repository->sudo(
function (Repository $repository) use ($locations) {
$contentInfoList = array_map(
function (Location $location) {
return $location->contentInfo;
},
$locations
);
// load Content list in all languages
return $repository->getContentService()->loadContentListByContentInfo(
$contentInfoList,
Language::ALL,
false
);
}
);
foreach ($locations as $location) {
try {
// ignore missing Content items
if (!isset($contentList[$location->contentId])) {
continue;
}
$content = $contentList[$location->contentId];
$this->repository->sudo(
function (Repository $repository) use ($location, $content) {
$repository->getURLAliasService()->refreshSystemUrlAliasesForLocation(
$location,
$content
);
}
);
} catch (Exception $e) {
$contentInfo = $location->getContentInfo();
$msg = sprintf(
'Failed processing location %d - [%d] %s (%s: %s)',
$location->id,
$contentInfo->id,
$contentInfo->name,
get_class($e),
$e->getMessage()
);
$this->logger->warning($msg);
// in debug mode log full exception with a trace
$this->logger->debug($e);
} finally {
$progressBar->advance(1);
}
}
}
}
@@ -0,0 +1,10 @@
services:
ezpublish.console.command.regenerate_url_aliases:
class: eZ\Bundle\EzPublishCoreBundle\Command\RegenerateUrlAliasesCommand
arguments:
# intentionally passing inner repository to avoid sending Signals due to performance issues
- '@ezpublish.api.inner_repository'
- '@ezpublish.http_cache.purger'
- '@?logger'
tags:
- { name: console.command }
@@ -32,6 +32,7 @@ services:
- "@ezpublish.field_type_nameable_collection.factory"
- "%ezpublish.api.inner_repository.class%"
- "%ezpublish.api.role.policy_map%"
- "@?logger"
calls:
- [setContainer, ["@service_container"]]

@@ -1,3 +1,6 @@
imports:
- { resource: commands.yml }

parameters:
ezpublish.siteaccess.class: eZ\Publish\Core\MVC\Symfony\SiteAccess
ezpublish.siteaccess.default.name: default
@@ -24,6 +24,9 @@
use Exception;
use PDO;
/**
* @deprecated since 6.7.8, use the ezplatform:urls:regenerate-aliases command instead.
*/
class RegenerateUrlAliasesCommand extends ContainerAwareCommand
{
const MIGRATION_TABLE = '__migration_ezurlalias_ml';
@@ -79,7 +82,7 @@ protected function configure()
$this
->setName('ezplatform:regenerate:legacy_storage_url_aliases')
->setDescription(
'Regenerates Location URL aliases (autogenerated) and migrates custom Location ' .
'[DEPRECATED] Regenerates Location URL aliases (autogenerated) and migrates custom Location ' .
'and global URL aliases with Legacy Storage Engine'
)
->addArgument(
@@ -95,6 +98,7 @@ protected function configure()
)
->setHelp(
<<<EOT
<error>This command is deprecated, use the ezplatform:urls:regenerate-aliases command instead.</error>
The command <info>%command.name%</info> regenerates URL aliases for Locations
and migrates existing custom Location and global URL aliases to a separate database table. Separate
table must be named '__migration_ezurlalias_ml' and should be created manually to be identical (but
@@ -122,6 +126,15 @@ protected function configure()
protected function execute(InputInterface $input, OutputInterface $output)
{
@trigger_error(
sprintf(
'%s is deprecated since 6.7.8. Use the ezplatform:urls:regenerate-aliases command instead.',
$this->getName()
),
E_USER_DEPRECATED
);
$output->writeln('<error>This command is deprecated, use the ezplatform:urls:regenerate-aliases command instead.</error>');
$this->checkStorage();
$action = $input->getArgument('action');
@@ -148,6 +148,23 @@ public function loadContent($contentId, array $languages = null, $versionNo = nu
*/
public function loadContentByRemoteId($remoteId, array $languages = null, $versionNo = null, $useAlwaysAvailable = true);
/**
* Bulk-load Content items by the list of ContentInfo Value Objects.
*
* Note: it does not throw exceptions on load, just ignores erroneous Content item.
* Moreover, since the method works on pre-loaded ContentInfo list, it is assumed that user is
* allowed to access every Content on the list.
*
* @param \eZ\Publish\API\Repository\Values\Content\ContentInfo[] $contentInfoList
* @param string[] $languages A language priority, filters returned fields and is used as prioritized language code on
* returned value object. If not given all languages are returned.
* @param bool $useAlwaysAvailable Add Main language to \$languages if true (default) and if alwaysAvailable is true,
* unless all languages have been asked for.
*
* @return \eZ\Publish\API\Repository\Values\Content\Content[] list of Content items with Content Ids as keys
*/
public function loadContentListByContentInfo(array $contentInfoList, array $languages = [], $useAlwaysAvailable = true);
/**
* Creates a new content draft assigned to the authenticated user.
*

0 comments on commit 2a36848

Please sign in to comment.
You can’t perform that action at this time.