Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

BUGFIX: Set up missing alias during node indexing #373

Merged
merged 6 commits into from
Feb 13, 2021
Merged

BUGFIX: Set up missing alias during node indexing #373

Changes from 3 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
1ed01ec
TASK: Code cleanup
kdambekalns Feb 11, 2021
5ad0cd9
BUGFIX: Set up missing alias during nodeindex:build
kdambekalns Feb 11, 2021
f4db0b6
BUGFIX: Set up missing alias during nodeindex:indexnode
kdambekalns Feb 11, 2021
cbd99b2
Merge branch '7.0' into bugfix/370-alias-creation
kdambekalns Feb 12, 2021
44d9c38
TASK: Remove JSON_THROW_ON_ERROR for PHP 7.2 compatibility
kdambekalns Feb 12, 2021
1c978a8
TASK: Remove redundant default arguments
kdambekalns Feb 12, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
152 changes: 110 additions & 42 deletions Classes/Command/NodeIndexCommandController.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,11 @@
*/

use Doctrine\Common\Collections\ArrayCollection;
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\IndexDriverInterface;
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\NodeTypeMappingBuilderInterface;
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\ElasticSearchClient;
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\ErrorHandling\ErrorHandlingService;
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception;
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception\ConfigurationException;
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception\RuntimeException;
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Indexer\NodeIndexer;
Expand All @@ -35,12 +38,9 @@
use Neos\Flow\Configuration\ConfigurationManager;
use Neos\Flow\Core\Booting\Exception\SubProcessException;
use Neos\Flow\Core\Booting\Scripts;
use Neos\Flow\Exception;
use Neos\Utility\Exception\FilesException;
use Neos\Utility\Files;
use Neos\Flow\Log\Utility\LogEnvironment;
use Neos\Utility\Files;
use Psr\Log\LoggerInterface;
use Symfony\Component\Yaml\Yaml;

/**
* Provides CLI features for index handling
Expand Down Expand Up @@ -133,41 +133,60 @@ class NodeIndexCommandController extends CommandController
*/
protected $workspaceIndexer;

/**
* @Flow\Inject
* @var ElasticSearchClient
*/
protected $searchClient;

/**
* @var IndexDriverInterface
* @Flow\Inject
*/
protected $indexDriver;

/**
* Index a single node by the given identifier and workspace name
*
* @param string $identifier
* @param string $workspace
* @param string|null $workspace
* @param string|null $postfix
* @return void
* @throws ApiException
* @throws ConfigurationException
* @throws FilesException
* @throws StopCommandException
* @throws \Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception
* @throws Exception
* @throws RuntimeException
* @throws SubProcessException
* @throws \Flowpack\ElasticSearch\Exception
* @throws \JsonException
*/
public function indexNodeCommand(string $identifier, string $workspace = null, string $postfix = null): void
{
if ($workspace === null && $this->settings['indexAllWorkspaces'] === false) {
$workspace = 'live';
}

$updateAliases = false;
if ($postfix !== null) {
$this->nodeIndexer->setIndexNamePostfix($postfix);
} elseif ($this->aliasesExist() === false) {
$postfix = (string)time();
$updateAliases = true;
$this->nodeIndexer->setIndexNamePostfix($postfix);
}

$indexNode = function ($identifier, Workspace $workspace, array $dimensions) {
$context = $this->createContentContext($workspace->getName(), $dimensions);
$node = $context->getNodeByIdentifier($identifier);

if ($node === null) {
return [$workspace->getName(), '-', json_encode($dimensions), 'not found'];
return [$workspace->getName(), '-', json_encode($dimensions, JSON_THROW_ON_ERROR), 'not found'];
}

$this->nodeIndexer->setDimensions($dimensions);
$this->nodeIndexer->indexNode($node);

return [$workspace->getName(), $node->getNodeType()->getName(), json_encode($dimensions), '<success>indexed</success>'];
return [$workspace->getName(), $node->getNodeType()->getName(), json_encode($dimensions, JSON_THROW_ON_ERROR), '<success>indexed</success>'];
};

$indexInWorkspace = function ($identifier, Workspace $workspace) use ($indexNode) {
Expand Down Expand Up @@ -201,6 +220,22 @@ public function indexNodeCommand(string $identifier, string $workspace = null, s
}

$this->nodeIndexer->flush();

if ($updateAliases) {
$combinations = $this->contentDimensionCombinator->getAllAllowedCombinations();
$combinations = $combinations === [] ? [[]] : $combinations;

foreach ($combinations as $combination) {
$this->executeInternalCommand('aliasInternal', [
'dimensionsValues' => json_encode($combination, JSON_THROW_ON_ERROR),
'postfix' => $postfix,
'update' => false
]);
}

$this->nodeIndexer->updateMainAlias();
}

$this->outputErrorHandling();
}

Expand All @@ -209,14 +244,15 @@ public function indexNodeCommand(string $identifier, string $workspace = null, s
*
* This command (re-)indexes all nodes contained in the content repository and sets the schema beforehand.
*
* @param int $limit Amount of nodes to index at maximum
* @param int|null $limit Amount of nodes to index at maximum
* @param bool $update if TRUE, do not throw away the index at the start. Should *only be used for development*.
* @param string $workspace name of the workspace which should be indexed
* @param string $postfix Index postfix, index with the same postfix will be deleted if exist
* @param string|null $workspace name of the workspace which should be indexed
* @param string|null $postfix Index postfix, index with the same postfix will be deleted if exist
* @return void
* @throws StopCommandException
* @throws \Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception
* @throws Exception
* @throws ConfigurationException
* @throws ApiException
*/
public function buildCommand(int $limit = null, bool $update = false, string $workspace = null, string $postfix = null): void
{
Expand All @@ -232,26 +268,26 @@ public function buildCommand(int $limit = null, bool $update = false, string $wo

$createIndicesAndApplyMapping = function (array $dimensionsValues) use ($update, $postfix) {
$this->executeInternalCommand('createInternal', [
'dimensionsValues' => json_encode($dimensionsValues),
'dimensionsValues' => json_encode($dimensionsValues, JSON_THROW_ON_ERROR),
'update' => $update,
'postfix' => $postfix,
]);
};

$buildIndex = function (array $dimensionsValues) use ($workspace, $limit, $update, $postfix) {
$buildIndex = function (array $dimensionsValues) use ($workspace, $limit, $postfix) {
$this->build($dimensionsValues, $workspace, $postfix, $limit);
};

$refresh = function (array $dimensionsValues) use ($postfix) {
$this->executeInternalCommand('refreshInternal', [
'dimensionsValues' => json_encode($dimensionsValues),
'dimensionsValues' => json_encode($dimensionsValues, JSON_THROW_ON_ERROR),
'postfix' => $postfix,
]);
};

$updateAliases = function (array $dimensionsValues) use ($update, $postfix) {
$this->executeInternalCommand('aliasInternal', [
'dimensionsValues' => json_encode($dimensionsValues),
'dimensionsValues' => json_encode($dimensionsValues, JSON_THROW_ON_ERROR),
'postfix' => $postfix,
'update' => $update,
]);
Expand All @@ -268,10 +304,9 @@ public function buildCommand(int $limit = null, bool $update = false, string $wo

$runAndLog($createIndicesAndApplyMapping, 'Creating indices and apply mapping');

// $timeStart = microtime(true);
// $this->output(str_pad('Indexing nodes ... ', 20));
// $buildIndex([]);
// $this->outputLine('<success>Done</success> (took %s seconds)', [number_format(microtime(true) - $timeStart, 2)]);
if ($this->aliasesExist() === false) {
$runAndLog($updateAliases, 'Set up aliases');
}

$runAndLog($buildIndex, 'Indexing nodes');

Expand All @@ -285,14 +320,40 @@ public function buildCommand(int $limit = null, bool $update = false, string $wo
$this->outputMemoryUsage();
}

/**
* @return bool
* @throws ApiException
* @throws ConfigurationException
* @throws Exception
*/
private function aliasesExist(): bool
{
$aliasName = $this->searchClient->getIndexName();
$aliasesExist = false;
try {
$aliasesExist = $this->indexDriver->getIndexNamesByAlias($aliasName) !== [];
} catch (ApiException $exception) {
// in case of 404, do not throw an error...
if ($exception->getResponse()->getStatusCode() !== 404) {
throw $exception;
}
}

return $aliasesExist;
}

/**
* Build up the node index
*
* @param array $dimensionsValues
* @param string $postfix
* @param string $workspace
* @param int $limit
* @param string|null $workspace
* @param string|null $postfix
* @param int|null $limit
* @throws ConfigurationException
* @throws Exception
* @throws RuntimeException
* @throws SubProcessException
* @throws \JsonException
*/
private function build(array $dimensionsValues, ?string $workspace = null, ?string $postfix = null, ?int $limit = null): void
{
Expand All @@ -307,7 +368,7 @@ private function build(array $dimensionsValues, ?string $workspace = null, ?stri
$buildWorkspaceCommandOptions = static function ($workspace, array $dimensionsValues, ?int $limit, ?string $postfix) {
return [
'workspace' => $workspace instanceof Workspace ? $workspace->getName() : $workspace,
'dimensionsValues' => json_encode($dimensionsValues),
'dimensionsValues' => json_encode($dimensionsValues, JSON_THROW_ON_ERROR),
'postfix' => $postfix,
'limit' => $limit,
];
Expand Down Expand Up @@ -342,24 +403,24 @@ private function build(array $dimensionsValues, ?string $workspace = null, ?stri
* @param string $dimensionsValues
* @param bool $update
* @param string|null $postfix
* @throws \Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception
* @throws Exception
* @throws \Flowpack\ElasticSearch\Exception
* @throws \Neos\Flow\Http\Exception
* @throws \Exception
* @Flow\Internal
*/
public function createInternalCommand(string $dimensionsValues, bool $update = false, ?string $postfix = null): void
public function createInternalCommand(string $dimensionsValues, bool $update = false, string $postfix = null): void
{
if ($update === true) {
$this->logger->warning('!!! Update Mode (Development) active!', LogEnvironment::fromMethodName(__METHOD__));
} else {
$dimensionsValuesArray = $this->configureNodeIndexer(json_decode($dimensionsValues, true), $postfix);
$dimensionsValuesArray = $this->configureNodeIndexer(json_decode($dimensionsValues, true, 512, JSON_THROW_ON_ERROR), $postfix);
if ($this->nodeIndexer->getIndex()->exists() === true) {
$this->logger->warning(sprintf('Deleted index with the same postfix (%s)!', $postfix), LogEnvironment::fromMethodName(__METHOD__));
$this->nodeIndexer->getIndex()->delete();
}
$this->nodeIndexer->getIndex()->create();
$this->logger->info('Created index ' . $this->nodeIndexer->getIndexName() . ' with dimensions ' . json_encode($dimensionsValuesArray), LogEnvironment::fromMethodName(__METHOD__));
$this->logger->info('Created index ' . $this->nodeIndexer->getIndexName() . ' with dimensions ' . json_encode($dimensionsValuesArray, JSON_THROW_ON_ERROR), LogEnvironment::fromMethodName(__METHOD__));
}

$this->applyMapping();
Expand All @@ -370,19 +431,20 @@ public function createInternalCommand(string $dimensionsValues, bool $update = f
* @param string $workspace
* @param string $dimensionsValues
* @param string $postfix
* @param int $limit
* @param int|null $limit
* @return void
* @Flow\Internal
* @throws \JsonException
*/
public function buildWorkspaceInternalCommand(string $workspace, string $dimensionsValues, string $postfix, int $limit = null): void
{
$dimensionsValuesArray = $this->configureNodeIndexer(json_decode($dimensionsValues, true), $postfix);
$dimensionsValuesArray = $this->configureNodeIndexer(json_decode($dimensionsValues, true, 512, JSON_THROW_ON_ERROR), $postfix);

$workspaceLogger = function ($workspaceName, $indexedNodes, $dimensions) {
if ($dimensions === []) {
$message = 'Workspace "' . $workspaceName . '" without dimensions done. (Indexed ' . $indexedNodes . ' nodes)';
} else {
$message = 'Workspace "' . $workspaceName . '" and dimensions "' . json_encode($dimensions) . '" done. (Indexed ' . $indexedNodes . ' nodes)';
$message = 'Workspace "' . $workspaceName . '" and dimensions "' . json_encode($dimensions, JSON_THROW_ON_ERROR) . '" done. (Indexed ' . $indexedNodes . ' nodes)';
}
$this->outputLine($message);
};
Expand All @@ -397,15 +459,16 @@ public function buildWorkspaceInternalCommand(string $workspace, string $dimensi
*
* @param string $dimensionsValues
* @param string $postfix
* @throws \Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception
* @throws Exception
* @throws \Flowpack\ElasticSearch\Exception
* @throws \Neos\Flow\Http\Exception
* @throws ConfigurationException
* @throws \JsonException
* @Flow\Internal
*/
public function refreshInternalCommand(string $dimensionsValues, string $postfix): void
{
$this->configureNodeIndexer(json_decode($dimensionsValues, true), $postfix);
$this->configureNodeIndexer(json_decode($dimensionsValues, true, 512, JSON_THROW_ON_ERROR), $postfix);

$this->logger->info(vsprintf('Refreshing index %s', [$this->nodeIndexer->getIndexName()]), LogEnvironment::fromMethodName(__METHOD__));
$this->nodeIndexer->getIndex()->refresh();
Expand All @@ -417,18 +480,19 @@ public function refreshInternalCommand(string $dimensionsValues, string $postfix
* @param string $dimensionsValues
* @param string $postfix
* @param bool $update
* @throws \Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception
* @throws Exception
* @throws \Flowpack\ElasticSearch\Exception
* @throws ApiException
* @throws ConfigurationException
* @throws \JsonException
* @Flow\Internal
*/
public function aliasInternalCommand(string $dimensionsValues, string $postfix, bool $update = false): void
{
if ($update === true) {
return;
}
$this->configureNodeIndexer(json_decode($dimensionsValues, true), $postfix);
$this->configureNodeIndexer(json_decode($dimensionsValues, true, 512, JSON_THROW_ON_ERROR), $postfix);

$this->logger->info(vsprintf('Update alias for index %s', [$this->nodeIndexer->getIndexName()]), LogEnvironment::fromMethodName(__METHOD__));
$this->nodeIndexer->updateIndexAlias();
Expand All @@ -451,7 +515,9 @@ private function configureNodeIndexer(array $dimensionsValues, string $postfix):
* Clean up old indexes (i.e. all but the current one)
*
* @return void
* @throws \Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception
* @throws ConfigurationException
* @throws Exception
* @throws \JsonException
*/
public function cleanupCommand(): void
{
Expand All @@ -468,7 +534,7 @@ public function cleanupCommand(): void
}
} catch (ApiException $exception) {
$exception->getResponse()->getBody()->rewind();
$response = json_decode($exception->getResponse()->getBody()->getContents(), false);
$response = json_decode($exception->getResponse()->getBody()->getContents(), false, 512, JSON_THROW_ON_ERROR);
$message = sprintf('Nothing removed. ElasticSearch responded with status %s', $response->status);

if (isset($response->error->type)) {
Expand Down Expand Up @@ -499,6 +565,7 @@ private function outputErrorHandling(): void
* @return string
* @throws RuntimeException
* @throws SubProcessException
* @throws \JsonException
*/
private function executeInternalCommand(string $command, array $arguments): string
{
Expand All @@ -509,7 +576,7 @@ private function executeInternalCommand(string $command, array $arguments): stri
$status = Scripts::executeCommand($commandIdentifier, $this->flowSettings, true, array_filter($arguments));

if ($status !== true) {
throw new RuntimeException(vsprintf('Command: %s with parameters: %s', [$commandIdentifier, json_encode($arguments)]), 1426767159);
throw new RuntimeException(vsprintf('Command: %s with parameters: %s', [$commandIdentifier, json_encode($arguments, JSON_THROW_ON_ERROR)]), 1426767159);
}
} else {
$commandIdentifier = $command . 'Command';
Expand All @@ -536,7 +603,7 @@ private function createContentContext(string $workspaceName, array $dimensions =

if ($dimensions !== []) {
$contextProperties['dimensions'] = $dimensions;
$contextProperties['targetDimensions'] = array_map(function ($dimensionValues) {
$contextProperties['targetDimensions'] = array_map(static function ($dimensionValues) {
return array_shift($dimensionValues);
}, $dimensions);
}
Expand All @@ -548,9 +615,10 @@ private function createContentContext(string $workspaceName, array $dimensions =
* Apply the mapping to the current index.
*
* @return void
* @throws \Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception
* @throws Exception
* @throws \Flowpack\ElasticSearch\Exception
* @throws ConfigurationException
* @throws \Neos\Flow\Http\Exception
*/
private function applyMapping(): void
{
Expand Down