From 169860b97dcca230b456c858dbf643266e9bd7c0 Mon Sep 17 00:00:00 2001 From: Martin Grossmann Date: Thu, 21 Sep 2023 11:34:06 +0200 Subject: [PATCH] elasticsearch now can have a different index setting per environment --- ....github-actions.review.simplified.yml.dist | 1 + ...ker-compose.github-actions.review.yml.dist | 1 + .../Elasticsearch/IndexDefinition.php | 4 +- .../Elasticsearch/IndexDefinitionLoader.php | 4 +- .../Elasticsearch/IndexDefinitionModifier.php | 34 +++++++++++++ .../src/Resources/config/services.yaml | 5 ++ .../Elasticsearch/IndexDefinitionTest.php | 48 +++++++++++++++++-- project-base/app/.env | 2 + project-base/app/deploy/deploy-project.sh | 1 + .../gitlab/docker-compose-ci-review.yml | 1 + upgrade/UPGRADE-v13.0.0-dev.md | 21 ++++++++ 11 files changed, 115 insertions(+), 7 deletions(-) create mode 100644 packages/framework/src/Component/Elasticsearch/IndexDefinitionModifier.php diff --git a/docker/conf/docker-compose.github-actions.review.simplified.yml.dist b/docker/conf/docker-compose.github-actions.review.simplified.yml.dist index c1a52c97d8d..ba91347ee7f 100644 --- a/docker/conf/docker-compose.github-actions.review.simplified.yml.dist +++ b/docker/conf/docker-compose.github-actions.review.simplified.yml.dist @@ -33,6 +33,7 @@ services: DATABASE_NAME: BRANCH_NAME ELASTIC_SEARCH_INDEX_PREFIX: BRANCH_NAME REDIS_PREFIX: BRANCH_NAME + FORCE_ELASTIC_LIMITS: true networks: - default - services-network diff --git a/docker/conf/docker-compose.github-actions.review.yml.dist b/docker/conf/docker-compose.github-actions.review.yml.dist index 87df6e1cc67..dafafc38d3d 100644 --- a/docker/conf/docker-compose.github-actions.review.yml.dist +++ b/docker/conf/docker-compose.github-actions.review.yml.dist @@ -56,6 +56,7 @@ services: - postgres environment: IGNORE_DEFAULT_ADMIN_PASSWORD_CHECK: 1 + FORCE_ELASTIC_LIMITS: true networks: - default labels: diff --git a/packages/framework/src/Component/Elasticsearch/IndexDefinition.php b/packages/framework/src/Component/Elasticsearch/IndexDefinition.php index 381b0903a45..7e9cce1b4fc 100644 --- a/packages/framework/src/Component/Elasticsearch/IndexDefinition.php +++ b/packages/framework/src/Component/Elasticsearch/IndexDefinition.php @@ -15,12 +15,14 @@ class IndexDefinition * @param string $definitionsDirectory * @param string $indexPrefix * @param int $domainId + * @param \Shopsys\FrameworkBundle\Component\Elasticsearch\IndexDefinitionModifier $indexDefinitionModifier */ public function __construct( protected readonly string $indexName, protected readonly string $definitionsDirectory, protected readonly string $indexPrefix, protected readonly int $domainId, + protected readonly IndexDefinitionModifier $indexDefinitionModifier, ) { } @@ -38,7 +40,7 @@ public function getDefinition(): array ); } - return $decodedDefinition; + return $this->indexDefinitionModifier->modifyDefinition($decodedDefinition); } /** diff --git a/packages/framework/src/Component/Elasticsearch/IndexDefinitionLoader.php b/packages/framework/src/Component/Elasticsearch/IndexDefinitionLoader.php index 5895db0710c..760481a701b 100644 --- a/packages/framework/src/Component/Elasticsearch/IndexDefinitionLoader.php +++ b/packages/framework/src/Component/Elasticsearch/IndexDefinitionLoader.php @@ -9,10 +9,12 @@ class IndexDefinitionLoader /** * @param string $indexDefinitionsDirectory * @param string $indexPrefix + * @param \Shopsys\FrameworkBundle\Component\Elasticsearch\IndexDefinitionModifier $indexDefinitionModifier */ public function __construct( protected readonly string $indexDefinitionsDirectory, protected readonly string $indexPrefix, + protected readonly IndexDefinitionModifier $indexDefinitionModifier, ) { } @@ -23,6 +25,6 @@ public function __construct( */ public function getIndexDefinition(string $indexName, int $domainId): IndexDefinition { - return new IndexDefinition($indexName, $this->indexDefinitionsDirectory, $this->indexPrefix, $domainId); + return new IndexDefinition($indexName, $this->indexDefinitionsDirectory, $this->indexPrefix, $domainId, $this->indexDefinitionModifier); } } diff --git a/packages/framework/src/Component/Elasticsearch/IndexDefinitionModifier.php b/packages/framework/src/Component/Elasticsearch/IndexDefinitionModifier.php new file mode 100644 index 00000000000..fd6a00db75f --- /dev/null +++ b/packages/framework/src/Component/Elasticsearch/IndexDefinitionModifier.php @@ -0,0 +1,34 @@ +environment !== EnvironmentType::PRODUCTION || $this->forceElasticLimit) { + $decodedDefinition['settings']['index']['number_of_shards'] = 1; + $decodedDefinition['settings']['index']['number_of_replicas'] = 0; + } + + return $decodedDefinition; + } +} diff --git a/packages/framework/src/Resources/config/services.yaml b/packages/framework/src/Resources/config/services.yaml index 8706bd7f885..a98beba0b12 100644 --- a/packages/framework/src/Resources/config/services.yaml +++ b/packages/framework/src/Resources/config/services.yaml @@ -1030,3 +1030,8 @@ services: Shopsys\FrameworkBundle\Component\Redis\RedisClientFacade: arguments: - '@snc_redis.global' + + Shopsys\FrameworkBundle\Component\Elasticsearch\IndexDefinitionModifier: + arguments: + $environment: '%kernel.environment%' + $forceElasticLimit: '%env(FORCE_ELASTIC_LIMITS)%' diff --git a/packages/framework/tests/Unit/Component/Elasticsearch/IndexDefinitionTest.php b/packages/framework/tests/Unit/Component/Elasticsearch/IndexDefinitionTest.php index 662a3493a11..3a22e064555 100644 --- a/packages/framework/tests/Unit/Component/Elasticsearch/IndexDefinitionTest.php +++ b/packages/framework/tests/Unit/Component/Elasticsearch/IndexDefinitionTest.php @@ -7,6 +7,8 @@ use PHPUnit\Framework\TestCase; use Shopsys\FrameworkBundle\Component\Elasticsearch\Exception\ElasticsearchIndexException; use Shopsys\FrameworkBundle\Component\Elasticsearch\IndexDefinition; +use Shopsys\FrameworkBundle\Component\Elasticsearch\IndexDefinitionModifier; +use Shopsys\FrameworkBundle\Component\Environment\EnvironmentType; class IndexDefinitionTest extends TestCase { @@ -25,7 +27,7 @@ public function testGetIndexAlias( int $domainId, string $expectedResult, ) { - $indexDefinition = new IndexDefinition($indexName, $definitionsDirectory, $indexPrefix, $domainId); + $indexDefinition = new IndexDefinition($indexName, $definitionsDirectory, $indexPrefix, $domainId, new IndexDefinitionModifier(EnvironmentType::TEST, false)); $this->assertSame($expectedResult, $indexDefinition->getIndexAlias()); } @@ -46,14 +48,14 @@ public function indexDefinitionParametersForIndexAlias(): array public function testGetDefinitionReturnsDefinition(): void { $definitionDirectory = __DIR__ . '/__fixtures/definitions/valid/'; - $indexDefinition = new IndexDefinition('product', $definitionDirectory, '', 1); + $indexDefinition = new IndexDefinition('product', $definitionDirectory, '', 1, new IndexDefinitionModifier(EnvironmentType::PRODUCTION, false)); $this->assertSame(['foo' => 'bar'], $indexDefinition->getDefinition()); } public function testGetDefinitionOnInvalidJsonThrowsException(): void { $definitionDirectory = __DIR__ . '/__fixtures/definitions/invalidJson/'; - $indexDefinition = new IndexDefinition('product', $definitionDirectory, '', 1); + $indexDefinition = new IndexDefinition('product', $definitionDirectory, '', 1, new IndexDefinitionModifier(EnvironmentType::PRODUCTION, false)); $this->expectException(ElasticsearchIndexException::class); $this->expectExceptionMessage('Invalid JSON in "product" definition file'); @@ -63,7 +65,7 @@ public function testGetDefinitionOnInvalidJsonThrowsException(): void public function testGetDefinitionOnNonExistingDefinitionThrowsException(): void { $definitionDirectory = __DIR__ . '/__fixtures/definitions/non-existing-folder-id-3e85ba/'; - $indexDefinition = new IndexDefinition('product', $definitionDirectory, '', 1); + $indexDefinition = new IndexDefinition('product', $definitionDirectory, '', 1, new IndexDefinitionModifier(EnvironmentType::PRODUCTION, false)); $this->expectException(ElasticsearchIndexException::class); $this->expectExceptionMessage('Can\'t read definition file at path'); @@ -73,8 +75,44 @@ public function testGetDefinitionOnNonExistingDefinitionThrowsException(): void public function testGetVersionedIndexName(): void { $definitionDirectory = __DIR__ . '/__fixtures/definitions/valid/'; - $indexDefinition = new IndexDefinition('product', $definitionDirectory, '', 1); + $indexDefinition = new IndexDefinition('product', $definitionDirectory, '', 1, new IndexDefinitionModifier(EnvironmentType::PRODUCTION, false)); $this->assertSame('product_1_49a3696adf0fbfacc12383a2d7400d51', $indexDefinition->getVersionedIndexName()); } + + public function testDevEnvironmentIsLimited(): void + { + $definitionDirectory = __DIR__ . '/__fixtures/definitions/valid/'; + $indexDefinition = new IndexDefinition('product', $definitionDirectory, '', 1, new IndexDefinitionModifier(EnvironmentType::DEVELOPMENT, false)); + $this->assertSame( + [ + 'foo' => 'bar', + 'settings' => [ + 'index' => [ + 'number_of_shards' => 1, + 'number_of_replicas' => 0, + ], + ], + ], + $indexDefinition->getDefinition(), + ); + } + + public function testProdEnvironmentIsLimitedWhenForced(): void + { + $definitionDirectory = __DIR__ . '/__fixtures/definitions/valid/'; + $indexDefinition = new IndexDefinition('product', $definitionDirectory, '', 1, new IndexDefinitionModifier(EnvironmentType::PRODUCTION, true)); + $this->assertSame( + [ + 'foo' => 'bar', + 'settings' => [ + 'index' => [ + 'number_of_shards' => 1, + 'number_of_replicas' => 0, + ], + ], + ], + $indexDefinition->getDefinition(), + ); + } } diff --git a/project-base/app/.env b/project-base/app/.env index ad701b6de38..2f4e97f3975 100644 --- a/project-base/app/.env +++ b/project-base/app/.env @@ -67,3 +67,5 @@ PACKETERY_SENDER='shopsys_test' SENTRY_DSN= SENTRY_ENVIRONMENT= SENTRY_RELEASE= + +FORCE_ELASTIC_LIMITS=false diff --git a/project-base/app/deploy/deploy-project.sh b/project-base/app/deploy/deploy-project.sh index 89ccb7a95e0..9e945e3091d 100644 --- a/project-base/app/deploy/deploy-project.sh +++ b/project-base/app/deploy/deploy-project.sh @@ -68,6 +68,7 @@ function deploy() { ["SENTRY_DSN"]=${SENTRY_DSN} ["SENTRY_ENVIRONMENT"]=${CI_ENVIRONMENT_SLUG} ["SENTRY_RELEASE"]=${CI_COMMIT_SHORT_SHA} + ["FORCE_ELASTIC_LIMITS"]=${FORCE_ELASTIC_LIMITS:-false} ) declare -A STOREFRONT_ENVIRONMENT_VARIABLES=( diff --git a/project-base/gitlab/docker-compose-ci-review.yml b/project-base/gitlab/docker-compose-ci-review.yml index db45ade13a1..1234df9ebd7 100644 --- a/project-base/gitlab/docker-compose-ci-review.yml +++ b/project-base/gitlab/docker-compose-ci-review.yml @@ -45,6 +45,7 @@ services: PACKETERY_REST_API_URL: ~ PACKETERY_API_PASSWORD: ~ PACKETERY_SENDER: ~ + FORCE_ELASTIC_LIMITS: true labels: - traefik.enable=false networks: diff --git a/upgrade/UPGRADE-v13.0.0-dev.md b/upgrade/UPGRADE-v13.0.0-dev.md index 3dad5cf7c9e..219a072ff10 100644 --- a/upgrade/UPGRADE-v13.0.0-dev.md +++ b/upgrade/UPGRADE-v13.0.0-dev.md @@ -222,3 +222,24 @@ You will need to follow these steps: public function setFileKeyAsUploaded(string $key): void; ``` - see #project-base-diff to update your project +- allow elasticsearch different index setting per environment for elasticsearch ([#2823](https://github.com/shopsys/shopsys/pull/2823)) + - new environment variable `FORCE_ELASTIC_LIMITS` may be used to force 1 shard and 0 replicas for elasticsearch indexes regardless of the settings in index definition JSON files + - method `Shopsys\FrameworkBundle\Component\Elasticsearch\IndexDefinition::__construct()` changed its interface + ```diff + public function __construct( + protected readonly string $indexName, + protected readonly string $definitionsDirectory, + protected readonly string $indexPrefix, + protected readonly int $domainId, + + protected readonly IndexDefinitionModifier $indexDefinitionModifier, + ) { + ``` + - method `Shopsys\FrameworkBundle\Component\Elasticsearch\IndexDefinitionLoader::__construct()` changed its interface + ``` + public function __construct( + protected readonly string $indexDefinitionsDirectory, + protected readonly string $indexPrefix, + + protected readonly IndexDefinitionModifier $indexDefinitionModifier, + ) { + ``` + - see #project-base-diff to update your project