elasticsearch now can have a different index setting per environment

shopsys · Sep 25, 2023 · 169860b · 169860b
1 parent 82ccc12
commit 169860b
Show file tree

Hide file tree

Showing 11 changed files with 115 additions and 7 deletions.
diff --git a/docker/conf/docker-compose.github-actions.review.simplified.yml.dist b/docker/conf/docker-compose.github-actions.review.simplified.yml.dist
@@ -33,6 +33,7 @@ services:
             DATABASE_NAME: BRANCH_NAME
             ELASTIC_SEARCH_INDEX_PREFIX: BRANCH_NAME
             REDIS_PREFIX: BRANCH_NAME
+            FORCE_ELASTIC_LIMITS: true
         networks:
             - default
             - services-network

diff --git a/docker/conf/docker-compose.github-actions.review.yml.dist b/docker/conf/docker-compose.github-actions.review.yml.dist
@@ -56,6 +56,7 @@ services:
             - postgres
         environment:
             IGNORE_DEFAULT_ADMIN_PASSWORD_CHECK: 1
+            FORCE_ELASTIC_LIMITS: true
         networks:
             - default
         labels:

diff --git a/packages/framework/src/Component/Elasticsearch/IndexDefinition.php b/packages/framework/src/Component/Elasticsearch/IndexDefinition.php
@@ -15,12 +15,14 @@ class IndexDefinition
      * @param string $definitionsDirectory
      * @param string $indexPrefix
      * @param int $domainId
+     * @param \Shopsys\FrameworkBundle\Component\Elasticsearch\IndexDefinitionModifier $indexDefinitionModifier
      */
     public function __construct(
         protected readonly string $indexName,
         protected readonly string $definitionsDirectory,
         protected readonly string $indexPrefix,
         protected readonly int $domainId,
+        protected readonly IndexDefinitionModifier $indexDefinitionModifier,
     ) {
     }
 
@@ -38,7 +40,7 @@ public function getDefinition(): array
             );
         }
 
-        return $decodedDefinition;
+        return $this->indexDefinitionModifier->modifyDefinition($decodedDefinition);
     }
 
     /**

diff --git a/packages/framework/src/Component/Elasticsearch/IndexDefinitionLoader.php b/packages/framework/src/Component/Elasticsearch/IndexDefinitionLoader.php
@@ -9,10 +9,12 @@ class IndexDefinitionLoader
     /**
      * @param string $indexDefinitionsDirectory
      * @param string $indexPrefix
+     * @param \Shopsys\FrameworkBundle\Component\Elasticsearch\IndexDefinitionModifier $indexDefinitionModifier
      */
     public function __construct(
         protected readonly string $indexDefinitionsDirectory,
         protected readonly string $indexPrefix,
+        protected readonly IndexDefinitionModifier $indexDefinitionModifier,
     ) {
     }
 
@@ -23,6 +25,6 @@ public function __construct(
      */
     public function getIndexDefinition(string $indexName, int $domainId): IndexDefinition
     {
-        return new IndexDefinition($indexName, $this->indexDefinitionsDirectory, $this->indexPrefix, $domainId);
+        return new IndexDefinition($indexName, $this->indexDefinitionsDirectory, $this->indexPrefix, $domainId, $this->indexDefinitionModifier);
     }
 }
diff --git a/packages/framework/src/Component/Elasticsearch/IndexDefinitionModifier.php b/packages/framework/src/Component/Elasticsearch/IndexDefinitionModifier.php
@@ -0,0 +1,34 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shopsys\FrameworkBundle\Component\Elasticsearch;
+
+use Shopsys\FrameworkBundle\Component\Environment\EnvironmentType;
+
+class IndexDefinitionModifier
+{
+    /**
+     * @param string $environment
+     * @param bool $forceElasticLimit
+     */
+    public function __construct(
+        protected readonly string $environment,
+        protected readonly bool $forceElasticLimit,
+    ) {
+    }
+
+    /**
+     * @param array $decodedDefinition
+     * @return array
+     */
+    public function modifyDefinition(array $decodedDefinition): array
+    {
+        if ($this->environment !== EnvironmentType::PRODUCTION || $this->forceElasticLimit) {
+            $decodedDefinition['settings']['index']['number_of_shards'] = 1;
+            $decodedDefinition['settings']['index']['number_of_replicas'] = 0;
+        }
+
+        return $decodedDefinition;
+    }
+}
diff --git a/packages/framework/src/Resources/config/services.yaml b/packages/framework/src/Resources/config/services.yaml
@@ -1030,3 +1030,8 @@ services:
     Shopsys\FrameworkBundle\Component\Redis\RedisClientFacade:
         arguments:
             - '@snc_redis.global'
+
+    Shopsys\FrameworkBundle\Component\Elasticsearch\IndexDefinitionModifier:
+        arguments:
+            $environment: '%kernel.environment%'
+            $forceElasticLimit: '%env(FORCE_ELASTIC_LIMITS)%'
diff --git a/packages/framework/tests/Unit/Component/Elasticsearch/IndexDefinitionTest.php b/packages/framework/tests/Unit/Component/Elasticsearch/IndexDefinitionTest.php
@@ -7,6 +7,8 @@
 use PHPUnit\Framework\TestCase;
 use Shopsys\FrameworkBundle\Component\Elasticsearch\Exception\ElasticsearchIndexException;
 use Shopsys\FrameworkBundle\Component\Elasticsearch\IndexDefinition;
+use Shopsys\FrameworkBundle\Component\Elasticsearch\IndexDefinitionModifier;
+use Shopsys\FrameworkBundle\Component\Environment\EnvironmentType;
 
 class IndexDefinitionTest extends TestCase
 {
@@ -25,7 +27,7 @@ public function testGetIndexAlias(
         int $domainId,
         string $expectedResult,
     ) {
-        $indexDefinition = new IndexDefinition($indexName, $definitionsDirectory, $indexPrefix, $domainId);
+        $indexDefinition = new IndexDefinition($indexName, $definitionsDirectory, $indexPrefix, $domainId, new IndexDefinitionModifier(EnvironmentType::TEST, false));
         $this->assertSame($expectedResult, $indexDefinition->getIndexAlias());
     }
 
@@ -46,14 +48,14 @@ public function indexDefinitionParametersForIndexAlias(): array
     public function testGetDefinitionReturnsDefinition(): void
     {
         $definitionDirectory = __DIR__ . '/__fixtures/definitions/valid/';
-        $indexDefinition = new IndexDefinition('product', $definitionDirectory, '', 1);
+        $indexDefinition = new IndexDefinition('product', $definitionDirectory, '', 1, new IndexDefinitionModifier(EnvironmentType::PRODUCTION, false));
         $this->assertSame(['foo' => 'bar'], $indexDefinition->getDefinition());
     }
 
     public function testGetDefinitionOnInvalidJsonThrowsException(): void
     {
         $definitionDirectory = __DIR__ . '/__fixtures/definitions/invalidJson/';
-        $indexDefinition = new IndexDefinition('product', $definitionDirectory, '', 1);
+        $indexDefinition = new IndexDefinition('product', $definitionDirectory, '', 1, new IndexDefinitionModifier(EnvironmentType::PRODUCTION, false));
 
         $this->expectException(ElasticsearchIndexException::class);
         $this->expectExceptionMessage('Invalid JSON in "product" definition file');
@@ -63,7 +65,7 @@ public function testGetDefinitionOnInvalidJsonThrowsException(): void
     public function testGetDefinitionOnNonExistingDefinitionThrowsException(): void
     {
         $definitionDirectory = __DIR__ . '/__fixtures/definitions/non-existing-folder-id-3e85ba/';
-        $indexDefinition = new IndexDefinition('product', $definitionDirectory, '', 1);
+        $indexDefinition = new IndexDefinition('product', $definitionDirectory, '', 1, new IndexDefinitionModifier(EnvironmentType::PRODUCTION, false));
 
         $this->expectException(ElasticsearchIndexException::class);
         $this->expectExceptionMessage('Can\'t read definition file at path');
@@ -73,8 +75,44 @@ public function testGetDefinitionOnNonExistingDefinitionThrowsException(): void
     public function testGetVersionedIndexName(): void
     {
         $definitionDirectory = __DIR__ . '/__fixtures/definitions/valid/';
-        $indexDefinition = new IndexDefinition('product', $definitionDirectory, '', 1);
+        $indexDefinition = new IndexDefinition('product', $definitionDirectory, '', 1, new IndexDefinitionModifier(EnvironmentType::PRODUCTION, false));
 
         $this->assertSame('product_1_49a3696adf0fbfacc12383a2d7400d51', $indexDefinition->getVersionedIndexName());
     }
+
+    public function testDevEnvironmentIsLimited(): void
+    {
+        $definitionDirectory = __DIR__ . '/__fixtures/definitions/valid/';
+        $indexDefinition = new IndexDefinition('product', $definitionDirectory, '', 1, new IndexDefinitionModifier(EnvironmentType::DEVELOPMENT, false));
+        $this->assertSame(
+            [
+                'foo' => 'bar',
+                'settings' => [
+                    'index' => [
+                        'number_of_shards' => 1,
+                        'number_of_replicas' => 0,
+                    ],
+                ],
+            ],
+            $indexDefinition->getDefinition(),
+        );
+    }
+
+    public function testProdEnvironmentIsLimitedWhenForced(): void
+    {
+        $definitionDirectory = __DIR__ . '/__fixtures/definitions/valid/';
+        $indexDefinition = new IndexDefinition('product', $definitionDirectory, '', 1, new IndexDefinitionModifier(EnvironmentType::PRODUCTION, true));
+        $this->assertSame(
+            [
+                'foo' => 'bar',
+                'settings' => [
+                    'index' => [
+                        'number_of_shards' => 1,
+                        'number_of_replicas' => 0,
+                    ],
+                ],
+            ],
+            $indexDefinition->getDefinition(),
+        );
+    }
 }
diff --git a/project-base/app/.env b/project-base/app/.env
@@ -67,3 +67,5 @@ PACKETERY_SENDER='shopsys_test'
 SENTRY_DSN=
 SENTRY_ENVIRONMENT=
 SENTRY_RELEASE=
+
+FORCE_ELASTIC_LIMITS=false
diff --git a/project-base/app/deploy/deploy-project.sh b/project-base/app/deploy/deploy-project.sh
@@ -68,6 +68,7 @@ function deploy() {
         ["SENTRY_DSN"]=${SENTRY_DSN}
         ["SENTRY_ENVIRONMENT"]=${CI_ENVIRONMENT_SLUG}
         ["SENTRY_RELEASE"]=${CI_COMMIT_SHORT_SHA}
+        ["FORCE_ELASTIC_LIMITS"]=${FORCE_ELASTIC_LIMITS:-false}
     )
 
     declare -A STOREFRONT_ENVIRONMENT_VARIABLES=(

diff --git a/project-base/gitlab/docker-compose-ci-review.yml b/project-base/gitlab/docker-compose-ci-review.yml
@@ -45,6 +45,7 @@ services:
             PACKETERY_REST_API_URL: ~
             PACKETERY_API_PASSWORD: ~
             PACKETERY_SENDER: ~
+            FORCE_ELASTIC_LIMITS: true
         labels:
             - traefik.enable=false
         networks:

diff --git a/upgrade/UPGRADE-v13.0.0-dev.md b/upgrade/UPGRADE-v13.0.0-dev.md
@@ -222,3 +222,24 @@ You will need to follow these steps:
             public function setFileKeyAsUploaded(string $key): void;
         ```
     - see #project-base-diff to update your project
+- allow elasticsearch different index setting per environment for elasticsearch ([#2823](https://github.com/shopsys/shopsys/pull/2823))
+    - new environment variable `FORCE_ELASTIC_LIMITS` may be used to force 1 shard and 0 replicas for elasticsearch indexes regardless of the settings in index definition JSON files
+    - method `Shopsys\FrameworkBundle\Component\Elasticsearch\IndexDefinition::__construct()` changed its interface
+        ```diff
+            public function __construct(
+                protected readonly string $indexName,
+                protected readonly string $definitionsDirectory,
+                protected readonly string $indexPrefix,
+                protected readonly int $domainId,
+        +       protected readonly IndexDefinitionModifier $indexDefinitionModifier,
+            ) {
+        ```
+    - method `Shopsys\FrameworkBundle\Component\Elasticsearch\IndexDefinitionLoader::__construct()` changed its interface
+        ```
+            public function __construct(
+                protected readonly string $indexDefinitionsDirectory,
+                protected readonly string $indexPrefix,
+        +       protected readonly IndexDefinitionModifier $indexDefinitionModifier,
+            ) {
+        ```
+    - see #project-base-diff to update your project