From fb48dc37b0171c3bbb764ebaa035ce08c5935439 Mon Sep 17 00:00:00 2001 From: Yevhen Miroshnychenko Date: Mon, 23 Mar 2020 12:35:58 -0500 Subject: [PATCH 1/3] MCLOUD-3059: [Spike] Investigate issue with elasticsearch image on AWS --- images/elasticsearch/1.7/Dockerfile | 2 +- images/elasticsearch/2.4/Dockerfile | 5 ++++- images/elasticsearch/5.2/Dockerfile | 2 +- images/elasticsearch/6.5/Dockerfile | 2 +- images/elasticsearch/7.5/Dockerfile | 2 +- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/images/elasticsearch/1.7/Dockerfile b/images/elasticsearch/1.7/Dockerfile index 83cae2f2..85088248 100644 --- a/images/elasticsearch/1.7/Dockerfile +++ b/images/elasticsearch/1.7/Dockerfile @@ -6,6 +6,6 @@ RUN plugin --install elasticsearch/elasticsearch-analysis-icu/2.7.0 && \ ADD docker-healthcheck.sh /docker-healthcheck.sh -HEALTHCHECK --retries=3 CMD ["sh", "/docker-healthcheck.sh"] +HEALTHCHECK --retries=3 CMD ["bash", "/docker-healthcheck.sh"] EXPOSE 9200 9300 diff --git a/images/elasticsearch/2.4/Dockerfile b/images/elasticsearch/2.4/Dockerfile index af008b1b..b38ee54e 100644 --- a/images/elasticsearch/2.4/Dockerfile +++ b/images/elasticsearch/2.4/Dockerfile @@ -1,11 +1,14 @@ FROM elasticsearch:2.4 +RUN apt update -y && \ + apt install procps -y + RUN echo "xpack.security.enabled: false" >> /usr/share/elasticsearch/config/elasticsearch.yml RUN bin/plugin install analysis-icu && \ bin/plugin install analysis-phonetic ADD docker-healthcheck.sh /docker-healthcheck.sh -HEALTHCHECK --retries=3 CMD ["sh", "/docker-healthcheck.sh"] +HEALTHCHECK --retries=3 CMD ["bash", "/docker-healthcheck.sh"] EXPOSE 9200 9300 diff --git a/images/elasticsearch/5.2/Dockerfile b/images/elasticsearch/5.2/Dockerfile index 283825ec..5511ba78 100644 --- a/images/elasticsearch/5.2/Dockerfile +++ b/images/elasticsearch/5.2/Dockerfile @@ -6,6 +6,6 @@ RUN bin/elasticsearch-plugin install analysis-icu && \ ADD docker-healthcheck.sh /docker-healthcheck.sh -HEALTHCHECK --retries=3 CMD ["sh", "/docker-healthcheck.sh"] +HEALTHCHECK --retries=3 CMD ["bash", "/docker-healthcheck.sh"] EXPOSE 9200 9300 diff --git a/images/elasticsearch/6.5/Dockerfile b/images/elasticsearch/6.5/Dockerfile index a3066de9..4f040ca8 100644 --- a/images/elasticsearch/6.5/Dockerfile +++ b/images/elasticsearch/6.5/Dockerfile @@ -6,6 +6,6 @@ RUN bin/elasticsearch-plugin install analysis-icu && \ ADD docker-healthcheck.sh /docker-healthcheck.sh -HEALTHCHECK --retries=3 CMD ["sh", "/docker-healthcheck.sh"] +HEALTHCHECK --retries=3 CMD ["bash", "/docker-healthcheck.sh"] EXPOSE 9200 9300 diff --git a/images/elasticsearch/7.5/Dockerfile b/images/elasticsearch/7.5/Dockerfile index 198bc2a1..3613ad4c 100755 --- a/images/elasticsearch/7.5/Dockerfile +++ b/images/elasticsearch/7.5/Dockerfile @@ -7,6 +7,6 @@ RUN bin/elasticsearch-plugin install -b analysis-icu && \ ADD docker-healthcheck.sh /docker-healthcheck.sh -HEALTHCHECK --retries=3 CMD ["sh", "/docker-healthcheck.sh"] +HEALTHCHECK --retries=3 CMD ["bash", "/docker-healthcheck.sh"] EXPOSE 9200 9300 From b628c55f425f412ad131b845f7d26297fdca9186 Mon Sep 17 00:00:00 2001 From: Yevhen Miroshnychenko Date: Mon, 23 Mar 2020 12:50:36 -0500 Subject: [PATCH 2/3] MCLOUD-3059: [Spike] Investigate issue with elasticsearch image on AWS --- src/Command/BuildCompose.php | 6 ++ src/Compose/ProductionBuilder.php | 11 ++- src/Config/Source/CliSource.php | 21 ++++-- src/Config/Source/SourceInterface.php | 5 ++ .../Acceptance/ElasticsearchCest.php | 74 +++++++++++++++++++ 5 files changed, 111 insertions(+), 6 deletions(-) create mode 100644 src/Test/Functional/Acceptance/ElasticsearchCest.php diff --git a/src/Command/BuildCompose.php b/src/Command/BuildCompose.php index d2755257..4d436b61 100644 --- a/src/Command/BuildCompose.php +++ b/src/Command/BuildCompose.php @@ -235,6 +235,12 @@ protected function configure(): void null, InputOption::VALUE_OPTIONAL, 'Port' + ) + ->addOption( + Source\CliSource::OPTION_ES_ENVIRONMENT_VARIABLE, + null, + InputOption::VALUE_OPTIONAL | InputOption::VALUE_IS_ARRAY, + 'Environment variable for elasticsearch service' ); parent::configure(); diff --git a/src/Compose/ProductionBuilder.php b/src/Compose/ProductionBuilder.php index e07602bc..fcdc8cfa 100644 --- a/src/Compose/ProductionBuilder.php +++ b/src/Compose/ProductionBuilder.php @@ -13,6 +13,7 @@ use Magento\CloudDocker\Compose\ProductionBuilder\VolumeResolver; use Magento\CloudDocker\Config\Config; use Magento\CloudDocker\Config\Environment\Converter; +use Magento\CloudDocker\Config\Source\SourceInterface; use Magento\CloudDocker\Filesystem\FileList; use Magento\CloudDocker\Service\ServiceFactory; use Magento\CloudDocker\Service\ServiceInterface; @@ -207,6 +208,8 @@ public function build(Config $config): Manager $this->addDbService(self::SERVICE_DB_SALES, $manager, $dbVersion, $volumesMount, $config); } + $esEnvVars = $config->get(SourceInterface::SERVICES_ES_VARS); + foreach (self::$standaloneServices as $service) { if (!$config->hasServiceEnabled($service)) { continue; @@ -214,7 +217,13 @@ public function build(Config $config): Manager $manager->addService( $service, - $this->serviceFactory->create((string)$service, (string)$config->getServiceVersion($service)), + $this->serviceFactory->create( + (string)$service, + (string)$config->getServiceVersion($service), + self::SERVICE_ELASTICSEARCH === $service && !empty($esEnvVars) + ? ['environment' => $esEnvVars] + : [] + ), [self::NETWORK_MAGENTO], [] ); diff --git a/src/Config/Source/CliSource.php b/src/Config/Source/CliSource.php index 28373c55..12e2f508 100644 --- a/src/Config/Source/CliSource.php +++ b/src/Config/Source/CliSource.php @@ -55,6 +55,11 @@ class CliSource implements SourceInterface public const OPTION_HOST = 'host'; public const OPTION_PORT = 'port'; + /** + * Environment variable for elasticsearch service. + */ + public const OPTION_ES_ENVIRONMENT_VARIABLE = 'es-env-var'; + /** * Option key to config name map * @@ -110,12 +115,14 @@ public function read(): Repository ]); } - foreach (self::$optionsMap as $option => $service) { + foreach (self::$optionsMap as $option => $services) { if ($value = $this->input->getOption($option)) { - $repository->set([ - $service . '.enabled' => true, - $service . '.version' => $value - ]); + foreach ($services as $service) { + $repository->set([ + $service . '.enabled' => true, + $service . '.version' => $value + ]); + } } } @@ -185,6 +192,10 @@ public function read(): Repository $repository->set(self::SYSTEM_EXPOSE_DB_SALES_PORTS, $port); } + if ($esEnvVars = $this->input->getOption(self::OPTION_ES_ENVIRONMENT_VARIABLE)) { + $repository->set(self::SERVICES_ES_VARS, $esEnvVars); + } + return $repository; } } diff --git a/src/Config/Source/SourceInterface.php b/src/Config/Source/SourceInterface.php index 1e726480..8013195a 100644 --- a/src/Config/Source/SourceInterface.php +++ b/src/Config/Source/SourceInterface.php @@ -73,6 +73,11 @@ interface SourceInterface */ public const SERVICES_ES = self::SERVICES . '.' . ServiceInterface::SERVICE_ELASTICSEARCH; + /** + * ES environment variables + */ + public const SERVICES_ES_VARS = self::SERVICES_ES.'.'.'env-vars'; + /** * Node */ diff --git a/src/Test/Functional/Acceptance/ElasticsearchCest.php b/src/Test/Functional/Acceptance/ElasticsearchCest.php new file mode 100644 index 00000000..ffb7c731 --- /dev/null +++ b/src/Test/Functional/Acceptance/ElasticsearchCest.php @@ -0,0 +1,74 @@ +runEceDockerCommand($command); + $I->startEnvironment(); + $I->runDockerComposeCommand('exec -T elasticsearch ps aux | grep elasticsearch'); + $I->seeInOutput('-Xms' . $xms); + $I->seeInOutput('-Xmx' . $xmx); + + if (in_array($data['version'], $versionsForCheckingParams)) { + $I->runDockerComposeCommand('exec -T elasticsearch curl http://localhost:9200/_nodes/settings'); + $I->seeInOutput(sprintf('"store":{"%s":"false"}', $param)); + } + } + + /** + * @return array + */ + protected function dataProvider(): array + { + return [ + ['version' => '1.7'], + ['version' => '2.4'], + ['version' => '5.2'], + ['version' => '6.5'], + ['version' => '7.5'], + ]; + } +} From 0df1b6a8b1d67bf2c65b39de992954b3d5b11d44 Mon Sep 17 00:00:00 2001 From: Yevhen Miroshnychenko Date: Mon, 23 Mar 2020 15:42:30 -0500 Subject: [PATCH 3/3] MCLOUD-3059: [Spike] Investigate issue with elasticsearch image on AWS --- .../Acceptance/ElasticsearchCest.php | 69 +++++++++++++------ 1 file changed, 49 insertions(+), 20 deletions(-) diff --git a/src/Test/Functional/Acceptance/ElasticsearchCest.php b/src/Test/Functional/Acceptance/ElasticsearchCest.php index ffb7c731..7a24f0ff 100644 --- a/src/Test/Functional/Acceptance/ElasticsearchCest.php +++ b/src/Test/Functional/Acceptance/ElasticsearchCest.php @@ -30,31 +30,25 @@ class ElasticsearchCest extends AbstractAcceptanceCest */ public function testElasticsearch(CliTester $I, Example $data) { - $xms = '512m'; - $xmx = '512m'; - $param = 'allow_mmap'; - $versionsForCheckingParams = ['6.5', '7.5']; $command = sprintf( 'build:compose --mode=production --es=%s --es-env-var=ES_JAVA_OPTS="-Xms%s -Xmx%s"', $data['version'], - $xms, - $xmx + $data['xms'], + $data['xmx'] ); - if (in_array($data['version'], $versionsForCheckingParams)) { - if ('6.5' === $data['version']) { - $param .= 'fs'; - } - $command .= " --es-env-var=node.store.$param=false"; + + if (!empty($data['param'])) { + $command .= " --es-env-var={$data['param']['key']}={$data['param']['value']}"; } $I->runEceDockerCommand($command); $I->startEnvironment(); $I->runDockerComposeCommand('exec -T elasticsearch ps aux | grep elasticsearch'); - $I->seeInOutput('-Xms' . $xms); - $I->seeInOutput('-Xmx' . $xmx); + $I->seeInOutput('-Xms' . $data['xms']); + $I->seeInOutput('-Xmx' . $data['xmx']); - if (in_array($data['version'], $versionsForCheckingParams)) { + if (!empty($data['param'])) { $I->runDockerComposeCommand('exec -T elasticsearch curl http://localhost:9200/_nodes/settings'); - $I->seeInOutput(sprintf('"store":{"%s":"false"}', $param)); + $I->seeInOutput($data['param']['needle']); } } @@ -64,11 +58,46 @@ public function testElasticsearch(CliTester $I, Example $data) protected function dataProvider(): array { return [ - ['version' => '1.7'], - ['version' => '2.4'], - ['version' => '5.2'], - ['version' => '6.5'], - ['version' => '7.5'], + [ + 'version' => '1.7', + 'xms' => '512m', + 'xmx' => '512m', + ], + [ + 'version' => '2.4', + 'xms' => '514m', + 'xmx' => '514m', + ], + [ + 'version' => '5.2', + 'xms' => '516m', + 'xmx' => '516m', + 'param' => [ + 'key' => 'index.store.type', + 'value' => 'fs', + 'needle' => '"index":{"store":{"type":"fs"}}', + ] + ], + [ + 'version' => '6.5', + 'xms' => '518m', + 'xmx' => '518m', + 'param' => [ + 'key' => 'node.store.allow_mmapfs', + 'value' => 'false', + 'needle' => '"store":{"allow_mmapfs":"false"}', + ] + ], + [ + 'version' => '7.5', + 'xms' => '520m', + 'xmx' => '520m', + 'param' => [ + 'key' => 'node.store.allow_mmap', + 'value' => 'false', + 'needle' => '"store":{"allow_mmap":"false"}', + ] + ], ]; } }