MAGECLOUD-4486: Docker functional tests and CI with ece-tools test framework

.gitignore

@@ -7,3 +7,4 @@
 /docker-compose.yml
 /auth.json
 /codeception.yml
+/_workdir

.travis.yml

@@ -1,4 +1,4 @@
-dist: xenial
+dist: bionic
 
 addons:
   hosts:
@@ -86,10 +86,7 @@ install:
   - composer update -n --no-suggest
 
 before_script:
-  - if [ $TEST_SUITE == "functional-ce" ]; then echo "COMPOSER_MAGENTO_USERNAME=${REPO_USERNAME_CE}" >> ./.docker/composer.env && echo "COMPOSER_MAGENTO_PASSWORD=${REPO_PASSWORD_CE}" >> ./.docker/composer.env; fi;
-  - if [ $TEST_SUITE == "functional-ee" ]; then echo "COMPOSER_MAGENTO_USERNAME=${REPO_USERNAME}" >> ./.docker/composer.env && echo "COMPOSER_MAGENTO_PASSWORD=${REPO_PASSWORD}" >> ./.docker/composer.env; fi;
-  - echo "COMPOSER_GITHUB_TOKEN=${GITHUB_TOKEN}" >> ./.docker/composer.env
-  - if [ $XDEBUG == "true" ]; then echo "PHP_ENABLE_XDEBUG=true" >> ./.docker/global.env; fi;
+  - if [ $TEST_SUITE == "functional-ce" ]; then cp codeception.dist.yml codeception.yml && sed -i "s/REPO_USERNAME/REPO_USERNAME_CE/" codeception.yml && sed -i "s/REPO_PASSWORD/REPO_PASSWORD_CE/" codeception.yml; fi;
 
 script:
   - if [ $TEST_SUITE == "functional-ce" ]; then ./tests/travis/functional_ce.sh; fi;

codeception.dist.yml

@@ -2,8 +2,7 @@ paths:
   tests: src/Test/Functional
   output: tests/functional/_output
   data: tests/functional/_data
-  support: tests/functional/_support
-  envs: tests/functional/_envs
+  support: vendor/magento/magento-cloud-docker/tests/functional/_support
 actor_suffix: Tester
 settings:
   colors: true
@@ -12,27 +11,27 @@ extensions:
     - Codeception\Extension\RunFailed
     - Codeception\Extension\FailedInfo
 params:
-  - tests/functional/configuration.dist.yml
+  - vendor/magento/magento-cloud-docker/tests/functional/configuration.dist.yml
+  - env
 modules:
   config:
-    Magento\MagentoCloud\Test\Functional\Codeception\Docker:
-      db_host: "%Magento.docker.settings.db.host%"
-      db_port: "%Magento.docker.settings.db.port%"
-      db_username: "%Magento.docker.settings.db.username%"
-      db_password: "%Magento.docker.settings.db.password%"
-      db_path: "%Magento.docker.settings.db.path%"
-      repo_url: "%Magento.docker.settings.repo.url%"
-      repo_branch: "%Magento.docker.settings.repo.branch%"
-      system_ece_tools_dir: "%Magento.docker.settings.system.ece_dir%"
+    Magento\CloudDocker\Test\Functional\Codeception\TestInfrastructure:
+      template_repo: "https://github.com/magento/magento-cloud.git"
+      mcd_repo: "https://github.com/magento/magento-cloud-docker.git"
+      mcc_repo: "https://github.com/magento/magento-cloud-components.git"
+      mcp_repo: "https://github.com/magento/magento-cloud-patches.git"
+      composer_magento_username: "%REPO_USERNAME%"
+      composer_magento_password: "%REPO_PASSWORD%"
+      composer_github_token: "%GITHUB_TOKEN%"
+      printOutput: false
+    Magento\CloudDocker\Test\Functional\Codeception\Docker:
       system_magento_dir: "%Magento.docker.settings.system.magento_dir%"
-      env_base_url: "%Magento.docker.settings.env.url.base%"
-      env_secure_base_url: "%Magento.docker.settings.env.url.secure_base%"
-      volumes: []
       printOutput: false
     PhpBrowser:
       url: "%Magento.docker.settings.env.url.base%"
-    Magento\MagentoCloud\Test\Functional\Codeception\MagentoDb:
-      dsn: "mysql:host=127.0.0.1;port=3306;dbname=%Magento.docker.settings.db.path%"
+    Magento\CloudDocker\Test\Functional\Codeception\MagentoDb:
+      dsn: "mysql:host=%Magento.docker.settings.db.host%;port=%Magento.docker.settings.db.port%;dbname=%Magento.docker.settings.db.path%"
       user: "%Magento.docker.settings.db.username%"
       password: "%Magento.docker.settings.db.password%"
+      exposed_port: "%Magento.docker.settings.db.port%"
       reconnect: true

composer.json

@@ -93,6 +93,12 @@
   },
   "prefer-stable": true,
   "extra": {
+    "map": [
+      [
+        "dist/.magento.env.yaml",
+        ".magento.env.yaml.dist"
+      ]
+    ],
     "include_files": [
       "vendor/codeception/phpunit-wrapper/src/phpunit7-interfaces.php"
     ]

config/schema.yaml

@@ -195,14 +195,14 @@ MIN_LOGGING_LEVEL:
   type: string
   allowed:
     - ''
-    - !php/const Magento\MagentoCloud\Config\Log::LEVEL_DEBUG
-    - !php/const Magento\MagentoCloud\Config\Log::LEVEL_INFO
-    - !php/const Magento\MagentoCloud\Config\Log::LEVEL_NOTICE
-    - !php/const Magento\MagentoCloud\Config\Log::LEVEL_WARNING
-    - !php/const Magento\MagentoCloud\Config\Log::LEVEL_ERROR
-    - !php/const Magento\MagentoCloud\Config\Log::LEVEL_CRITICAL
-    - !php/const Magento\MagentoCloud\Config\Log::LEVEL_ALERT
-    - !php/const Magento\MagentoCloud\Config\Log::LEVEL_EMERGENCY
+    - 'debug'
+    - 'info'
+    - 'notice'
+    - 'warning'
+    - 'error'
+    - 'critical'
+    - 'alert'
+    - 'emergency'
   stages:
     - global
   default:

src/Config/Schema.php

@@ -8,6 +8,7 @@
 namespace Magento\MagentoCloud\Config;
 
 use Magento\MagentoCloud\Filesystem\SystemList;
+use Magento\MagentoCloud\Filesystem\Driver\File;
 use Symfony\Component\Yaml\Parser;
 use Symfony\Component\Yaml\Yaml;
 
@@ -39,6 +40,11 @@ class Schema
      */
     private $parser;
 
+    /**
+     * @var File
+     */
+    private $file;
+
     /**
      * @var array
      */
@@ -47,18 +53,24 @@ class Schema
     /**
      * @param SystemList $systemList
      * @param Parser $parser
+     * @param File $file
      */
-    public function __construct(SystemList $systemList, Parser $parser)
-    {
+    public function __construct(
+        SystemList $systemList,
+        Parser $parser,
+        File $file
+    ) {
         $this->systemList = $systemList;
         $this->parser = $parser;
+        $this->file = $file;
     }
 
     /**
      * Returns default values for given stage.
      *
      * @param string $stage
      * @return array
+     * @throws \Magento\MagentoCloud\Filesystem\FileSystemException
      */
     public function getDefaults(string $stage): array
     {
@@ -85,12 +97,12 @@ public function getDefaults(string $stage): array
      * 'default_values' - array of default values
      *
      * @return array
-     * @SuppressWarnings(PHPMD.ExcessiveMethodLength)
+     * @throws \Magento\MagentoCloud\Filesystem\FileSystemException
      */
     public function getSchema(): array
     {
-        return $this->parser->parseFile(
-            $this->systemList->getConfig() . '/schema.yaml',
+        return $this->parser->parse(
+            $this->file->fileGetContents($this->systemList->getConfig() . '/schema.yaml'),
             Yaml::PARSE_CONSTANT
         );
     }

src/Test/Functional/Acceptance.suite.dist.yml

@@ -1,7 +1,8 @@
 actor: CliTester
 modules:
   enabled:
-    - Magento\MagentoCloud\Test\Functional\Codeception\Docker
-    - Magento\MagentoCloud\Test\Functional\Codeception\MagentoDb
+    - Magento\CloudDocker\Test\Functional\Codeception\TestInfrastructure
+    - Magento\CloudDocker\Test\Functional\Codeception\Docker
+    - Magento\CloudDocker\Test\Functional\Codeception\MagentoDb
     - PhpBrowser
     - Asserts

src/Test/Functional/Acceptance/AbstractCest.php

@@ -13,13 +13,22 @@
  */
 abstract class AbstractCest
 {
+    /**
+     * @var boolean
+     */
+    protected $removeEs = true;
+
+    /**
+     * @var string
+     */
+    protected $magentoCloudTemplate = 'master';
+
     /**
      * @param \CliTester $I
      */
-    public function _before(\CliTester $I)
+    public function _before(\CliTester $I): void
     {
-        $I->generateDockerCompose();
-        $I->cleanUpEnvironment();
+        $this->prepareWorkplace($I, $this->magentoCloudTemplate);
     }
 
     /**
@@ -28,6 +37,66 @@ public function _before(\CliTester $I)
     public function _after(\CliTester $I): void
     {
         $I->stopEnvironment();
-        $I->removeDockerCompose();
+        $I->removeWorkDir();
+    }
+
+    /**
+     * @param array $data
+     * @return string
+     */
+    protected function convertEnvFromArrayToJson(array $data): string
+    {
+        return addslashes(json_encode($data));
+    }
+
+    /**
+     * @param \CliTester $I
+     * @param string $version
+     */
+    protected function prepareWorkplace(\CliTester $I, string $version): void
+    {
+        $I->cleanupWorkDir();
+        $I->cloneTemplateToWorkDir($version);
+        $I->createAuthJson();
+        $I->createArtifactsDir();
+        $I->createArtifactCurrentTestedCode('ece-tools', '2002.1.99');
+        $I->addArtifactsRepoToComposer();
+        $I->addDependencyToComposer('magento/ece-tools', '2002.1.99');
+        $I->addEceDockerGitRepoToComposer();
+        $I->addCloudComponentsGitRepoToComposer();
+        $I->addCloudPatchesGitRepoToComposer();
+        $I->addDependencyToComposer(
+            'magento/magento-cloud-docker',
+            $I->getDependencyVersion('magento/magento-cloud-docker')
+        );
+        $I->addDependencyToComposer(
+            'magento/magento-cloud-components',
+            $I->getDependencyVersion('magento/magento-cloud-components')
+        );
+        $I->addDependencyToComposer(
+            'magento/magento-cloud-patches',
+            $I->getDependencyVersion('magento/magento-cloud-patches')
+        );
+        $I->composerUpdate();
+        $this->removeESIfExists($I);
+    }
+
+    /**
+     * @param \CliTester $I
+     */
+    protected function removeESIfExists(\CliTester $I): void
+    {
+        if ($this->removeEs) {
+            $services = $I->readServicesYaml();
+
+            if (isset($services['elasticsearch'])) {
+                unset($services['elasticsearch']);
+                $I->writeServicesYaml($services);
+
+                $app = $I->readAppMagentoYaml();
+                unset($app['relationships']['elasticsearch']);
+                $I->writeAppMagentoYaml($app);
+            }
+        }
     }
 }

src/Test/Functional/Acceptance/AcceptanceCeCest.php

@@ -7,7 +7,7 @@
 
 namespace Magento\MagentoCloud\Test\Functional\Acceptance;
 
-use Magento\MagentoCloud\Test\Functional\Codeception\Docker;
+use Magento\CloudDocker\Test\Functional\Codeception\Docker;
 use Robo\Exception\TaskException;
 use CliTester;
 
@@ -16,9 +16,16 @@
  *
  * @group edition-ce
  */
-class AcceptanceCeCest extends AbstractInstallCest
+class AcceptanceCeCest extends AbstractCest
 {
-    public const EDITION = 'CE';
+    public function _before(\CliTester $I): void
+    {
+        parent::_before($I);
+
+        $I->removeDependencyFromComposer('magento/magento-cloud-metapackage');
+        $I->addDependencyToComposer('magento/product-community-edition', '@stable');
+        $I->composerUpdate();
+    }
 
     /**
      * @param CliTester $I
@@ -27,11 +34,10 @@ class AcceptanceCeCest extends AbstractInstallCest
      */
     public function testWithSplitBuildCommand(\CliTester $I): void
     {
-        $I->assertTrue($I->runEceToolsCommand('build:generate', Docker::BUILD_CONTAINER));
-        $I->assertTrue($I->runEceToolsCommand('build:transfer', Docker::BUILD_CONTAINER));
+        $I->runEceDockerCommand('build:compose --mode=production');
         $I->startEnvironment();
-        $I->assertTrue($I->runEceToolsCommand('deploy', Docker::DEPLOY_CONTAINER));
-        $I->assertTrue($I->runEceToolsCommand('post-deploy', Docker::DEPLOY_CONTAINER));
+        $I->runDockerComposeCommand('run build cloud-build');
+        $I->runDockerComposeCommand('run deploy cloud-deploy');
         $I->amOnPage('/');
         $I->see('Home page');
         $I->see('CMS homepage content goes here.');