SC-7808: support apple m1 (#273)

* Docker Compose V2 * Revert wait * SC-8299: added test images * Fix docker-compose version check bug. (#265) With newer Version docker the "docker-compose version --short" command gives out a version number string with "v" at the beginning, so the later comparison failed with this misleading error message: `Docker Compose version v2.1.1 is not supported. Please update Docker Compose to at least 1.22.0. ` With the help of "tr" to remove the leading "v" will fix this problem. * SC-8301: warning functionality * Docker Compose V2 + Mutage * Update compose.sh * Adjust docker compose runner * SC-8242: adjusted templates for choosing compose version * SC-8242: provided env variable for compose version * SC-8301: updated organisation for socat image * SC-8301: added new version of services * SC-8301: fix typo jenkins version * SC-8301: reverted default version * SC-8242: updated after QA * SC-8242: updated troubleshooting * Update docker-compose.sh * SC-8301: adjusted prerelease version * SC-8301: adjusted local version * SC-8301: adjusted local version * SC-8301: added jenkins csrf protection * SC-8301: added jenkins csrf protection * fix missing ; in the templates * SC-8259: update troubleshooting * SC-8259: renamed csrf-protection property to csrf-protection-enabled * SC-8259: updated service version compatibility validation * SC-8259: updated README.md * SC-8259: added jenkins csrf protection issue into troubleshooting * SC-8259: reverted generator image * Update deploy.bash.twig * Update mutagen.sh Co-authored-by: alexanderM91 <oleksandr.myrnyi@spryker.com> Co-authored-by: Filip Sushko <filip.sushko@spruker.com> Co-authored-by: greenflute <greenflute@hotmail.com> Co-authored-by: Sylvain Gaffie <sylvain.gaffie@spryker.com>
spryker · May 2, 2022 · 3966b3b · 3966b3b
1 parent c55f507
commit 3966b3b
Show file tree

Hide file tree

Showing 16 changed files with 324 additions and 35 deletions.
diff --git a/README.md b/README.md
@@ -54,31 +54,35 @@ For contribution guidelines, see [Code contribution guide](https://documentation
 
 ## Supported services
 
-| Service  | Engine  | Version(s)  |
-|---|---|---|
-| database  | postgres  | 9.6*  |
-|   |   | 10  |
-|   |   | 11  |
-|   |   | 12  |
-|   | mysql  | 5.7  |
-|   |   | mariadb-10.2  |
-|   |   | mariadb-10.3  |
-|   |   | mariadb-10.4  |
-|   |   | mariadb-10.5  |
-| broker  | rabbitmq  | 3.7  |
-| session  | redis  | 5.0  |
-| key_value_store  | redis  | 5.0  |
-| search  | elastic  | 5.6* |
-|   |   | 6.8  |
-|   |   | 7.6  |
-|   |   | 7.10 |
-| scheduler  | jenkins  | 2.176  |
-| webdriver  | phantomjs  | latest* |
-|   | chromedriver  | latest |
-| mail_catcher  | mailhog  | 1.0  |
-| swagger  | swagger-ui  | v3.24  |
-| kibana  | kibana  | 5.6* |
-|   |   | 6.8 |
-|   |   | 7.6 |
-|   |   | 7.10 |
-| blackfire  | blackfire  | latest |
+| Service         | Engine       | Version(s)   | ARM support |
+|-----------------|--------------|--------------|-------------|
+| database        | postgres     | 9.6*         | x           |
+|                 |              | 10           | x           |
+|                 |              | 11           | x           |
+|                 |              | 12           | x           |
+|                 | mysql        | 5.7          |             |
+|                 |              | mariadb-10.2 | x           |
+|                 |              | mariadb-10.3 | x           |
+|                 |              | mariadb-10.4 | x           |
+|                 |              | mariadb-10.5 | x           |
+| broker          | rabbitmq     | 3.7          |             |
+|                 |              | 3.8          | x           |
+|                 |              | 3.9          | x           |
+| session         | redis        | 5.0          | x           |
+| key_value_store | redis        | 5.0          | x           |
+| search          | elastic      | 5.6*         |             |
+|                 |              | 6.8          |             |
+|                 |              | 7.6          | x           |
+|                 |              | 7.10         | x           |
+| scheduler       | jenkins      | 2.176        |             |
+|                 |              | 2.305        | x           |
+|                 |              | 2.324        | x           |
+| webdriver       | phantomjs    | latest*      |             |
+|                 | chromedriver | latest       | x           |
+| mail_catcher    | mailhog      | 1.0          | x           |
+| swagger         | swagger-ui   | v3.24        | x           |
+| kibana          | kibana       | 5.6*         |             |
+|                 |              | 6.8          |             |
+|                 |              | 7.6          | x           |
+|                 |              | 7.10         | x           |
+| blackfire       | blackfire    | latest       | x           |
diff --git a/bin/service/scheduler/jenkins.sh b/bin/service/scheduler/jenkins.sh
@@ -8,7 +8,7 @@ function Service::Scheduler::isInstalled() {
 
     # shellcheck disable=SC2016
     # For avoid https://github.com/docker/compose/issues/9104
-    local jobsCount=$(Compose::exec 'curl -sL ${SPRYKER_SCHEDULER_HOST}:${SPRYKER_SCHEDULER_PORT}/scriptText -d "script=println Jenkins.instance.projects.collect{ it.name }.size" | tail -n 1' "${DOCKER_COMPOSE_TTY_DISABLED}"| tr -d " \n\r")
+    local jobsCount=$(Jenkins::callJenkins 'scriptText -d "script=println Jenkins.instance.projects.collect{ it.name }.size"| tail -n 1')
     [ "${jobsCount}" -gt 0 ] && Console::end "[INSTALLED]" && return "${TRUE}" || return "${FALSE}"
 }
 
@@ -28,7 +28,7 @@ Service::Scheduler::pause() {
     while :; do
         # shellcheck disable=SC2016
         # For avoid https://github.com/docker/compose/issues/9104
-        local runningJobsCount=$(Compose::exec 'curl -sL ${SPRYKER_SCHEDULER_HOST}:${SPRYKER_SCHEDULER_PORT}/computer/api/xml?xpath=*/busyExecutors/text\(\) | tail -n 1' "${DOCKER_COMPOSE_TTY_DISABLED}"| tr -d " \n\r")
+        local runningJobsCount=$(Jenkins::callJenkins 'computer/api/xml?xpath=*/busyExecutors/text\(\) | tail -n 1')
         [ "${runningJobsCount}" -eq 0 ] && break
         [ "${counter}" -ge "${waitFor}" ] && break
         counter=$((counter + interval))
@@ -87,3 +87,30 @@ function Service::Scheduler::_run() {
         done
     done
 }
+
+function Jenkins::callJenkins() {
+    local uri=${1}
+
+    local cookieJar="/data/jenkins_cookie_jar"
+    local statusCode=$(Compose::exec 'curl -o /dev/null -s -w "%{http_code}\n" ${SPRYKER_SCHEDULER_HOST}:${SPRYKER_SCHEDULER_PORT}/crumbIssuer/api/json | tail -n 1' "${DOCKER_COMPOSE_TTY_DISABLED}"| tr -d " \n\r")
+    local curlOptions='-sL'
+
+    local crumbToken=''
+    local crumbHeader=''
+    local composeCommand=''
+
+    if [ "${statusCode}" -ne '200' ]; then
+        composeCommand=$(printf 'curl %s ${SPRYKER_SCHEDULER_HOST}:${SPRYKER_SCHEDULER_PORT}/%s' "${curlOptions}" "${uri}")
+    else
+        Compose::exec 'rm -f '${cookieJar}' && touch '${cookieJar}
+        crumbToken=$(Compose::exec 'curl -sL --cookie-jar '"${cookieJar}"' ${SPRYKER_SCHEDULER_HOST}:${SPRYKER_SCHEDULER_PORT}/crumbIssuer/api/json | jq -r ".crumbRequestField + \":\" + .crumb"' "${DOCKER_COMPOSE_TTY_DISABLED}"| tr -d " \n\r")
+        crumbHeader="-H \"${crumbToken}\" --cookie \"${cookieJar}\""
+        curlOptions+=' '${crumbHeader}
+        composeCommand=$(printf 'curl %s ${SPRYKER_SCHEDULER_HOST}:${SPRYKER_SCHEDULER_PORT}/%s' "${curlOptions}" "${uri}")
+    fi
+
+    local result=$(Compose::exec ${composeCommand} "${DOCKER_COMPOSE_TTY_DISABLED}"| tr -d " \n\r")
+    Compose::exec 'rm -f '${cookieJar}
+
+    echo ${result}
+}
diff --git a/docs/09-troubleshooting.md b/docs/09-troubleshooting.md
@@ -327,3 +327,65 @@ You get an error after running `docker/sdk cli {ARGUMENT_1}`.
 
 **then**
 Wrap the command arguments into single quotes. For example, `docker/sdk cli 'composer require spryker/*'`
+
+**when**
+`Node Sass does not yet support your current environment: Linux Unsupported architecture (arm64) with Node.js`
+
+**then**
+1. remove `node-sass` dependencies in `package.json`
+2. add `sass` and `sass-loader`
+```
+...
+"sass": "~1.32.13",
+"sass-loader": "~10.2.0",
+...
+```
+3. update `@spryker/oryx-for-zed`
+```
+...
+"@spryker/oryx-for-zed": "~2.11.5",
+...
+```
+4. add option to sass-loader (`frontend/configs/development.js`)
+```
+loader: 'sass-loader',
+options: {
+   implementation: require('sass'),
+}
+```
+5. run `docker/sdk cli`
+6. run `npm install` to update `package-lock.json` and install dependencies
+7. (if yarn usage) run `yarn install` to update `package-lock.json` and install dependencies
+8. run `npm run yves` to rebuild yves
+9. run `npm run zed` to rebuild zed
+
+**when**
+Error 403 No valid crumb was included in the request
+
+**then**
+Check your project configuration. Jenkins CSRF protection should be enabled.
+```php
+...
+$config[SchedulerJenkinsConstants::JENKINS_CONFIGURATION] = [
+    SchedulerConfig::SCHEDULER_JENKINS => [
+        SchedulerJenkinsConfig::SCHEDULER_JENKINS_CSRF_ENABLED => true,
+    ],
+];
+...
+```
+You can use `SPRYKER_JENKINS_CSRF_PROTECTION_ENABLED` env variable. This variable depends on deploy file parameter from `scheduler`
+```yaml
+services:
+  scheduler:
+    csrf-protection-enabled: { true | false }
+
+```
+```php
+...
+$config[SchedulerJenkinsConstants::JENKINS_CONFIGURATION] = [
+    SchedulerConfig::SCHEDULER_JENKINS => [
+        SchedulerJenkinsConfig::SCHEDULER_JENKINS_CSRF_ENABLED => (bool)getenv('SPRYKER_JENKINS_CSRF_PROTECTION_ENABLED'),
+    ],
+];
+...
+```
diff --git a/generator/index.php b/generator/index.php
@@ -519,6 +519,18 @@ static function ($endpoint) use ($projectData) {
 
 verbose(implode(PHP_EOL, $output));
 
+$errorMessages = validateServiceVersions($projectData);
+
+if (count($errorMessages) > 0) {
+    $redColorCode = "\033[31m";
+
+    warn($redColorCode . 'Service version compatibility errors:' . PHP_EOL);
+    warn($redColorCode . ' * ' . implode(PHP_EOL . $redColorCode . ' * ' , $errorMessages));
+    warn(PHP_EOL . $redColorCode . 'Please check documentation.');
+
+    exit(1);
+}
+
 // -------------------------
 /**
  * @param array $projectData
@@ -1431,3 +1443,103 @@ function buildDefaultCredentialsForBroker(array $projectData): array
 
     return $projectData;
 }
+
+/**
+ * @param array $projectData
+ * @return string[]
+ */
+function validateServiceVersions(array $projectData): array
+{
+    $validationMessageTemplate = '`%s` service with `%s` engine and %s version are unsupported on ARM architecture.';
+    $validationMessages = [];
+
+    if (!isArmArchitecture()) {
+        return $validationMessages;
+    }
+
+    $services = $projectData['services'];
+    $unsupportedServiceVersions = getUnsupportedArmServiceMap();
+
+    foreach ($unsupportedServiceVersions as $serviceName => $serviceEngines) {
+        if (!array_key_exists($serviceName, $services)) {
+            continue;
+        }
+
+        $service = $services[$serviceName];
+        $serviceEngine = $service['engine'] ?? null;
+        $serviceVersion = $service['version'] ?? 'default';
+
+        if($serviceEngine == null || !array_key_exists($serviceEngine, $serviceEngines)) {
+            continue;
+        }
+
+        if (!array_key_exists($serviceVersion, $serviceEngines[$serviceEngine])) {
+            continue;
+        }
+
+        $validationMessages[] = sprintf($validationMessageTemplate, $serviceName, $serviceEngine, $serviceEngines[$serviceEngine][$serviceVersion]);
+    }
+
+    return $validationMessages;
+}
+
+/**
+ * @return string[][][]
+ */
+function getUnsupportedArmServiceMap(): array
+{
+    return [
+        'database' => [
+            'mysql' => [
+                '5.7' => '5.7',
+                'default' => '5.7',
+            ],
+        ],
+        'broker' => [
+            'rabbitmq' => [
+                '3.7' => '3.7',
+                'default' => '3.7',
+            ],
+        ],
+        'webdriver' => [
+            'phantomjs' => ['*'],
+        ],
+        'search' => [
+            'elastic' => [
+                '5.6' => '5.6',
+                '6.8' => '6.8',
+                'default' => '5.6',
+            ],
+        ],
+        'kibana' => [
+            'kibana' => [
+                '5.6' => '5.6',
+                '6.8' => '6.8',
+                'default' => '5.6',
+            ],
+        ],
+        'scheduler' => [
+            'jenkins' => [
+                '2.176' => '2.176',
+                'default' => '2.176',
+            ],
+        ],
+    ];
+}
+
+/**
+ * @return bool
+ */
+function isArmArchitecture(): bool
+{
+    $possibleValue = [
+        'arm',
+        'aarch64_be',
+        'aarch64',
+        'armv8l',
+    ];
+
+    $currentArchitecture = php_uname('m');
+
+    return in_array($currentArchitecture, $possibleValue);
+}
diff --git a/generator/src/templates/docker-compose.yml.twig b/generator/src/templates/docker-compose.yml.twig
@@ -8,7 +8,7 @@ x-volumes:
 services:
 
   cli_ssh_relay:
-    image: bpack/socat
+    image: spryker/socat:latest
     privileged: true
     command: UNIX-LISTEN:/tmp/ssh-relay/ssh-auth.sock,fork,unlink-early,user=1000,group=1000,mode=666 UNIX-CONNECT:/tmp/ssh-relay/input-ssh-auth.sock
     networks:

diff --git a/generator/src/templates/env/scheduler/jenkins.env.twig b/generator/src/templates/env/scheduler/jenkins.env.twig
@@ -1,3 +1,6 @@
 SPRYKER_SCHEDULER_HOST={{ serviceName }}
 SPRYKER_SCHEDULER_PORT={{ serviceData['port'] | default(8080) }}
 SPRYKER_JENKINS_TEMPLATE_PATH=/home/spryker/jenkins.docker.xml.twig
+
+SPRYKER_JENKINS_CSRF_PROTECTION_ENABLED="{{ serviceData['csrf-protection-enabled'] }}"
+
diff --git a/generator/src/templates/nginx/http/application.server.conf.twig b/generator/src/templates/nginx/http/application.server.conf.twig
@@ -38,6 +38,7 @@
         fastcgi_param SPRYKER_SSL_ENABLE {{ (project['docker']['ssl']['enabled'] | default(false)) ? 1 : 0 }};
         fastcgi_param SPRYKER_ZED_SSL_ENABLED {{ (project['docker']['ssl']['enabled'] | default(false)) ? 1 : 0 }};
         fastcgi_param HTTPS {{ (project['docker']['ssl']['enabled'] | default(false)) ? 'on' : 'off' }};
+        fastcgi_param SPRYKER_JENKINS_CSRF_PROTECTION_ENABLED {{ (services['scheduler']['csrf-protection-enabled'] | default(false)) ? 1 : 0 }};
 {% endblock location %}
     }
 {% endblock locations %}
diff --git a/generator/src/templates/service/jenkins/2.305/jenkins.yml.twig b/generator/src/templates/service/jenkins/2.305/jenkins.yml.twig
@@ -0,0 +1,18 @@
+  {{ serviceName }}:
+    image: spryker/jenkins:2.305
+    networks:
+      - private
+    labels:
+      'spryker.app.name': scheduler
+      'spryker.app.type': services
+      'spryker.project': ${SPRYKER_DOCKER_PREFIX}:${SPRYKER_DOCKER_TAG}
+    healthcheck:
+      test: [ "CMD", "nc", "-z", "localhost", "8080" ]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    environment:
+        JAVA_OPTS: '-Djenkins.install.runSetupWizard=false -Xms128m -Xmx128m -Dhudson.security.csrf.GlobalCrumbIssuerConfiguration.DISABLE_CSRF_PROTECTION={{ serviceData['csrf-protection-enabled'] ? 'false' : 'true' }}'
+    volumes:
+      - {{ serviceName }}-{{ serviceData['engine'] }}-data:/var/jenkins_home:rw
+      - ${DEPLOYMENT_PATH}/context/jenkins/spryker.sh:/usr/bin/spryker.sh:rw
diff --git a/generator/src/templates/service/jenkins/2.324/jenkins.yml.twig b/generator/src/templates/service/jenkins/2.324/jenkins.yml.twig
@@ -0,0 +1,18 @@
+  {{ serviceName }}:
+    image: spryker/jenkins:2.324
+    networks:
+      - private
+    labels:
+      'spryker.app.name': scheduler
+      'spryker.app.type': services
+      'spryker.project': ${SPRYKER_DOCKER_PREFIX}:${SPRYKER_DOCKER_TAG}
+    healthcheck:
+      test: [ "CMD", "nc", "-z", "localhost", "8080" ]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    environment:
+        JAVA_OPTS: '-Djenkins.install.runSetupWizard=false -Xms128m -Xmx128m -Dhudson.security.csrf.GlobalCrumbIssuerConfiguration.DISABLE_CSRF_PROTECTION={{ serviceData['csrf-protection-enabled'] ? 'false' : 'true' }}'
+    volumes:
+      - {{ serviceName }}-{{ serviceData['engine'] }}-data:/var/jenkins_home:rw
+      - ${DEPLOYMENT_PATH}/context/jenkins/spryker.sh:/usr/bin/spryker.sh:rw
diff --git a/generator/src/templates/service/jenkins/scheduler.bash.twig b/generator/src/templates/service/jenkins/scheduler.bash.twig
@@ -0,0 +1 @@
+readonly SPRYKER_JENKINS_CSRF_PROTECTION_ENABLED="{{ services['scheduler']['csrf-protection-enabled'] ? '1' : '0' }}"
diff --git a/generator/src/templates/service/rabbitmq/3.8/rabbitmq.yml.twig b/generator/src/templates/service/rabbitmq/3.8/rabbitmq.yml.twig
@@ -0,0 +1,19 @@
+  {{ serviceName }}:
+    image: spryker/rabbitmq:3.8
+    hostname: {{ serviceName }} # RabbitMQ determines the cluster by its hostname
+    networks:
+      - private
+    labels:
+      'spryker.app.name': broker
+      'spryker.app.type': services
+      'spryker.project': ${SPRYKER_DOCKER_PREFIX}:${SPRYKER_DOCKER_TAG}
+    healthcheck:
+      test: [ "CMD", "nc", "-z", "localhost", "5672" ]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+    environment:
+      RABBITMQ_DEFAULT_USER: "{{ serviceData['api']['username'] }}"
+      RABBITMQ_DEFAULT_PASS: "{{ serviceData['api']['password'] }}"
+    volumes:
+      - {{ serviceName }}-{{ serviceData['engine'] }}-data:/var/lib/rabbitmq:rw