Permalink
Browse files

Merge pull request #827 from amazeeio/release/v0.19.0

Release/v0.19.0
  • Loading branch information...
Schnitzel committed Jan 14, 2019
2 parents 2de6dc1 + aae899b commit 7d1d265b41211be5f493b80a53aa158f1d60d7f8
Showing with 6,670 additions and 903 deletions.
  1. +15 −7 Makefile
  2. +6 −0 docker-compose.yaml
  3. +2 −1 docs/administering_lagoon/create-project.gql
  4. +1 −1 docs/using_lagoon/docker_images/php-fpm.md
  5. +2 −1 docs/using_lagoon/drupal/drupal7-mariadb/sites/default/settings.php
  6. +13 −3 docs/using_lagoon/drupal/services/solr.md
  7. +2 −2 docs/using_lagoon/environment_types.md
  8. +6 −0 docs/using_lagoon/lagoon_yml.md
  9. +3 −1 docs/using_lagoon/service_types.md
  10. +1 −1 helpers/annotate-pvc-backup.sh
  11. +100 −0 helpers/mariadb-single2shared.sh
  12. +5 −3 images/mariadb-galera/Dockerfile
  13. 0 images/mariadb-galera/{fix-data-subpath.sh → entrypoints/100-fix-data-subpath.sh}
  14. 0 images/{mariadb/logging.bash → mariadb-galera/entrypoints/100-mariadb-logging.bash}
  15. +13 −0 images/mariadb-galera/entrypoints/150-mariadb-performance.bash
  16. 0 images/{mariadb/envplate.bash → mariadb-galera/entrypoints/200-mariadb-envplate.bash}
  17. 0 images/mariadb-galera/{maxscale-entrypoint.sh → entrypoints/201-maxscale-envplate.bash}
  18. +0 −2 images/mariadb-galera/{docker-entrypoint.bash → entrypoints/9999-mariadb-galera-init.bash}
  19. +9 −8 images/mariadb-galera/my.cnf
  20. +1 −1 images/mariadb-galera/mysql-backup.sh
  21. +4 −11 images/mariadb/Dockerfile
  22. +24 −0 images/mariadb/entrypoints/100-mariadb-logging.bash
  23. +13 −0 images/mariadb/entrypoints/150-mariadb-performance.bash
  24. +5 −0 images/mariadb/entrypoints/200-mariadb-envplate.bash
  25. +0 −2 images/mariadb/{docker-entrypoint.bash → entrypoints/9999-mariadb-init.bash}
  26. +7 −8 images/mariadb/my.cnf
  27. +1 −1 images/mariadb/mysql-backup.sh
  28. +0 −13 images/mariadb/root/usr/include/bits/wordsize.h
  29. +11 −0 images/nginx/helpers/030_ipv6-xfwdfor.conf
  30. +65 −3 images/oc-build-deploy-dind/build-deploy-docker-compose.sh
  31. +1 −1 images/oc-build-deploy-dind/openshift-templates/cli-persistent/deployment.yml
  32. +1 −1 images/oc-build-deploy-dind/openshift-templates/cli/deployment.yml
  33. +1 −1 images/oc-build-deploy-dind/openshift-templates/mariadb-galera/deployment.yml
  34. +2 −0 images/oc-build-deploy-dind/openshift-templates/mariadb-galera/statefulset.yml
  35. +8 −2 images/oc-build-deploy-dind/openshift-templates/mariadb-shared/servicebroker.yml
  36. 0 images/oc-build-deploy-dind/openshift-templates/{mariadb → mariadb-single}/cronjobs.yml
  37. +1 −1 images/oc-build-deploy-dind/openshift-templates/{mariadb → mariadb-single}/deployment.yml
  38. 0 images/oc-build-deploy-dind/openshift-templates/{mariadb → mariadb-single}/pvc.yml
  39. 0 images/oc-build-deploy-dind/openshift-templates/{mariadb → mariadb-single}/services.yml
  40. +1 −1 images/oc-build-deploy-dind/openshift-templates/solr/deployment.yml
  41. +20 −0 images/oc-build-deploy-dind/scripts/exec-openshift-create-servicebroker.sh
  42. +3 −1 images/oc/Dockerfile
  43. +2 −0 images/php/cli/Dockerfile
  44. +1 −1 images/php/fpm/00-lagoon-php.ini.tpl
  45. +10 −3 images/php/fpm/Dockerfile
  46. +4 −1 images/php/fpm/entrypoints/60-php-xdebug.sh
  47. +3 −3 images/php/fpm/entrypoints/71-php-newrelic.sh
  48. +3 −1 images/solr-drupal/Dockerfile
  49. +10 −10 images/solr-drupal/solr5.5/conf/solrconfig.xml
  50. +9 −9 images/solr-drupal/solr6.6/conf/solrconfig.xml
  51. +27 −0 images/solr-drupal/solr7.5/conf/elevate.xml
  52. +14 −0 images/solr-drupal/solr7.5/conf/mapping-ISOLatin1Accent.txt
  53. +7 −0 images/solr-drupal/solr7.5/conf/protwords.txt
  54. +696 −0 images/solr-drupal/solr7.5/conf/schema.xml
  55. +13 −0 images/solr-drupal/solr7.5/conf/schema_extra_fields.xml
  56. +59 −0 images/solr-drupal/solr7.5/conf/schema_extra_types.xml
  57. +61 −0 images/solr-drupal/solr7.5/conf/schema_legacy_fields.xml
  58. +14 −0 images/solr-drupal/solr7.5/conf/schema_legacy_types.xml
  59. +1,413 −0 images/solr-drupal/solr7.5/conf/solrconfig.xml
  60. +12 −0 images/solr-drupal/solr7.5/conf/solrconfig_extra.xml
  61. 0 images/solr-drupal/solr7.5/conf/solrconfig_index.xml
  62. +89 −0 images/solr-drupal/solr7.5/conf/solrconfig_spellcheck.xml
  63. +20 −0 images/solr-drupal/solr7.5/conf/solrcore.properties
  64. +4 −0 images/solr-drupal/solr7.5/conf/stopwords.txt
  65. +3 −0 images/solr-drupal/solr7.5/conf/synonyms.txt
  66. +133 −0 images/solr/20-solr-datadir.sh
  67. +10 −4 images/solr/Dockerfile
  68. +5 −0 images/varnish-drupal/drupal.vcl
  69. +23 −2 local-dev/api-data/01-populate-api-data.gql
  70. +2 −1 local-dev/git/Dockerfile
  71. +1 −1 node-packages/commons/package.json
  72. +1 −1 node-packages/commons/src/api.js
  73. +6 −0 node-packages/commons/src/jwt.js
  74. +655 −0 openshift-setup/router-template.cfg
  75. +14 −0 services/api-db/docker-entrypoint-initdb.d/00-tables.sql
  76. +21 −0 services/api-db/docker-entrypoint-initdb.d/01-migrations.sql
  77. +6 −2 services/api/package.json
  78. +67 −0 services/api/src/apolloServer.js
  79. +23 −46 services/api/src/app.js
  80. +0 −125 services/api/src/authKeycloakMiddleware.js
  81. +54 −112 services/api/src/authMiddleware.js
  82. +22 −0 services/api/src/clients/aws.js
  83. +81 −0 services/api/src/clients/pubSub.js
  84. +1 −4 services/api/src/index.js
  85. +25 −0 services/api/src/resolvers.js
  86. +7 −0 services/api/src/resources/backup/events.js
  87. +36 −7 services/api/src/resources/backup/resolvers.js
  88. +6 −0 services/api/src/resources/deployment/events.js
  89. +19 −3 services/api/src/resources/deployment/resolvers.js
  90. +1 −1 services/api/src/resources/deployment/sql.js
  91. +4 −4 services/api/src/resources/env-variables/sql.js
  92. +15 −0 services/api/src/resources/environment/helpers.js
  93. +5 −10 services/api/src/resources/environment/resolvers.js
  94. +72 −0 services/api/src/resources/environment/validators.js
  95. +152 −0 services/api/src/resources/file/resolvers.js
  96. +69 −0 services/api/src/resources/file/sql.js
  97. +5 −1 services/api/src/resources/index.js
  98. +6 −0 services/api/src/resources/task/events.js
  99. +117 −0 services/api/src/resources/task/helpers.js
  100. +144 −115 services/api/src/resources/task/resolvers.js
  101. +0 −6 services/api/src/resources/task/sql.js
  102. +22 −0 services/api/src/resources/user/resolvers.js
  103. +4 −0 services/api/src/resources/user/sql.js
  104. +0 −22 services/api/src/routes/graphql.js
  105. +0 −4 services/api/src/routes/index.js
  106. +6 −11 services/api/src/server.js
  107. +35 −0 services/api/src/typeDefs.js
  108. +144 −1 services/api/src/util/auth.js
  109. +1 −1 services/auto-idler/idle-services.sh
  110. +108 −11 services/logs2rocketchat/src/readFromRabbitMQ.js
  111. +1 −0 services/openshiftbuilddeploy/package.json
  112. +11 −0 services/openshiftbuilddeploy/src/index.js
  113. +17 −7 services/openshiftbuilddeploymonitor/src/index.js
  114. +3 −2 services/openshiftjobs/Dockerfile
  115. +45 −1 services/openshiftjobs/src/index.js
  116. +7 −2 services/openshiftremove/src/index.js
  117. +33 −6 services/rest2tasks/src/index.js
  118. +3 −0 services/ssh/etc/ssh/sshd_config
  119. +1 −0 services/ui/package.json
  120. +3 −0 services/ui/src/components/AddTask/components/Completed.js
  121. +96 −0 services/ui/src/components/AddTask/components/DrushArchiveDump.js
  122. +120 −0 services/ui/src/components/AddTask/components/DrushRsyncFiles.js
  123. +120 −0 services/ui/src/components/AddTask/components/DrushSqlSync.js
  124. +4 −0 services/ui/src/components/AddTask/components/Empty.js
  125. +3 −0 services/ui/src/components/AddTask/components/Error.js
  126. +36 −0 services/ui/src/components/AddTask/components/logic.js
  127. +104 −0 services/ui/src/components/AddTask/index.js
  128. +91 −0 services/ui/src/components/AddTask/logic.js
  129. +51 −1 services/ui/src/components/Deployments/index.js
  130. +1 −0 services/ui/src/components/Project/index.js
  131. +1 −1 services/ui/src/components/RestoreButton/Prepare.js
  132. +1 −1 services/ui/src/components/RestoreButton/index.js
  133. +22 −0 services/ui/src/components/Task/index.js
  134. +5 −93 services/ui/src/components/Tasks/index.js
  135. +0 −28 services/ui/src/components/Tasks/logic.js
  136. +0 −44 services/ui/src/components/Tasks/withTaskMutation.js
  137. +29 −6 services/ui/src/lib/ApiConnection.js
  138. +2 −1 services/ui/src/lib/withKeycloak.js
  139. +69 −8 services/ui/src/pages/backups.js
  140. +67 −3 services/ui/src/pages/tasks.js
  141. +7 −3 services/webhooks2tasks/src/handlers/bitbucketBranchDeleted.js
  142. +6 −1 services/webhooks2tasks/src/handlers/bitbucketPullRequestClosed.js
  143. +3 −0 services/webhooks2tasks/src/handlers/bitbucketPullRequestUpdated.js
  144. +8 −2 services/webhooks2tasks/src/handlers/bitbucketPush.js
  145. +5 −2 services/webhooks2tasks/src/handlers/githubBranchDeleted.js
  146. +7 −1 services/webhooks2tasks/src/handlers/githubPullRequestClosed.js
  147. +3 −0 services/webhooks2tasks/src/handlers/githubPullRequestOpened.js
  148. +3 −0 services/webhooks2tasks/src/handlers/githubPullRequestSynchronize.js
  149. +8 −1 services/webhooks2tasks/src/handlers/githubPush.js
  150. +10 −5 services/webhooks2tasks/src/handlers/gitlabBranchDeleted.js
  151. +6 −1 services/webhooks2tasks/src/handlers/gitlabPullRequestClosed.js
  152. +3 −0 services/webhooks2tasks/src/handlers/gitlabPullRequestOpened.js
  153. +5 −1 services/webhooks2tasks/src/handlers/gitlabPullRequestUpdated.js
  154. +9 −1 services/webhooks2tasks/src/handlers/gitlabPush.js
  155. +2 −0 services/webhooks2tasks/src/webhooks/projects.js
  156. +9 −0 tests/files/drupal8-dockerfiles/php7.3/Dockerfile.builder
  157. +2 −0 tests/files/drupal8-dockerfiles/php7.3/Dockerfile.mariadb
  158. +9 −0 tests/files/drupal8-dockerfiles/php7.3/Dockerfile.nginx
  159. +7 −0 tests/files/drupal8-dockerfiles/php7.3/Dockerfile.php
  160. +2 −0 tests/files/drupal8-dockerfiles/php7.3/Dockerfile.postgres
  161. +2 −0 tests/files/drupal8-dockerfiles/php7.3/Dockerfile.redis
  162. +3 −0 tests/files/drupal8-dockerfiles/php7.3/Dockerfile.solr
  163. +2 −0 tests/files/drupal8-dockerfiles/php7.3/Dockerfile.varnish
  164. +5 −5 tests/files/drupal8-mariadb/web/sites/default/settings.php
  165. +11 −0 tests/tests/drupal.yaml
  166. +9 −0 tests/tests/features.yaml
  167. +25 −0 tests/tests/features/openshift-limit.yaml
  168. +4 −4 tests/tests/github/pullrequest.yaml
  169. +600 −72 yarn.lock
@@ -55,7 +55,7 @@ SHELL := /bin/bash
DOCKER_BUILD_PARAMS := --quiet

# Version and Hash of the OpenShift cli that should be downloaded
MINISHIFT_VERSION := 1.15.1
MINISHIFT_VERSION := 1.16.1

MINISHIFT_CPUS := 6
MINISHIFT_MEMORY := 2GB
@@ -180,14 +180,17 @@ phpimages := php__5.6-fpm \
php__7.0-fpm \
php__7.1-fpm \
php__7.2-fpm \
php__7.3-fpm \
php__5.6-cli \
php__7.0-cli \
php__7.1-cli \
php__7.2-cli \
php__7.3-cli \
php__5.6-cli-drupal \
php__7.0-cli-drupal \
php__7.1-cli-drupal \
php__7.2-cli-drupal
php__7.2-cli-drupal \
php__7.3-cli-drupal


build-phpimages = $(foreach image,$(phpimages),build/$(image))
@@ -212,15 +215,17 @@ $(build-phpimages): build/commons
base-images += $(phpimages)
s3-images += php

build/php__5.6-fpm build/php__7.0-fpm build/php__7.1-fpm build/php__7.2-fpm: images/commons
build/php__5.6-fpm build/php__7.0-fpm build/php__7.1-fpm build/php__7.2-fpm build/php__7.3-fpm: images/commons
build/php__5.6-cli: build/php__5.6-fpm
build/php__7.0-cli: build/php__7.0-fpm
build/php__7.1-cli: build/php__7.1-fpm
build/php__7.2-cli: build/php__7.2-fpm
build/php__7.3-cli: build/php__7.3-fpm
build/php__5.6-cli-drupal: build/php__5.6-cli
build/php__7.0-cli-drupal: build/php__7.0-cli
build/php__7.1-cli-drupal: build/php__7.1-cli
build/php__7.2-cli-drupal: build/php__7.2-cli
build/php__7.3-cli-drupal: build/php__7.3-cli

#######
####### Solr Images
@@ -229,8 +234,10 @@ build/php__7.2-cli-drupal: build/php__7.2-cli

solrimages := solr__5.5 \
solr__6.6 \
solr__7.5 \
solr__5.5-drupal \
solr__6.6-drupal
solr__6.6-drupal \
solr__7.5-drupal


build-solrimages = $(foreach image,$(solrimages),build/$(image))
@@ -250,9 +257,10 @@ $(build-solrimages): build/commons
base-images += $(solrimages)
s3-images += solr

build/solr__5.5 build/solr__6.6: images/commons
build/solr__5.5 build/solr__6.6 build/solr__7.5: images/commons
build/solr__5.5-drupal: build/solr__5.5
build/solr__6.6-drupal: build/solr__6.6
build/solr__7.5-drupal: build/solr__7.5

#######
####### Node Images
@@ -458,7 +466,7 @@ $(run-rest-tests): minishift build/node__6-builder build/node__8-builder build/o
IMAGE_REPO=$(CI_BUILD_TAG) docker-compose -p $(CI_BUILD_TAG) up -d $(deployment-test-services-rest)
IMAGE_REPO=$(CI_BUILD_TAG) docker exec -i $$(docker-compose -p $(CI_BUILD_TAG) ps -q tests) ansible-playbook /ansible/tests/$(testname).yaml $(testparameter)

tests/drupal tests/drupal-postgres tests/drupal-galera: minishift build/varnish-drupal build/solr__5.5-drupal build/nginx-drupal build/redis build/php__5.6-cli-drupal build/php__7.0-cli-drupal build/php__7.1-cli-drupal build/php__7.2-cli-drupal build/api-db build/postgres-drupal build/mariadb-drupal build/oc-build-deploy-dind $(foreach image,$(deployment-test-services-rest),build/$(image)) build/drush-alias push-minishift
tests/drupal tests/drupal-postgres tests/drupal-galera: minishift build/varnish-drupal build/solr__5.5-drupal build/nginx-drupal build/redis build/php__5.6-cli-drupal build/php__7.0-cli-drupal build/php__7.1-cli-drupal build/php__7.2-cli-drupal build/php__7.3-cli-drupal build/api-db build/postgres-drupal build/mariadb-drupal build/oc-build-deploy-dind $(foreach image,$(deployment-test-services-rest),build/$(image)) build/drush-alias push-minishift
$(eval testname = $(subst tests/,,$@))
IMAGE_REPO=$(CI_BUILD_TAG) docker-compose -p $(CI_BUILD_TAG) up -d $(deployment-test-services-rest) drush-alias
IMAGE_REPO=$(CI_BUILD_TAG) docker exec -i $$(docker-compose -p $(CI_BUILD_TAG) ps -q tests) ansible-playbook /ansible/tests/$(testname).yaml $(testparameter)
@@ -613,7 +621,7 @@ openshift:
# that has been assigned to the machine is not the default one and then replace the IP in the yaml files with it
minishift: local-dev/minishift/minishift
$(info starting minishift with name $(CI_BUILD_TAG))
./local-dev/minishift/minishift --profile $(CI_BUILD_TAG) start --cpus $(MINISHIFT_CPUS) --memory $(MINISHIFT_MEMORY) --disk-size $(MINISHIFT_DISK_SIZE) --vm-driver virtualbox --openshift-version="v3.9.0"
MINISHIFT_ENABLE_EXPERIMENTAL=y ./local-dev/minishift/minishift --profile $(CI_BUILD_TAG) start --cpus $(MINISHIFT_CPUS) --memory $(MINISHIFT_MEMORY) --disk-size $(MINISHIFT_DISK_SIZE) --vm-driver virtualbox --openshift-version="v3.9.0" --extra-clusterup-flags "--service-catalog"
ifeq ($(ARCH), Darwin)
@OPENSHIFT_MACHINE_IP=$$(./local-dev/minishift/minishift --profile $(CI_BUILD_TAG) ip); \
echo "replacing IP in local-dev/api-data/01-populate-api-data.gql and docker-compose.yaml with the IP '$$OPENSHIFT_MACHINE_IP'"; \
@@ -192,6 +192,12 @@ services:
- auth-server
ports:
- '2020:2020'
# command:
# - "/usr/sbin/sshd"
# - "-e"
# - "-ddd"
# - "-f"
# - "/etc/ssh/sshd_config"
environment:
- OPENSHIFT_CONSOLE_URL=https://192.168.99.100:8443
user: '111111111'
@@ -55,7 +55,8 @@ mutation {
# This is the OpenShift ID from the response of the first query
openshift: 0
# TODO: Fill in the gitUrl field
# This is the URL to the git repository where a `.lagoon.yml` file should be committed in the root.
# This is the SSH URL to the Git repository where a `.lagoon.yml` file should be committed in the root.
# HTTPS URLs are currently not supported.
gitUrl: ""
}
) {
@@ -52,7 +52,7 @@ Environment variables are meant to do common behavior changes of php.
| Environment Variable | Default | Description |
| --------------------------------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| `PHP_MAX_EXECUTION_TIME` | `900` | Maximum execution time of each script, in seconds, [see php.net](http://php.net/max-execution-time) |
| `PHP_MAX_INPUT_VARS` | `1000` | How many input variables will be accepted, [see php.net](http://php.net/manual/en/info.configuration.php#ini.max-input-vars)
| `PHP_MAX_INPUT_VARS` | `2000` | How many input variables will be accepted, [see php.net](http://php.net/manual/en/info.configuration.php#ini.max-input-vars)
| `PHP_MAX_FILE_UPLOADS` | `20` | The maximum number of files allowed to be uploaded simultaneously, [see php.net](http://php.net/manual/en/ini.core.php#ini.max-file-uploads)|
| `PHP_MEMORY_LIMIT` | `400M` | Maximum amount of memory a script may consume, [see php.net](http://php.net/memory-limit) |
| `PHP_DISPLAY_ERRORS` | `Off` | This determines whether errors should be printed to the screen as part of the output or if they should be hidden from the user, [see php.net](http://php.net/display-errors) |
@@ -64,7 +64,8 @@
### Base URL
if (getenv('LAGOON_ROUTE')) {
$base_url = getenv('LAGOON_ROUTE');
$proto = isset($_SERVER["HTTP_X_FORWARDED_PROTO"]) ? $_SERVER["HTTP_X_FORWARDED_PROTO"] : 'http';
$base_url = $proto . "://" . $_SERVER["HTTP_HOST"];
}
### Temp directory
@@ -22,15 +22,25 @@ FROM amazeeio/solr:6.6
COPY .lagoon/solr /solr-conf/conf
CMD ["solr-precreate", "drupal", "/solr-conf"]
RUN precreate-core drupal /solr-conf
CMD ["solr-foreground"]
```

The goal is to have your solr configuration files exist at `/solr-conf/conf` in the image you are building.

## Multiple cores

To implement multiple cores, you will also need to ship your own solr schema as above, the only change needed is to the `CMD` of the Dockerfile, repeat the pattern of `precreate corename /solr-conf/ ;` for each core you require.
To implement multiple cores, you will also need to ship your own solr schema as above, the only change needed is to the `CMD` of the Dockerfile, repeat the pattern of `precreate-core corename /solr-conf/ ;` for each core you require.

```
CMD ["sh", "-c", "precreate-core drupal /solr-conf/ ; precreate-core core1 /solr-conf/ ; precreate-core core2 /solr-conf/ ; precreate-core core3 /solr-conf/ ; solr start -f"]
FROM amazeeio/solr:6.6-drupal
RUN precreate-core drupal-index1 /solr-conf && \
precreate-core drupal-index2 /solr-conf && \
precreate-core drupal-index3 /solr-conf
CMD ["solr-foreground"]
```
@@ -13,5 +13,5 @@ There are a couple of things that will use this information:

- If your Lagoon Administrator has idling enabled, non-production environments will automatically be idled after four hours of non usage (no worry, they are started automatically when you visit them).
- Our default Drupal settings.php files load additional settings files for `development.settings.php` and `production.settings.php` so you can define settings and configurations different per environment type.
- If you try to delete an environment that is defined as the production environment (either via Webhooks or REST) Lagoon will politely refuse to delete the production environment, as it tries to prevent you from doing a mistake. In order to delete a production environment you can either change the `productionEnvironment` in the API or use the secret `forceDeleteProductionEnvironment: true` POST payload for the REST api.
- The Lagoon Administrator might use the production environment information for some additional things. Like at amazee.io we're calculating only the hits of the production environments to calculate the price of the hosting.
- If you try to delete an environment that is defined as the production environment (either via Webhooks or REST) Lagoon will politely refuse to delete the production environment, as it tries to prevent you from making a mistake. In order to delete a production environment you can either change the `productionEnvironment` in the API or use the secret `forceDeleteProductionEnvironment: true` POST payload for the REST api.
- The Lagoon Administrator might use the production environment information for some additional things. Like at amazee.io we're calculating only the hits of the production environments to calculate the price of the hosting.
@@ -40,6 +40,9 @@ routes:
environments:
master:
monitoring_urls:
- "www.example.com"
- "www.example.com/special_page"
routes:
- nginx:
- example.com
@@ -190,6 +193,9 @@ environments:
mariadb: statefulset
```

#### `environments.[name].monitoring_urls`
At the end of a deplpoy, Lagoon will check this field for any URLs which you specify to add to the API for the purpose of monitoring. The default value for this field is the first route for a project. It is useful for adding specific paths of a project to the API, for consumption by a monitoring service. Please note, Lagoon does not provide any direct integration to a monitoring service, this just adds the URLs to the API. On amazee.io, we take the monitoring_urls and add them to our Statuscake account.

## Polysite

In Lagoon, the same git repository can be added to multiple projects, creating what is called a Polysite. This allows you to run the same codebase but allow for different, isolated, databases and persistent files. In `.lagoon.yml` we currently only support specifying custom routes for a polysite project. The key difference from a standard project is that the `environments` becomes the second-level element, and the project name the top level.
@@ -9,7 +9,9 @@ This table lists all service types that can be defined via `lagoon.type` within
| `nginx` | Nginx container. No persistent storage | `localhost:50000/nginx_status` | `8080` | yes | - |
| `nginx-php` | Like `nginx`, but additionally a `php` container. | nginx: `localhost:50000/nginx_status`, php: `/usr/sbin/check_fcgi` | `8080` | yes | - |
| `nginx-php-persistent` | Like `nginx-php`, will generate persistent storage, define mount location via `lagoon.persistent` | nginx: `localhost:50000/nginx_status`, php: `/usr/sbin/check_fcgi` | http on `8080` | yes | `lagoon.persistent`, `lagoon.persistent.name`, `lagoon.persistent.size`, `lagoon.persistent.class` |
| `mariadb` | MariaDB container. Creates cronjob for Backups running every 24h executing `/lagoon/mysql-backup.sh 127.0.0.1` | TCP connection on `3306` | `3306` | - | `lagoon.persistent.size` |
| `mariadb` | A meta service which will tell lagoon to automatically decide between `mariadb-single` and `mariadb-shared` | - | - | - | - |
| `mariadb-single` | MariaDB container. Creates cronjob for Backups running every 24h executing `/lagoon/mysql-backup.sh 127.0.0.1` | TCP connection on `3306` | `3306` | - | `lagoon.persistent.size` |
| `mariadb-shared` | Uses a shared mariadb server via a mariadb service broker | not needed | `3306` | - | - |
| `mariadb-galera` | MariaDB Galera Cluster with 3 nodes, uses Statefulset, generates persistent storage for each Cluster node. Creates cronjob for Backups running every 24h executing `/lagoon/mysql-backup.sh 127.0.0.1`. Starts additional maxscale container where the service points to (no direct connection to galera nodes) | TCP connection on `3306` | `3306` | - | `lagoon.persistent.size` |
| `postgres` | Postgres container. Creates cronjob for Backups running every 24h executing `/lagoon/postgres-backup.sh localhost` | TCP connection on `5432` | `5432` | - | `lagoon.persistent.size` |
| `redis` | Redis container. | TCP connection on `6379` | `6379` | - | - |
@@ -7,4 +7,4 @@
oc get pvc --all-namespaces | grep solr | sed '1d' | awk '{ print $2, "--namespace", $1 }' | while read line; do oc annotate --overwrite pvc $line appuio.ch/backup="true"; done
oc get pvc --all-namespaces | grep nginx | sed '1d' | awk '{ print $2, "--namespace", $1 }' | while read line; do oc annotate --overwrite pvc $line appuio.ch/backup="true"; done

oc get --all-namespaces pod -l 'service in (cli)' | sed '1d' | awk '{ print "--namespace", $1, $2 }' | while read line; do oc annotate --overwrite pod $line appuio.ch/backupcommand='/bin/bash -c "if [[ $MARIADB_HOST ]]; then mysqldump --events --routines --quick --add-locks --no-autocommit --single-transaction --no-create-db -h $MARIADB_HOST -u $MARIADB_USERNAME -p$MARIADB_PASSWORD $MARIADB_DATABASE; fi"'; done
oc get --all-namespaces pod -l 'service in (cli)' | sed '1d' | awk '{ print "--namespace", $1, $2 }' | while read line; do oc annotate --overwrite pod $line appuio.ch/backupcommand='/bin/bash -c "if [[ $MARIADB_HOST ]]; then mysqldump --max-allowed-packet=500M --events --routines --quick --add-locks --no-autocommit --single-transaction --no-create-db -h $MARIADB_HOST -u $MARIADB_USERNAME -p$MARIADB_PASSWORD $MARIADB_DATABASE; fi"'; done
@@ -0,0 +1,100 @@
#!/bin/bash

set -euo pipefail

which shyaml > /dev/null
if [ $? -gt 0 ]; then
echo "please install shyaml (pip3 install shyaml)"
exit 1
fi

which jq > /dev/null
if [ $? -gt 0 ]; then
echo "please install jq"
exit 1
fi

PROJECT_NAME=$(oc project -q)

SERVICE_NAME=mariadb
SERVICE_NAME_UPPERCASE=$(echo $SERVICE_NAME | tr [:lower:] [:upper:])
SERVICE_TYPE=mariadb-shared

ENVIRONMENT_TYPE=$(oc get configmap lagoon-env -o json | jq -r '.data.LAGOON_ENVIRONMENT_TYPE')

# export old mariadb pod name
OLD_POD=$(oc get pod -o custom-columns=NAME:.metadata.name --no-headers -l service=$SERVICE_NAME)
echo "found $SERVICE_NAME pod $OLD_POD"

echo "*** Pausing nginx and cli"
oc scale dc/nginx --replicas=0
oc scale dc/cli --replicas=0


# create service broker
## taken from build-deploy-docker-compose.sh

OPENSHIFT_TEMPLATE="$(git rev-parse --show-toplevel)/images/oc-build-deploy-dind/openshift-templates/${SERVICE_TYPE}/servicebroker.yml"

# Only SERVICE_NAME is used in the template, fudge the other parameters.
oc process --local -o yaml --insecure-skip-tls-verify \
-f ${OPENSHIFT_TEMPLATE} \
-p SERVICE_NAME="${SERVICE_NAME}" \
-p SERVICEBROKER_NAME="lagoon-dbaas-mariadb-apb" \
-p SERVICEBROKER_PLAN_NAME="${ENVIRONMENT_TYPE}" \
-p SAFE_BRANCH="NULL" \
-p SAFE_PROJECT="NULL" \
-p BRANCH="NULL" \
-p PROJECT="NULL" \
-p LAGOON_GIT_SHA="NULL" \
-p SERVICE_ROUTER_URL="NULL" \
-p REGISTRY="NULL" \
-p OPENSHIFT_PROJECT="NULL" \
| oc apply -f -

# ServiceBrokers take a bit, wait until the credentials secret is available
until oc get --insecure-skip-tls-verify secret ${SERVICE_NAME}-servicebroker-credentials
do
echo "Secret ${SERVICE_NAME}-servicebroker-credentials not available yet, waiting for 10 secs"
sleep 10
done

# Load credentials out of secret
SECRETS=/tmp/${PROJECT_NAME}-${OLD_POD}-migration.yaml
oc get --insecure-skip-tls-verify secret ${SERVICE_NAME}-servicebroker-credentials -o yaml > $SECRETS

DB_HOST=$(cat $SECRETS | shyaml get-value data.DB_HOST | base64 -D)
DB_USER=$(cat $SECRETS | shyaml get-value data.DB_USER | base64 -D)
DB_PASSWORD=$(cat $SECRETS | shyaml get-value data.DB_PASSWORD | base64 -D)
DB_NAME=$(cat $SECRETS | shyaml get-value data.DB_NAME | base64 -D)
DB_PORT=$(cat $SECRETS | shyaml get-value data.DB_PORT | base64 -D)

echo "*** Transfering 'drupal' database from $OLD_POD to $DB_HOST"
# transfer database between from old to new
oc exec $OLD_POD -- bash -c "mysqldump --max-allowed-packet=500M --events --routines --quick --add-locks --no-autocommit --single-transaction --no-create-db drupal | mysql -h $DB_HOST -u $DB_USER -p${DB_PASSWORD} -P $DB_PORT $DB_NAME"

CONFIG_BAK="/tmp/${PROJECT_NAME}-$(date +%F-%T)-lagoon-env.yaml"
echo "*** Backing up configmap in case we need to revert: ${CONFIG_BAK}"
oc get configmap lagoon-env -o yaml > $CONFIG_BAK

echo "*** updating configmap to point to ${DB_HOST}."
# Add credentials to our configmap, prefixed with the name of the servicename of this servicebroker
oc patch --insecure-skip-tls-verify configmap lagoon-env \
-p "{\"data\":{\"${SERVICE_NAME_UPPERCASE}_HOST\":\"${DB_HOST}\", \"${SERVICE_NAME_UPPERCASE}_USERNAME\":\"${DB_USER}\", \"${SERVICE_NAME_UPPERCASE}_PASSWORD\":\"${DB_PASSWORD}\", \"${SERVICE_NAME_UPPERCASE}_DATABASE\":\"${DB_NAME}\", \"${SERVICE_NAME_UPPERCASE}_PORT\":\"${DB_PORT}\"}}"


echo "*** Deleting mariadb service. Scaling old mariadb to 0; you can clean up the DC and pv later"
oc delete service mariadb
oc scale dc/mariadb --replicas=0

# transfer complete, clean up
rm -f $SECRETS

oc scale dc/nginx --replicas=1
oc scale dc/cli --replicas=1

oc rollout latest dc/nginx
oc rollout latest dc/cli
oc rollout status dc/nginx
oc rollout status dc/cli
echo "*** done."
Oops, something went wrong.

0 comments on commit 7d1d265

Please sign in to comment.