Permalink
Browse files

Merge pull request #493 from amazeeio/release/0.15

Release/0.15
  • Loading branch information...
Schnitzel committed Jul 11, 2018
2 parents 0f6403f + b40b66c commit 6d3deb0e94e8d5f28ebd1ffceb34a118ab30dd1f
Showing with 1,573 additions and 420 deletions.
  1. +24 −0 .github/ISSUE_TEMPLATE/bug_report.md
  2. +15 −4 Makefile
  3. +11 −4 docker-compose.yaml
  4. +3 −1 docs/administering_lagoon/create-project.gql
  5. +181 −8 docs/administering_lagoon/graphql_api.md
  6. +3 −3 docs/administering_lagoon/install.md
  7. +0 −35 docs/create-project.gql
  8. +1 −1 docs/developing_lagoon/contributing.md
  9. +1 −1 docs/index.md
  10. +70 −11 docs/using_lagoon/lagoon_yml.md
  11. +1 −0 images/mariadb/maxscale.sql
  12. +2 −2 images/mariadb/my.cnf
  13. +24 −14 images/oc-build-deploy-dind/build-deploy-docker-compose.sh
  14. +4 −0 images/oc-build-deploy-dind/build-deploy.sh
  15. +6 −0 images/oc-build-deploy-dind/openshift-templates/elasticsearch/deployment.yml
  16. +9 −1 images/oc-build-deploy-dind/openshift-templates/elasticsearch/pvc.yml
  17. +8 −0 images/oc-build-deploy-dind/openshift-templates/mariadb/deployment.yml
  18. +8 −1 images/oc-build-deploy-dind/openshift-templates/mariadb/pvc.yml
  19. +7 −4 images/oc-build-deploy-dind/scripts/convert-crontab.sh
  20. +22 −0 images/oc-build-deploy-dind/scripts/exec-monitor-daemonset.sh
  21. +17 −15 images/varnish-drupal/drupal.vcl
  22. +5 −4 images/varnish/Dockerfile
  23. +0 −2 images/varnish/docker-entrypoint
  24. +1 −1 images/varnish/varnish-start.sh
  25. +24 −0 local-dev/api-data/api-data.gql
  26. +2 −1 local-dev/git/Dockerfile
  27. +3 −0 node-packages/commons/src/api.js
  28. +19 −0 openshift-setup/clusterrole-daemonset-admin.yaml
  29. +0 −98 openshift-setup/docker-host-minishift.yaml
  30. +42 −0 services/api-db/docker-entrypoint-initdb.d/00-setup.sql
  31. +6 −0 services/api-db/docker-entrypoint-initdb.d/01-procedures.sql
  32. +1 −0 services/api/src/dao/openshift.js
  33. +1 −0 services/api/src/dao/project.js
  34. +6 −0 services/api/src/schema.js
  35. +5 −0 services/auto-idler/.lagoon.yml
  36. +11 −5 services/auto-idler/idle-services.sh
  37. +51 −19 {openshift-setup → services/docker-host}/docker-host.yaml
  38. +1 −1 services/drush-alias/web/aliases.drushrc.php.stub
  39. +217 −76 services/logs-collector/.lagoon.yml
  40. +0 −12 services/logs-collector/deploy.sh
  41. +0 −9 services/logs-collector/fluent.conf
  42. 0 services/logs-collector/plugins/.gitignore
  43. +176 −0 services/logs-forwarder/.lagoon.yml
  44. +6 −3 services/{logs-collector → logs-forwarder}/Dockerfile
  45. +19 −0 services/logs-forwarder/ca_cert.pem
  46. +30 −0 services/logs-forwarder/ca_key.pem
  47. 0 services/{logs-collector → logs-forwarder}/entrypoint.sh
  48. +21 −5 services/logs2logs-db/.lagoon.yml
  49. +1 −0 services/logs2logs-db/Dockerfile
  50. +17 −0 services/logs2logs-db/certs/lumberjack.cert
  51. +28 −0 services/logs2logs-db/certs/lumberjack.key
  52. +6 −4 services/logs2logs-db/pipeline/router-logs.conf
  53. +6 −1 services/openshiftbuilddeploy/src/index.js
  54. +1 −0 tests/files/features-subfolder/subfolder1/subfolder2/.dockerignore
  55. +1 −0 tests/files/features-subfolder/subfolder1/subfolder2/.lagoon.yml
  56. +18 −0 tests/files/features-subfolder/subfolder1/subfolder2/Dockerfile
  57. +23 −0 tests/files/features-subfolder/subfolder1/subfolder2/docker-compose.yml
  58. +24 −0 tests/files/features-subfolder/subfolder1/subfolder2/index.js
  59. +12 −0 tests/files/features-subfolder/subfolder1/subfolder2/package.json
  60. +247 −0 tests/files/features-subfolder/subfolder1/subfolder2/yarn.lock
  61. +82 −74 tests/tests/features.yaml
  62. +43 −0 tests/tests/features/subfolder.yaml
@@ -0,0 +1,24 @@
---
name: Bug report
about: Create a report to help us improve
---
**Describe the bug**
A clear and concise description of what the bug is.
**To Reproduce**
Steps to reproduce the behavior:
1. Go to '...'
2. Click on '....'
3. Scroll down to '....'
4. See error
**Expected behavior**
A clear and concise description of what you expected to happen.
**Screenshots**
If applicable, add screenshots to help explain your problem.
**Additional context**
Add any other context about the problem here.
View
@@ -308,7 +308,7 @@ services := api \
webhooks2tasks \
hacky-rest2tasks-ui \
rabbitmq \
logs-collector \
logs-forwarder \
logs-db \
logs-db-ui \
logs2logs-db \
@@ -620,18 +620,28 @@ minishift/login-docker-registry:
openshift-lagoon-setup:
# Only use the minishift provided oc if we don't have one yet (allows system engineers to use their own oc)
if ! which oc; then eval $$(./local-dev/minishift/minishift --profile $(CI_BUILD_TAG) oc-env); fi; \
oc -n default set env dc/router -e ROUTER_LOG_LEVEL=info -e ROUTER_SYSLOG_ADDRESS=192.168.99.1:5140; \
oc -n default set env dc/router -e ROUTER_LOG_LEVEL=info -e ROUTER_SYSLOG_ADDRESS=router-logs.lagoon.svc:5140; \
oc new-project lagoon; \
oc adm pod-network make-projects-global lagoon; \
oc -n lagoon create serviceaccount openshiftbuilddeploy; \
oc -n lagoon policy add-role-to-user admin -z openshiftbuilddeploy; \
oc -n lagoon create -f openshift-setup/clusterrole-openshiftbuilddeploy.yaml; \
oc -n lagoon adm policy add-cluster-role-to-user openshiftbuilddeploy -z openshiftbuilddeploy; \
oc -n lagoon create -f openshift-setup/shared-resource-viewer.yaml; \
oc -n lagoon create -f openshift-setup/policybinding.yaml | oc -n lagoon create -f openshift-setup/rolebinding.yaml; \
oc -n lagoon create serviceaccount docker-host; \
oc -n lagoon adm policy add-scc-to-user privileged -z docker-host; \
oc -n lagoon policy add-role-to-user edit -z docker-host; \
bash -c "oc process -n lagoon -f openshift-setup/docker-host.yaml | oc -n lagoon apply -f -"; \
oc -n lagoon create serviceaccount logs-collector; \
oc -n lagoon adm policy add-cluster-role-to-user cluster-reader -z logs-collector; \
oc -n lagoon adm policy add-scc-to-user hostaccess -z logs-collector; \
oc -n lagoon adm policy add-scc-to-user privileged -z logs-collector; \
oc -n lagoon adm policy add-cluster-role-to-user daemonset-admin -z lagoon-deployer; \
oc -n lagoon create serviceaccount lagoon-deployer; \
oc -n lagoon policy add-role-to-user edit -z openshiftbuilddeploy; \
oc -n lagoon create -f openshift-setup/clusterrole-daemonset-admin.yaml; \
oc -n lagoon adm policy add-cluster-role-to-user daemonset-admin -z lagoon-deployer; \
bash -c "oc process -n lagoon -f services/docker-host/docker-host.yaml | oc -n lagoon apply -f -"; \
echo -e "\n\nAll Setup, use this token as described in the Lagoon Install Documentation:" \
oc -n lagoon serviceaccounts get-token openshiftbuilddeploy
@@ -641,7 +651,8 @@ openshift-lagoon-setup:
.PHONY: openshift/configure-lagoon-local
minishift/configure-lagoon-local: openshift-lagoon-setup
eval $$(./local-dev/minishift/minishift --profile $(CI_BUILD_TAG) oc-env); \
bash -c "oc process -n lagoon -p IMAGE=docker-registry.default.svc:5000/lagoon/docker-host:latest -p REPOSITORY_TO_UPDATE=lagoon -f openshift-setup/docker-host-minishift.yaml | oc -n lagoon apply -f -";
bash -c "oc process -n lagoon -p SERVICE_IMAGE=172.30.1.1:5000/lagoon/docker-host:latest -p REPOSITORY_TO_UPDATE=lagoon -f services/docker-host/docker-host.yaml | oc -n lagoon apply -f -"; \
oc -n default set env dc/router -e ROUTER_LOG_LEVEL=info -e ROUTER_SYSLOG_ADDRESS=192.168.99.1:5140; \
# Stop OpenShift Cluster
.PHONY: minishift/stop
View
@@ -260,13 +260,13 @@ services:
lagoon.type: elasticsearch
lagoon.template: services/logs-db/.lagoon.yml
lagoon.image: amazeeiolagoon/logs-db:${SAFE_BRANCH:-master}
logs-collector:
image: ${IMAGE_REPO:-lagoon}/logs-collector
logs-forwarder:
image: ${IMAGE_REPO:-lagoon}/logs-forwarder
user: '111111111'
labels:
lagoon.type: custom
lagoon.template: services/logs-collector/.lagoon.yml
lagoon.image: amazeeiolagoon/logs-collector:${SAFE_BRANCH:-master}
lagoon.template: services/logs-forwarder/.lagoon.yml
lagoon.image: amazeeiolagoon/logs-forwarder:${SAFE_BRANCH:-master}
logs-db-ui:
image: ${IMAGE_REPO:-lagoon}/logs-db-ui
user: '111111111'
@@ -281,6 +281,7 @@ services:
user: '111111111'
ports:
- "5140:5140/udp"
- "5044:5044"
labels:
lagoon.type: logstash
lagoon.template: services/logs2logs-db/.lagoon.yml
@@ -305,3 +306,9 @@ services:
lagoon.type: custom
lagoon.template: services/storage-calculator/.lagoon.yml
lagoon.image: amazeeiolagoon/storage-calculator:${SAFE_BRANCH:-master}
logs-collector:
image: openshift/origin-logging-fluentd:v3.6.1
labels:
lagoon.type: custom
lagoon.template: services/logs-collector/.lagoon.yml
lagoon.rollout: daemonset
@@ -5,6 +5,7 @@ mutation {
# see an example in /local-dev/api-data/api-data.sql)
addCustomer(input: {name: "customer-name", private_key: "[fill me]"}) {
name
id
}
# The OpenShift Cluster that Lagoon should use to deploy to. Yes Lagoon is not only capable to deploy into the OpenShift that
@@ -14,10 +15,11 @@ mutation {
# `token` - the token of the `lagoon` Service Account creted in this OpenShift (this is the same token that we also used during installation of Lagoon)
addOpenshift(input: {name: "my-openshift", console_url:"[fill me]", token: "[fill me]"}) {
name
id
}
# This is your git repository that should be deployed, it needs to contain a `.lagoon.yml` file so Lagoon knows what it should do.
addProject(input:{name: "first-project", customer:"customer-name", openshift: "my-openshift", git_url: "[fill me]"}) {
addProject(input:{name: "first-project", customer:[customer-id], openshift:[openshift-id], git_url: "[fill me]"}) {
name
customer {
name
@@ -35,17 +35,106 @@ And press the Play button (or press CTRL+ENTER). If all went well, you should se
In order for Lagoon to deploy a project there is an example graphql in `create-project.gql`, which will create three API Objects:
1. `project` This is your git repository that should be deployed, it needs to contain a `.lagoon.yml` file so Lagoon knows what it should do.
1. `customer` The customer of the project. Can be used for an actual customer (if you use Lagoon in a multi-customer setup), or just to group multiple projects together. `customer` will hold the SSH Private Key that Lagoon will use to clone the Git repository of the project (the private key needs to be in a single string, where new lines are replaced by `\n` - see an example in /local-dev/api-data/api-data.sql)
2. `openshift` The OpenShift Cluster that Lagoon should use to deploy to. Yes Lagoon is not only capable to deploy into the OpenShift that it is running itself, but actually to any OpenShift anywhere in the world. We need to know the following infos for this to work:
1. `name` - Unique identifier of the OpenShift
2. `console_url` - URL of the OpenShift console (without any `/console` suffix)
3. `token` - the token of the `lagoon` Service Account created in this OpenShift (this is the same token that we also used during installation of Lagoon)
3. `customer` The customer of the project. Can be used for an actual customer (if you use Lagoon in a multi-customer setup), or just to group multiple projects together. `customer` will hold the SSH Private Key that Lagoon will use to clone the Git repository of the project (the private key needs to be in a single string, where new lines are replaced by `\n` - see an example in /local-dev/api-data/api-data.sql)
3. `project` This is your git repository that should be deployed, it needs to contain a `.lagoon.yml` file so Lagoon knows what it should do.
Just fill all the `[fill me]` you can find in the examples below, copy it into the GraphiQL Client, press play and if everything went well, you should get a response which shows you the name of the customer & openshift object and the full project object that just has been created.
Congrats again 🎉!
#### Give Access to the Project
In Lagoon the individual developers are authenticating themselves via their SSH Keys. Via their SSH Keys they have access to multiple things:
1. The Lagoon API itself, where they can only see and edit projects they actually have access too
2. Remote Shell Access to containers that are running in projects they have access too
3. The Lagoon logging system, where a developer can find Request Logs, Container Logs, Lagoon Logs and many more.
First we need to add a new SSH Public key to the API:
```
mutation addSSHKey {
addSshKey(input:{name:"[name]", keyValue:"[keyValue]", keyType:SSH_RSA}) {
id
}
}
```
- `name` - Your identificator for this SSH Key, can by any string
- `keyValue` - The actual SSH Public Key Value (withouth the type on front and no name at the end, so just something like `AAAAB3NzaC1yc2EAAAADAQ...3QjzIOtdQERGZuMsi0p`)
- `keyType` - The type of the key, there are currently two types supported by Lagoon: `SSH_RSA` and `SSH_ED25519`
After we added the key we can give this key access to either a single project or a whole customer, while access to a whole customer means that this SSH key has automatically access to all projects that are assigned to this customer.
```
mutation addSshKeyToCustomer {
addSshKeyToCustomer(input:{customer:"[customer-name]", sshKey:"[sshKey-name]"}) {
id
}
}
```
or
```
mutation addSshKeyToProject {
addSshKeyToProject(input:{project:"[project-name]", sshKey:"[sshKey-name]"}) {
id
}
}
```
That's it, now this SSH key can create Tokens via SSH, access containers and more.
Of corse it is possible to add an SSH Key to multiple customers and projects, whatever you need.
#### Add Notifications to the Project
If you like to know what exactly is going on during a deployment, we suggest to configure notifications for your project, they will provide:
- Push messages
- Build start information
- Build success or failure messages
- Many more
Like with the SSH Keys, we first add the Notification and then we connect the Notification to the Projects. As the Notifications can be quite different of their information they need, the notification types are built a bit more sofisticated and each Notification Type has it's own mutation:
```
mutation addNotificationSlack {
addNotificationSlack(input:{name:"[name]]", channel:"[channel]", webhook:"[webhook]"}) {
id
}
}
```
```
mutation addNotificationRocketChat {
addNotificationSlack(input:{name:"[name]]", channel:"[channel]", webhook:"[webhook]"}) {
id
}
}
```
- `name` - Is your own identificator for this Notification
- `channel` - Which channel should the message be sent to
- `webhook` - The URL of the webhook where messages should be sent, this is usally provided by the Chat System to you.
After we create that we can now connect this notification to our project:
```
mutation addNotificationToProject {
addNotificationToProject(input:{notificationType: SLACK, project:"[project-name]", notificationName:"[notification-name]"}) {
id
}
}
```
Now for every deployment you should see messages appear in your defined channel.
## Example GraphQL queries
### Add New OpenShift Target
@@ -60,6 +149,7 @@ The OpenShift Cluster that Lagoon should use to deploy to. Yes, Lagoon is not on
mutation {
addOpenshift(input: {name: "my-openshift", console_url:"[fill me]", token: "[fill me]"}) {
name
id
}
}
```
@@ -72,6 +162,7 @@ The customer of the project. Can be used for an actual customer (if you use Lago
mutation {
addCustomer(input: {name: "[fill me]", private_key: "[fill me]"}) {
name
id
}
}
```
@@ -82,13 +173,15 @@ This is your git repository that should be deployed, it needs to contain a `.lag
```
mutation {
addProject(input:{name: "first-project", customer:"customer-name", openshift: "my-openshift", git_url: "[fill me]"}) {
addProject(input:{name: "first-project", customer:[customer-id], openshift:[openshift-id], git_url: "[fill me]"}) {
name
customer {
name
id
}
openshift {
name
id
}
git_url,
active_systems_deploy,
@@ -101,14 +194,13 @@ mutation {
### List Projects and Customers
This is a good comand to see an overview of all Projects, OpenShifts and Customers that exist within our Lagoon.
```
query whatIsThereAlready{
allProjects {
name
git_url
notifications {
...slack
}
}
allOpenshifts {
name
@@ -119,8 +211,89 @@ query whatIsThereAlready{
id
}
}
```
### Single Project
If you want to get an in depth look into a single project, this querry has been proven quite good:
```
query singleProject {
projectByName(name: "[projectname]") {
id
branches
git_url
pullrequests
production_environment
notifications(type: SLACK) {
... on NotificationSlack {
name
channel
webhook
id
}
}
environments {
name
deploy_type
environment_type
}
openshift {
id
}
customer {
id
name
sshKeys {
id
name
}
}
}
}
```
### Project by Git URL
Don't remember how a project was called, but now the Git URL? Search no longer, there is an GraphQL Query for that:
```
query projectByGitUrl{
projectByGitUrl(gitUrl: "git@server.com:org/repo.git") {
name
}
}
```
### Update Objects
The Lagoon GraphQL API cannot only display Objects and create Objects, it also has the capability to update exisitng Objects, all of this happens in full GraphQL best practices manner.
Update the branches to deploy within a project:
```
mutation editProjectBranches {
updateProject(input:{id:109, patch:{branches:"^(prod|stage|dev|update)$"}}) {
id
}
}
```
fragment slack on NotificationSlack {
name
Update the production Environment within a project (Important: Needs a redeploy in order for all changes to be reflected in the containers):
```
mutation editProjectProductionEnvironment {
updateProject(input:{id:109, patch:{production_environment:"master"}}) {
id
}
}
```
You can also combine multiple changes at once:
```
mutation editProjectProductionEnvironmentAndBranches {
updateProject(input:{id:109, patch:{production_environment:"master", branches:"^(prod|stage|dev|update)$"}}) {
id
}
}
```
@@ -35,9 +35,9 @@ In this example we create the Service Account `lagoon` in the OpenShift Project
In order to use a local Lagoon to deploy itself on an OpenShift, we need a subset of Lagoon running locally. We need to tech this local Lagoon how to connect to the OpenShift:
1. Edit `lagoon` inside local-dev/api-data/api-data.sql, in the `INSERT INTO openshift` section:
1. `[replace me with OpenShift console URL]` - The URL to the OpenShift Console, without `console` at the end.
2. `[replace me with OpenShift Token]` - The token of the lagoon service account that was shown to you during `make openshift-lagoon-setup`
1. Edit `lagoon` inside local-dev/api-data/api-data.sql, in the `Lagoon Kickstart Objects` section:
1. `[REPLACE ME WITH OPENSHIFT URL]` - The URL to the OpenShift Console, without `console` at the end.
2. `[REPLACE ME WITH OPENSHIFT LAGOON SERVICEACCOUTN TOKEN]` - The token of the lagoon service account that was shown to you during `make openshift-lagoon-setup`
2. Build required Images and start services:
Oops, something went wrong.

0 comments on commit 6d3deb0

Please sign in to comment.