Permalink
Browse files

Merge remote-tracking branch 'origin/289-elasticsearch-cluster-rebase'

  • Loading branch information...
Schnitzel committed Apr 19, 2018
2 parents e76702d + ea2d53a commit 1a72498fe3c54df216e8a7d12c9e2a307044000a
Showing with 408 additions and 18 deletions.
  1. +5 −0 .lagoon.yml
  2. +4 −2 Makefile
  3. +5 −0 images/elasticsearch/Dockerfile
  4. +44 −0 images/elasticsearch/configure-es.sh
  5. +18 −0 images/elasticsearch/docker-entrypoint.sh
  6. BIN images/elasticsearch/peer-finder
  7. +0 −1 images/mariadb/docker-entrypoint.bash
  8. +5 −0 images/oc-build-deploy-dind/build-deploy-docker-compose.sh
  9. +58 −0 images/oc-build-deploy-dind/openshift-templates/elasticsearch-cluster/services.yml
  10. +92 −0 images/oc-build-deploy-dind/openshift-templates/elasticsearch-cluster/statefulset.yml
  11. +1 −1 images/oc-build-deploy-dind/openshift-templates/elasticsearch/deployment.yml
  12. +10 −3 local-dev/api-data/api-data.gql
  13. +3 −2 local-dev/git/Dockerfile
  14. +1 −1 services/logs-db/Dockerfile
  15. +5 −8 services/logs-db/{conf/elasticsearch.yml → docker-entrypoint.sh}
  16. +11 −0 tests/checks/check-json-content.yaml
  17. +1 −0 tests/files/elasticsearch-cluster/.lagoon.yml
  18. +2 −0 tests/files/elasticsearch-cluster/Dockerfile.elasticsearch
  19. +4 −0 tests/files/elasticsearch-cluster/Dockerfile.nginx
  20. +10 −0 tests/files/elasticsearch-cluster/app.conf
  21. +23 −0 tests/files/elasticsearch-cluster/docker-compose.yml
  22. +1 −0 tests/files/elasticsearch/.lagoon.yml
  23. +2 −0 tests/files/elasticsearch/Dockerfile.elasticsearch
  24. +4 −0 tests/files/elasticsearch/Dockerfile.nginx
  25. +10 −0 tests/files/elasticsearch/app.conf
  26. +23 −0 tests/files/elasticsearch/docker-compose.yml
  27. +18 −0 tests/tests/elasticsearch.yaml
  28. +48 −0 tests/tests/elasticsearch/elasticsearch.yaml
View
@@ -15,3 +15,8 @@ tasks:
name: 'check\set elasticsearch watchers'
command: cd /lagoon/kibana-init/; sh watchers.sh
service: logs-db-ui
environments:
develop:
types:
logs-db: elasticsearch-cluster
View
@@ -395,7 +395,8 @@ all-tests-list:= features \
gitlab \
bitbucket \
rest \
nginx
nginx \
elasticsearch
all-tests = $(foreach image,$(all-tests-list),tests/$(image))
# Run all tests
@@ -414,7 +415,7 @@ tests-list:
deployment-test-services-main = rabbitmq openshiftremove openshiftbuilddeploy openshiftbuilddeploymonitor logs2slack api ssh auth-server local-git local-api-data-watcher-pusher local-es-kibana-watcher-pusher tests
# All Tests that use REST endpoints
rest-tests = rest node features nginx
rest-tests = rest node features nginx elasticsearch
run-rest-tests = $(foreach image,$(rest-tests),tests/$(image))
# List of Lagoon Services needed for REST endpoint testing
deployment-test-services-rest = $(deployment-test-services-main) rest2tasks
@@ -589,6 +590,7 @@ else
echo "replacing IP in local-dev/api-data/api-data.gql and docker-compose.yaml with the IP '$$OPENSHIFT_MACHINE_IP'"; \
sed -i "s/192.168\.[0-9]\{1,3\}\.[0-9]\{3\}/$${OPENSHIFT_MACHINE_IP}/g" local-dev/api-data/api-data.gql docker-compose.yaml;
endif
./local-dev/minishift/minishift ssh -- '/bin/sh -c "sudo sysctl -w vm.max_map_count=262144"'
eval $$(./local-dev/minishift/minishift --profile $(CI_BUILD_TAG) oc-env); \
oc login -u system:admin; \
bash -c "echo '{\"apiVersion\":\"v1\",\"kind\":\"Service\",\"metadata\":{\"name\":\"docker-registry-external\"},\"spec\":{\"ports\":[{\"port\":5000,\"protocol\":\"TCP\",\"targetPort\":5000,\"nodePort\":30000}],\"selector\":{\"docker-registry\":\"default\"},\"sessionAffinity\":\"None\",\"type\":\"NodePort\"}}' | oc --context="default/$$(./local-dev/minishift/minishift --profile $(CI_BUILD_TAG) ip | sed 's/\./-/g'):8443/system:admin" create -n default -f -"; \
@@ -9,6 +9,9 @@ ENV LAGOON=elasticsearch
COPY --from=commons /lagoon /lagoon
COPY --from=commons /bin/fix-permissions /bin/ep /bin/docker-sleep /bin/
COPY --from=commons /home /home
COPY peer-finder /bin/
COPY configure-es.sh /lagoon/
COPY docker-entrypoint.sh /lagoon/entrypoints/90-elasticsearch.sh
RUN chmod g+w /etc/passwd \
&& mkdir -p /home
@@ -23,3 +26,5 @@ ENV TMPDIR=/tmp TMP=/tmp HOME=/home ENV=/home/.bashrc BASH_ENV=/home/.bashrc
RUN echo "xpack.security.enabled: false" >> config/elasticsearch.yml
ENV ES_JAVA_OPTS "-Xms200m -Xmx200m"
ENTRYPOINT ["/lagoon/entrypoints.bash", "/usr/local/bin/docker-entrypoint.sh"]
@@ -0,0 +1,44 @@
#! /bin/bash
# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
CFG=/usr/share/elasticsearch/config/elasticsearch.yml
function join {
local IFS="$1"; shift; echo "$*";
}
HOSTNAME=$(hostname)
# Parse out cluster name, from service name:
CLUSTER_NAME="$(hostname -f | cut -d'.' -f2)"
while read -ra LINE; do
if [[ "${LINE}" == *"${HOSTNAME}"* ]]; then
MY_NAME=$LINE
fi
PEERS=("${PEERS[@]}" $LINE)
done
if [ "${#PEERS[@]}" = 1 ]; then
ES_CLUSTER_ADDRESS="$MY_NAME"
else
ES_CLUSTER_ADDRESS=$(join , "${PEERS[@]}")
fi
echo "discovery.zen.ping.unicast.hosts: ${ES_CLUSTER_ADDRESS}" >> ${CFG}
# don't need a restart, we're just writing the conf in case there's an
# unexpected restart on the node.
@@ -0,0 +1,18 @@
#!/usr/bin/env bash
set -eo pipefail
if [ -z "$POD_NAMESPACE" ]; then
# Single container runs in docker
echo "POD_NAMESPACE not set, spin up single node"
else
# Is running in Kubernetes/OpenShift, so find all other pods
# belonging to the namespace
echo "Elasticsearch: Finding peers"
K8S_SVC_NAME=$(hostname -f | cut -d"." -f2)
echo "Using service name: ${K8S_SVC_NAME}"
# copy the pristine version to the one that can be edited
/usr/bin/peer-finder -on-start="/lagoon/configure-es.sh" -service=${K8S_SVC_NAME}
fi
exec $@
Binary file not shown.
@@ -23,7 +23,6 @@ else
echo "Using service name: ${K8S_SVC_NAME}"
# copy the pristine version to the one that can be edited
cp ${CONTAINER_SCRIPTS_DIR}/galera.cnf ${EXTRA_DEFAULTS_FILE}
/usr/bin/peer-finder -on-start="${CONTAINER_SCRIPTS_DIR}/configure-galera.sh" -service=${K8S_SVC_NAME}
fi
@@ -500,6 +500,11 @@ do
SERVICE_NAME="${SERVICE_NAME}-maxscale"
. /oc-build-deploy/scripts/exec-monitor-deploy.sh
elif [ $SERVICE_TYPE == "elasticsearch-cluster" ]; then
STATEFULSET="${SERVICE_NAME}"
. /oc-build-deploy/scripts/exec-monitor-statefulset.sh
elif [ ! $SERVICE_ROLLOUT_TYPE == "false" ]; then
. /oc-build-deploy/scripts/exec-monitor-deploy.sh
fi
@@ -0,0 +1,58 @@
apiVersion: v1
kind: Template
metadata:
creationTimestamp: null
name: lagoon-openshift-template-elasticsearch
parameters:
- name: SERVICE_NAME
description: Name of this service
required: true
- name: SAFE_BRANCH
description: Which branch this belongs to, special chars replaced with dashes
required: true
- name: SAFE_PROJECT
description: Which project this belongs to, special chars replaced with dashes
required: true
- name: BRANCH
description: Which branch this belongs to, original value
required: true
- name: PROJECT
description: Which project this belongs to, original value
required: true
- name: LAGOON_GIT_SHA
description: git hash sha of the current deployment
required: true
- name: SERVICE_ROUTER_URL
description: URL of the Router for this service
value: ""
- name: OPENSHIFT_PROJECT
description: Name of the Project that this service is in
required: true
- name: REGISTRY
description: Registry where Images are pushed to
required: true
objects:
- apiVersion: v1
kind: Service
metadata:
annotations:
service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
creationTimestamp: null
labels:
service: ${SERVICE_NAME}
branch: ${SAFE_BRANCH}
project: ${SAFE_PROJECT}
name: ${SERVICE_NAME}
spec:
clusterIP: None
ports:
- name: 9200-tcp
port: 9200
protocol: TCP
- name: 9300-tcp
port: 9300
protocol: TCP
selector:
service: ${SERVICE_NAME}
status:
loadBalancer: {}
@@ -0,0 +1,92 @@
apiVersion: v1
kind: Template
metadata:
creationTimestamp: null
name: lagoon-openshift-template-elasticsearch-cluster
parameters:
- name: SERVICE_NAME
description: Name of this service
required: true
- name: SAFE_BRANCH
description: Which branch this belongs to, special chars replaced with dashes
required: true
- name: SAFE_PROJECT
description: Which project this belongs to, special chars replaced with dashes
required: true
- name: BRANCH
description: Which branch this belongs to, original value
required: true
- name: PROJECT
description: Which project this belongs to, original value
required: true
- name: LAGOON_GIT_SHA
description: git hash sha of the current deployment
required: true
- name: SERVICE_ROUTER_URL
description: URL of the Router for this service
value: ""
- name: OPENSHIFT_PROJECT
description: Name of the Project that this service is in
required: true
- name: REGISTRY
description: Registry where Images are pushed to
required: true
- name: SERVICE_IMAGE
description: Pullable image of service
required: true
objects:
- apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
name: ${SERVICE_NAME}
spec:
serviceName: ${SERVICE_NAME}
replicas: 3
selector:
matchLabels:
service: ${SERVICE_NAME}
template:
metadata:
labels:
branch: ${SAFE_BRANCH}
project: ${SAFE_PROJECT}
service: ${SERVICE_NAME}
spec:
containers:
- name: ${SERVICE_NAME}
image: ${SERVICE_IMAGE}
imagePullPolicy: Always
ports:
- containerPort: 9200
protocol: TCP
- containerPort: 9300
protocol: TCP
readinessProbe:
httpGet:
path: /_cluster/health?local=true
port: 9200
initialDelaySeconds: 20
livenessProbe:
httpGet:
path: /_cluster/health?local=true
port: 9200
initialDelaySeconds: 120
envFrom:
- configMapRef:
name: lagoon-env
resources:
requests:
cpu: 10m
memory: 10Mi
volumeMounts:
- mountPath: /usr/share/elasticsearch/data
name: ${SERVICE_NAME}
env:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
volumes:
- name: ${SERVICE_NAME}
emptyDir: {}
@@ -94,4 +94,4 @@ objects:
test: false
triggers:
- type: ConfigChange
status: {}
status: {}
@@ -155,18 +155,25 @@ mutation FillAPI {
CiFeatures: addProject(input: {id: 12, name: "ci-features", customer: 3, openshift: 2, git_url: "ssh://git@192.168.99.1:2222/git/features.git", production_environment:"master"}) {
id
}
CiFeaturesSlack: addNotificationToProject(input: {project: "ci-features", notificationType: "slack", notificationName: "amazeeio--lagoon-local-ci"}) {
CiFeaturesSlack: addNotificationToProject(input: {project: "ci-features", notificationType: "slack", notificationName: "amazeeio--lagoon-local-ci"}) {
id
}
CiDrupalGalera: addProject(input: {id: 14, name: "ci-drupal-galera", customer: 3, openshift: 2, git_url: "ssh://git@192.168.99.1:2222/git/drupal-galera.git"}) {
CiElasticsearch: addProject(input: {id: 14, name: "ci-elasticsearch", customer: 3, openshift: 2, git_url: "ssh://git@192.168.99.1:2222/git/elasticsearch.git", production_environment:"master"}) {
id
}
CiElasticsearchSlack: addNotificationToProject(input: {project: "ci-elasticsearch", notificationType: "slack", notificationName: "amazeeio--lagoon-local-ci"}) {
id
}
CiDrupalGalera: addProject(input: {id: 15, name: "ci-drupal-galera", customer: 3, openshift: 2, git_url: "ssh://git@192.168.99.1:2222/git/drupal-galera.git"}) {
id
}
CiDrupalGaleraSlack: addNotificationToProject(input: {project: "ci-drupal-galera", notificationType: "slack", notificationName: "amazeeio--lagoon-local-ci"}) {
id
}
CiDrupalPostgres: addProject(input: {id: 15, name: "ci-drupal-postgres", customer: 3, openshift: 2, git_url: "ssh://git@192.168.99.1:2222/git/drupal-postgres.git"}) {
CiDrupalPostgres: addProject(input: {id: 16, name: "ci-drupal-postgres", customer: 3, openshift: 2, git_url: "ssh://git@192.168.99.1:2222/git/drupal-postgres.git"}) {
id
}
CiDrupalPostgresSlack: addNotificationToProject(input: {project: "ci-drupal-postgres", notificationType: "slack", notificationName: "amazeeio--lagoon-local-ci"}) {
View
@@ -23,8 +23,9 @@ RUN mkdir -m 700 /git/.ssh && \
git --bare init /git/drupal-galera.git && \
git --bare init /git/drupal-postgres.git && \
git --bare init /git/nginx.git && \
git --bare init /git/features.git
git --bare init /git/features.git && \
git --bare init /git/elasticsearch.git
USER root
COPY authorized-keys.sh /etc/my_init.d/authorized-keys.sh
COPY authorized-keys.sh /etc/my_init.d/authorized-keys.sh
@@ -1,4 +1,4 @@
ARG IMAGE_REPO
FROM ${IMAGE_REPO:-lagoon}/elasticsearch
COPY conf/elasticsearch.yml /usr/share/elasticsearch/config/elasticsearch.yml
COPY docker-entrypoint.sh /lagoon/entrypoints/95-logs-db.sh
@@ -1,12 +1,8 @@
cluster.name: "docker-cluster"
network.host: 0.0.0.0
#!/usr/bin/env bash
# minimum_master_nodes need to be explicitly set when bound on a public IP
# set to 1 to allow single node clusters
# Details: https://github.com/elastic/elasticsearch/pull/17288
discovery.zen.minimum_master_nodes: 1
xpack.license.self_generated.type: trial
xpack.security.enabled: false
set -eo pipefail
cat <<EOT >> /usr/share/elasticsearch/config/elasticsearch.yml
xpack.notification:
slack:
account:
@@ -30,3 +26,4 @@ xpack.notification:
description: "Elasticsearch Watch notification"
incident_key: "lagoon_logs_key"
event_type: trigger
EOT
@@ -0,0 +1,11 @@
---
- name: "{{ testname }} - Check if URL {{url}} contains json {{expected_key}}:{{expected_value}}"
uri:
url: "{{ url }}"
return_content: yes
body_format: json
register: response
until: response.json is defined and response.json.number_of_nodes is defined and response.json.number_of_nodes == node_count
retries: 30
delay: 10
- debug: msg="Success!!!"
@@ -0,0 +1 @@
docker-compose-yaml: docker-compose.yml
@@ -0,0 +1,2 @@
ARG IMAGE_REPO
FROM ${IMAGE_REPO:-amazeeio}/elasticsearch
@@ -0,0 +1,4 @@
ARG IMAGE_REPO
FROM ${IMAGE_REPO:-amazeeio}/nginx
COPY app.conf /etc/nginx/conf.d/app.conf
@@ -0,0 +1,10 @@
server {
listen 8080 default_server;
include /etc/nginx/helpers/*.conf;
location / {
proxy_pass http://elasticsearch:9200/;
}
}
Oops, something went wrong.

0 comments on commit 1a72498

Please sign in to comment.