Skip to content
Permalink
Browse files
[FLAGON-450] Updated Elastisearch builds to 6.8.2
  • Loading branch information
poorejc committed Aug 30, 2019
1 parent 9f17ad9 commit f6edca0efb5915e11e024eb2e46982d484dbeef5
Show file tree
Hide file tree
Showing 15 changed files with 1,051 additions and 45 deletions.
@@ -0,0 +1,163 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Apache Flagon Single-Node Build

version: "3"
services:

# Startup
startup:
build: ./startup
depends_on:
- "elasticsearch"
command: bash /opt/entrypoint.sh

# Site
site:
build: ./site
ports:
- 8080:8080
depends_on:
- "logstash"
command: python -m SimpleHTTPServer 8080

# Elasticsearch
elasticsearch:
build: ./elasticsearch/6.6.2
environment:
- NODE_MASTER=true
- NUMBER_OF_MASTERS=1
- NODE_COORDINATE=true
- "DISCOVERY_SERVICE=0.0.0.0"
- NODE_DATA=true
- NODE_INGEST=true
- MAX_LOCAL_STORAGE_NODES=1
- NETWORK_HOST=0.0.0.0
- HTTP_ENABLE=true
- HTTP_CORS_ENABLE=true
- HTTP_CORS_ALLOW_ORIGIN=*
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
ulimits:
memlock:
soft: -1
hard: -1
healthcheck:
test: ["CMD", "curl","-s" ,"-f", "-u", "http://localhost:9200/_cluster/health"]
# Mount volumes to backup elasticsearch data
# volumes:
# - esdata:/usr/share/elasticsaerch/data
ports:
- "9200:9200"
networks:
- esnet

# Kibana
kibana:
build: ./kibana/6.6.2
environment:
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- "ELASTICSEARCH_URL=http://elasticsearch:9200"
ports:
- "5601:5601"
depends_on:
- "elasticsearch"
networks:
- esnet

# Logstash
logstash:
build: ./logstash/6.6.2
environment:
- "ELASTICSEARCH_URL=elasticsearch:9200"
ports:
- 8100:8100
volumes:
- ./logstash/6.6.2/pipeline/logstash-userale.conf:/usr/share/logstash/pipeline/logstash-userale.conf
- ./logstash/6.6.2/templates/userale.json:/usr/share/logstash/templates/userale.json
depends_on:
- "elasticsearch"
networks:
- esnet

# Distill
distill:
build: ./distill
ports:
- 8090:8090
depends_on:
- "elasticsearch"
networks:
- esnet

# TAP's Database
db:
build: ./db
image: senssoft/postgres:latest
ports:
- "5432:5432"

# TAP
tap:
build: ./tap
command: bash -c "/usr/src/app/wait-for-postgres.sh"
ports:
- "8010:8010"
depends_on:
- db
links:
- db:db

# Specific container and configuration to monitor the HOST (filesystem, memory, processes,...)
metricbeat:
image: docker.elastic.co/beats/metricbeat-oss:6.6.2
# restart: unless-stopped
user: root
pid: host
volumes:
#The commented sections below enable Metricbeat to monitor the Docker host
#rather than the Metricbeat container. These are used by the system module.
- /proc:/hostfs/proc:ro
- /sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro
#We mount the host filesystem so we can report on disk usage with the system module
- /:/hostfs:ro
#Mount the metricbeat configuration so users can make edits
- ./beats/metricbeat/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml
#Mount the modules.d directory into the container. This allows user to
#potentially make changes to the modules and they will be dynamically loaded.
- ./beats/metricbeat/modules.d/:/usr/share/metricbeat/modules.d/
#Allows us to report on docker from the hosts information
- /var/run/docker.sock:/var/run/docker.sock
depends_on:
- elasticsearch
environment:
- ELASTICSEARCH_URL=http://elasticsearch:9200
- KIBANA_URL=http://kibana:5601
- LOGSTASH_URL=http://logstash:9600
- POSTGRESQL_URL="postgresql://db:5432/tapdb?sslmode=disable"
- POSTGRESQL_USERNAME=tapuser
- POSTGRESQL_PASSWORD="Dr@p3rUs3r"
extra_hosts:
- "elasticsearch:172.17.0.1" # The IP of docker0 interface to access host from container
- "kibana:172.17.0.1" # The IP of docker0 interface to access host from container
- "logstash:172.17.0.1"
- "db:172.17.0.1"
command: metricbeat -e -system.hostfs=/hostfs -strict.perms=false
network_mode: host # Mandatory to monitor HOST filesystem, memory, processes,...

networks:
esnet:
external:
name: esnet
@@ -18,13 +18,6 @@
version: "3"
services:

# Startup
startup:
build: ./startup
depends_on:
- "elasticsearch"
command: bash /opt/entrypoint.sh

# Site
site:
build: ./site
@@ -36,7 +29,7 @@ services:

# Elasticsearch
elasticsearch:
build: ./elasticsearch/6.6.2
build: ./elasticsearch/6.8.2
environment:
- NODE_MASTER=true
- NUMBER_OF_MASTERS=1
@@ -66,7 +59,7 @@ services:

# Kibana
kibana:
build: ./kibana/6.6.2
build: ./kibana/6.8.2
environment:
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- "ELASTICSEARCH_URL=http://elasticsearch:9200"
@@ -79,50 +72,22 @@ services:

# Logstash
logstash:
build: ./logstash/6.6.2
build: ./logstash/6.8.2
environment:
- "ELASTICSEARCH_URL=elasticsearch:9200"
ports:
- 8100:8100
volumes:
- ./logstash/6.6.2/pipeline/logstash-userale.conf:/usr/share/logstash/pipeline/logstash-userale.conf
- ./logstash/6.6.2/templates/userale.json:/usr/share/logstash/templates/userale.json
- ./logstash/6.8.2/pipeline/logstash-userale.conf:/usr/share/logstash/pipeline/logstash-userale.conf
- ./logstash/6.8.2/templates/userale.json:/usr/share/logstash/templates/userale.json
depends_on:
- "elasticsearch"
networks:
- esnet

# Distill
distill:
build: ./distill
ports:
- 8090:8090
depends_on:
- "elasticsearch"
networks:
- esnet

# TAP's Database
db:
build: ./db
image: senssoft/postgres:latest
ports:
- "5432:5432"

# TAP
tap:
build: ./tap
command: bash -c "/usr/src/app/wait-for-postgres.sh"
ports:
- "8010:8010"
depends_on:
- db
links:
- db:db

# Specific container and configuration to monitor the HOST (filesystem, memory, processes,...)
metricbeat:
image: docker.elastic.co/beats/metricbeat-oss:6.6.2
image: docker.elastic.co/beats/metricbeat-oss:6.8.2
# restart: unless-stopped
user: root
pid: host
@@ -146,14 +111,10 @@ services:
- ELASTICSEARCH_URL=http://elasticsearch:9200
- KIBANA_URL=http://kibana:5601
- LOGSTASH_URL=http://logstash:9600
- POSTGRESQL_URL="postgresql://db:5432/tapdb?sslmode=disable"
- POSTGRESQL_USERNAME=tapuser
- POSTGRESQL_PASSWORD="Dr@p3rUs3r"
extra_hosts:
- "elasticsearch:172.17.0.1" # The IP of docker0 interface to access host from container
- "kibana:172.17.0.1" # The IP of docker0 interface to access host from container
- "logstash:172.17.0.1"
- "db:172.17.0.1"
command: metricbeat -e -system.hostfs=/hostfs -strict.perms=false
network_mode: host # Mandatory to monitor HOST filesystem, memory, processes,...

@@ -0,0 +1,30 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

FROM docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.2

MAINTAINER Joshua Poore <poorejc@apache.org>

# Override config
COPY config /usr/share/elasticsearch/config
RUN chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/config

# Set environment
ENV CLUSTER_NAME Flagon

# Kubernetes requires swap is turned off
ENV MEMORY_LOCK false

EXPOSE 9200
@@ -0,0 +1,51 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Custom config borrowed from Pires: https://github.com/pires/docker-elasticsearch-kubernetes

cluster:
name: ${CLUSTER_NAME:FLAGON}

node:
master: ${NODE_MASTER:true}
data: ${NODE_DATA:true}
name: ${NODE_NAME:${HOSTNAME}}
ingest: ${NODE_INGEST:true}
max_local_storage_nodes: ${MAX_LOCAL_STORAGE_NODES:1}

network.host: ${NETWORK_HOST:0.0.0.0}

search.remote.connect: ${NODE_COORDINATE:true}

path:
data: /usr/share/elasticsearch/data
logs: /usr/share/elasticsearch/log

bootstrap:
memory_lock: ${MEMORY_LOCK}

http:
enabled: ${HTTP_ENABLE:true}
compression: true
cors:
enabled: ${HTTP_CORS_ENABLE:true}
allow-origin: ${HTTP_CORS_ALLOW_ORIGIN:"*"}

# Prevent the "split brain" by configuring the majority of nodes (total number of nodes / 2 + 1):
discovery:
zen:
ping.unicast.hosts: ${DISCOVERY_SERVICE:127.0.0.1, [::1]}
ping.unicast.hosts.resolve_timeout: ${DISCOVERY_TIMEOUT:5s}
minimum_master_nodes: ${NUMBER_OF_MASTERS:1}
@@ -0,0 +1,24 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

status = error

appender.console.type = Console
appender.console.name = console
appender.console.layout.type = PatternLayout
appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name] %marker%m%n

rootLogger.level = info
rootLogger.appenderRef.console.ref = console
@@ -0,0 +1,20 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

FROM docker.elastic.co/kibana/kibana-oss:6.8.2

MAINTAINER Joshua Poore <poorejc@apache.org>

EXPOSE 5601

0 comments on commit f6edca0

Please sign in to comment.