added provision

.gitignore

@@ -14,3 +14,5 @@ classes/
 .ideaDataSources/
 dataSources/
 tmp.json
+*.retry
+.vagrant

README.md

@@ -16,10 +16,6 @@ Ecommerce项目包括：
 Spring Boot、Gradle、MySQL、Junit 5、Rest Assured、Docker、RabbitMQ、Ansible
 
 # 目录结构
-- local主要包含本地开发过程所需要用到的基础设施，比如RabbitMQ和ELK等，均通过Docker在本地机器启动
-- remote主要用于生产环境所需的基础设施，主要针对"虚拟机+Docker"的部署场景，使用Vagrant生产环境模拟虚拟机
+- local主要包含本地开发过程所需要用到的基础设施，比如RabbitMQ和ELK等，均通过Docker在本地机器启动。
+- remote主要用于生产环境所需的基础设施，主要针对"虚拟机+Docker"的部署场景，本地使用Vagrant虚拟机。
 
-
-# tmp
-
-discovery.zen.ping.unicast.hosts被discovery.seed_hosts取代

local/single-host-single-node-elk/README.md

@@ -1,5 +1,7 @@
 ## 用法
 - 本地启动整个ELK：`./start.sh`
+- 启动之后检查ES集群状态：`curl http://localhost:9201/_cluster/health?pretty`
+- 查看节点情况：`curl http://localhost:9201/_cat/nodes?v`
 - 启动之后可以访问Kibana：[http://localhsot:5602](http://localhsot:5602)
 - 首次启动Kibana需要创建Index(需要ES中有Index数据之后才行)，Logstash默认的Index为`logstash-*`格式。
 - 关闭ELK： `./stop.sh`，将清空所有数据
@@ -23,3 +25,29 @@
 |12202|logstash|12201(udp)|
 |5602|kibana|5601|
 |6380|redis|6379|
+
+## 启用Elasticsearch认证
+- 在`elasticsearch.yml`中配置：
+
+```
+xpack.security.enabled: true
+```
+
+- 启动ELK：`./start.sh`
+- 登录到elasticsearch容器中:`docker exec -it single-node-elk-es bash`
+- `cd`到 `/usr/share/elasticsearch/bin`目录
+- 设置密码：`./elasticsearch-setup-passwords auto`(自动生成)或者./elasticsearch-setup-passwords interactive`（手动生成）
+- 通过Basic Authentication访问API：
+
+```
+curl http://elastic:password-for-es@localhost:9201/_cluster/health?pretty
+```
+- 在`kibana.yml`中配置Kibana的访问用户：
+
+```
+elasticsearch.username: "kibana"
+elasticsearch.password: "password-for `kibana` user"
+```
+
+- 访问Kibana：[http://localhsot:5602](http://localhsot:5602)
+- 用设置的账户登录

local/single-host-single-node-elk/docker-compose.yml

@@ -10,7 +10,7 @@ services:
     networks:
       - single-node-elk-net
     environment:
-      ES_JAVA_OPTS: "-Xms2048m -Xmx2048m"
+      ES_JAVA_OPTS: "-Xms4096m -Xmx4096m"
       TZ: "Asia/Shanghai"
     ports:
       - 9201:9200
@@ -40,7 +40,7 @@ services:
     depends_on:
       - elasticsearch
     environment:
-      LS_JAVA_OPTS: "-Xms2048m -Xmx2048m"
+      LS_JAVA_OPTS: "-Xms4096m -Xmx4096m"
       TZ: "Asia/Shanghai"
 
   kibana:

local/single-host-single-node-elk/elasticsearch/elasticsearch.yml

@@ -3,3 +3,4 @@ cluster.name: "single-node-es-cluster"
 network.host: 0.0.0.0
 discovery.zen.minimum_master_nodes: 1
 discovery.type: single-node
+#xpack.security.enabled: true

remote/elk/README.md

@@ -0,0 +1,15 @@
+## Installation step
+- Create 5 VMs using Vagrant,`cd` to `vagrant` folder and run: `./start.sh`, every VM will get a DNS name
+- Make sure the 5 VMs is listed in `inventory.ini`
+- Provision all nodes: `cd` to project root, `./provision.sh elk-all`
+- Deploy commands:
+
+|Command|Usage|
+| --- | --- |
+|`deploy-all.sh`|Deploy the whole ELK stack|
+|`deploy-es-all.sh`|Deploy all ES nodes|
+|`deploy-es1.sh`|Deploy es1|
+|`deploy-es2.sh`|Deploy es2|
+|`deploy-es3.sh`|Deploy es3|
+|`deploy-kibana.sh`|Deploy Kibana|
+|`deploy-logstash.sh`|Deploy Logstash|

remote/elk/deploy-all.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+if [ -z "$1" ]
+  then
+    echo "Error: No inventory host provided, the host should exists in inventory.ini file, example: ./provision.sh elk-es1"
+    exit 1
+fi
+
+export ANSIBLE_HOST_KEY_CHECKING=false
+ansible-playbook -i inventory.ini -v provision-playbook.yml

remote/elk/playbook.yml

@@ -0,0 +1,45 @@
+---
+- name: Deploy Elasticsearch node1
+  hosts: es1
+  become: yes
+  roles:
+    - elasticsearch
+  tags:
+    - es
+    - es1
+
+- name: Deploy Elasticsearch node2
+  hosts: es2
+  become: yes
+  roles:
+    - elasticsearch
+  tags:
+    - es
+    - es2
+
+- name: Deploy Elasticsearch node3
+  hosts: es3
+  become: yes
+  roles:
+    - elasticsearch
+  tags:
+    - es
+    - es3
+
+
+- name: Deploy Kibana
+  hosts: kibana
+  become: yes
+  roles:
+    - kibana
+  tags:
+    - kibana
+
+- name: Deploy Logstash
+  hosts: logstash
+  become: yes
+  roles:
+    - logstash
+  tags:
+    - logstash
+

remote/elk/vagrant/README.md

@@ -0,0 +1,26 @@
+### Introduction
+- 5 nodes centos7 cluster with DHCP private network and DNS enabled
+- network: private(host and vm can access each other)
+- memory: 8096
+- cpu: 4
+- First step: install `landrush` vagrant plugin
+
+``` bash
+ vagrant plugin install landrush 
+```
+- Remember to run `vagrant landrush stop` when you switch wifi
+- start: `./start.sh`
+- destroy: `./destroy.sh`
+- Login: `vagrant ssh nodeName`
+- your own public key uploaded to the vm to enable generic SSH
+- DNS name:`[xxx].vagrant.local`
+
+## Node usage
+
+|Domain Name|Usage|
+| --- | --- |
+|es1.vagrant.local|ES node 1|
+|es2.vagrant.local|ES node 2|
+|es3.vagrant.local|ES node 3|
+|kibana.vagrant.local|Kibana|
+|logstash.vagrant.local|Logstash with Redis as input|

remote/elk/vagrant/Vagrantfile

@@ -0,0 +1,48 @@
+tld="vagrant.local"
+
+Vagrant.configure("2") do |config|
+  config.vm.box = "centos/7"
+  config.vm.provision "file", source: "~/.ssh/id_rsa.pub", destination: "~/.ssh/my_id_rsa.pub"
+  config.vm.provision "shell", inline: "cat ~/.ssh/my_id_rsa.pub >> ~/.ssh/authorized_keys", privileged: false
+
+  config.landrush.enabled = true
+  config.landrush.tld = tld
+
+  (1..3).each do |i|
+  config.vm.define "es#{i}" do |node|
+    hostName="es#{i}."+tld
+    node.vm.hostname = hostName
+      node.vm.network "private_network", type: "dhcp"
+      node.vm.provider "virtualbox" do |v|
+        v.memory = 8192
+        v.cpus = 4
+        v.name = hostName
+      end
+    end
+  end
+
+
+  config.vm.define "kibana" do |node|
+      hostName="kibana."+tld
+      node.vm.hostname = hostName
+      node.vm.network "private_network", type: "dhcp"
+      node.vm.provider "virtualbox" do |v|
+          v.memory = 8192
+          v.cpus = 4
+          v.name = hostName
+      end
+  end
+
+  config.vm.define "logstash" do |node|
+      hostName="logstash."+tld
+      node.vm.hostname = hostName
+      node.vm.network "private_network", type: "dhcp"
+      node.vm.provider "virtualbox" do |v|
+          v.memory = 8192
+          v.cpus = 4
+          v.name = hostName
+      end
+  end
+
+
+end

remote/elk/vagrant/destroy.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+vagrant destroy -f

remote/elk/vagrant/start.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+vagrant destroy -f
+vagrant up

remote/inventory.ini

@@ -0,0 +1,9 @@
+[elk-all]
+es1 ansible_ssh_host=es1.vagrant.local ansible_user=vagrant
+es2 ansible_ssh_host=es2.vagrant.local ansible_user=vagrant
+es3 ansible_ssh_host=es3.vagrant.local ansible_user=vagrant
+kibana ansible_ssh_host=kibana.vagrant.local ansible_user=vagrant
+logstash ansible_ssh_host=logstash.vagrant.local ansible_user=vagrant
+
+[nexus-all]
+nexus ansible_ssh_host=nexus.vagrant.local ansible_user=vagrant

remote/provision-playbook.yml

@@ -0,0 +1,9 @@
+---
+- hosts: "{{ host }}"
+  vars:
+    docker_registry_host: nexus.vagrant.local:5000
+    docker_registry_user: changeme
+    docker_registry_password: changeme
+  become: yes
+  roles:
+    - provision

remote/provision.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+if [ -z "$1" ]
+  then
+    echo "Error: No inventory host provided, the host should exists in inventory.ini file, example: ./provision.sh elk-es1"
+    exit 1
+fi
+
+export ANSIBLE_HOST_KEY_CHECKING=false
+ansible-playbook -i inventory.ini -v provision-playbook.yml --extra-vars "host=$1"

remote/roles/provision/README.md

@@ -0,0 +1,6 @@
+- Every Vagrant VM needs to do provision.
+- The provision process contains：
+    - docker and docker-compose
+    - Beijing Time
+    - Disable SELinux
+    - SSH user with in docker/wheel group

remote/roles/provision/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+docker_data_dir: /var/lib/docker

remote/roles/provision/handlers/main.yml

@@ -0,0 +1,3 @@
+---
+- name: Reboot system
+  reboot: