Skip to content

Commit 4173862

Browse files
bearddan2000bearddan2000
committed
initial commit
0 parents  commit 4173862

File tree

17 files changed

+586
-0
lines changed

17 files changed

+586
-0
lines changed

‎.env‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
STACK_VERSION=8.6.1

‎README.md‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# python-cli-sqlserver-to-multi-node-elasticsearch-client-without-ssl-pop
2+
3+
## Description
4+
Reads a multi node cluster for data in `pop-demo` document.
5+
6+
Uses `pop` table then covverts it to json for
7+
elasticsearch to use.
8+
9+
## Tech stack
10+
- python
11+
- pymssql
12+
- sqlalchemy
13+
- elasticsearch
14+
- kibana
15+
- mssql
16+
17+
## Docker stack
18+
- python
19+
- elasticsearch
20+
- kibana
21+
- mcr.microsoft.com/mssql/server:2017-CU17-ubuntu
22+
23+
## To run
24+
`sudo ./install.sh -u`
25+
26+
## To stop (optional)
27+
`sudo ./install.sh -d`
28+
29+
## For help
30+
`sudo ./install.sh -h`
31+
32+
## Credit
33+
- [Docker setup](https://lynn-kwong.medium.com/all-you-need-to-know-about-using-elasticsearch-in-python-b9ed00e0fdf0)
34+
- [Search setup](https://www.elastic.co/guide/en/elasticsearch/client/python-api/master/examples.html)

‎db/Dockerfile‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# We choose exact tag (not 'latest'), to be sure that new version won't break creating image
2+
FROM mcr.microsoft.com/mssql/server:2017-CU17-ubuntu
3+
4+
# Create app directory
5+
RUN mkdir -p /usr/src/app
6+
WORKDIR /usr/src/app
7+
8+
# Copy initialization scripts
9+
COPY . /usr/src/app
10+
# COPY sql /usr/src/app
11+
12+
# Grant permissions for the run-initialization script to be executable
13+
RUN chmod +x /usr/src/app/run-initialization.sh
14+
15+
# Set environment variables, not to have to write them with docker run command
16+
# Note: make sure that your password matches what is in the run-initialization script
17+
ENV SA_PASSWORD z!x<?oB1ab
18+
ENV ACCEPT_EULA Y
19+
ENV MSSQL_PID Express
20+
21+
# Expose port 1433 in case accesing from other container
22+
EXPOSE 1433
23+
24+
# Run Microsoft SQl Server and initialization script (at the same time)
25+
# Note: If you want to start MsSQL only (without initialization script) you can comment bellow line out, CMD entry from base image will be taken
26+
CMD /bin/bash ./entrypoint.sh

‎db/entrypoint.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
/usr/src/app/run-initialization.sh & /opt/mssql/bin/sqlservr

‎db/run-initialization.sh‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
function do-sql()
2+
{
3+
local cmd=/opt/mssql-tools/bin/sqlcmd
4+
local server="localhost"
5+
local user="sa"
6+
local password="z!x<?oB1ab"
7+
local sql_file=$1
8+
9+
$cmd -S $server -U $user -P $password -i $sql_file
10+
}
11+
# Wait to be sure that SQL Server came up
12+
sleep 90s
13+
14+
# Run the setup script to create the DB and the schema in the DB
15+
# Note: make sure that your password matches what is in the Dockerfile
16+
17+
echo "processing file /usr/src/app/schema.sql"
18+
do-sql /usr/src/app/schema.sql

‎db/schema.sql‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
USE master;
2+
Go
3+
4+
CREATE DATABASE beverage;
5+
Go
6+
7+
CREATE TABLE dbo.pop (
8+
id INT PRIMARY KEY IDENTITY(1,1),
9+
name VARCHAR(20) NOT NULL,
10+
color VARCHAR(20) NOT NULL
11+
);
12+
Go
13+
14+
INSERT INTO dbo.pop (name, color)
15+
VALUES
16+
('RC Cola', 'brown'),
17+
('Sprite', 'clear'),
18+
('Verners', 'brown'),
19+
('Mt. Lightening', 'green');
20+
Go

‎docker-compose.yml‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
version: "3.4"
2+
3+
services:
4+
5+
db:
6+
build: db
7+
8+
py-srv:
9+
build: py-srv
10+
healthcheck:
11+
test: "exit 0"
12+
command: sh -c "/wait && python app.py"
13+
environment:
14+
- WAIT_HOSTS=db:1433,es1:9200,es2:9200,es3:9200
15+
- WAIT_HOSTS_TIMEOUT=300
16+
- WAIT_SLEEP_INTERVAL=30
17+
- WAIT_HOST_CONNECT_TIMEOUT=30
18+
depends_on:
19+
- db
20+
- es1
21+
- es2
22+
- es3
23+
- kibana
24+
links:
25+
- "db:db"
26+
- "es1:es1"
27+
- "es2:es2"
28+
- "es3:es3"
29+
30+
es1:
31+
image: elasticsearch:${STACK_VERSION}
32+
container_name: es1
33+
hostname: es1
34+
restart: unless-stopped
35+
healthcheck:
36+
test: "exit 0"
37+
environment:
38+
- "node.store.allow_mmap=false"
39+
- "node.name=es1"
40+
- "bootstrap.memory_lock=true"
41+
- "cluster.name=es-cluster"
42+
- "discovery.seed_hosts=es2,es3"
43+
- "cluster.initial_master_nodes=es1,es2,es3"
44+
- "ES_JAVA_OPTS=-Xms512m -Xmx512m -Des.index.number_of_replicas=0"
45+
- "xpack.security.enabled=false"
46+
- "xpack.security.http.ssl.enabled=false"
47+
- "xpack.security.transport.ssl.enabled=false"
48+
- "xpack.ml.enabled=false"
49+
- "xpack.graph.enabled=false"
50+
- "xpack.watcher.enabled=false"
51+
ulimits:
52+
memlock:
53+
soft: -1
54+
hard: -1
55+
ports:
56+
- 9200:9200
57+
# volumes:
58+
# - ./es/es1/data:/usr/share/elasticsearch/data
59+
# - ./es/es1/log:/usr/share/elasticsearch/log
60+
61+
es2:
62+
image: elasticsearch:${STACK_VERSION}
63+
container_name: es2
64+
hostname: es2
65+
restart: unless-stopped
66+
healthcheck:
67+
test: "exit 0"
68+
environment:
69+
- "node.store.allow_mmap=false"
70+
- "node.name=es2"
71+
- "bootstrap.memory_lock=true"
72+
- "cluster.name=es-cluster"
73+
- "discovery.seed_hosts=es3,es1"
74+
- "cluster.initial_master_nodes=es1,es2,es3"
75+
- "ES_JAVA_OPTS=-Xms512m -Xmx512m -Des.index.number_of_replicas=0"
76+
- "xpack.security.enabled=false"
77+
- "xpack.security.http.ssl.enabled=false"
78+
- "xpack.security.transport.ssl.enabled=false"
79+
- "xpack.ml.enabled=false"
80+
- "xpack.graph.enabled=false"
81+
- "xpack.watcher.enabled=false"
82+
ulimits:
83+
memlock:
84+
soft: -1
85+
hard: -1
86+
ports:
87+
- 9201:9200
88+
# volumes:
89+
# - ./es/es2/data:/usr/share/elasticsearch/data
90+
# - ./es/es2/log:/usr/share/elasticsearch/log
91+
92+
es3:
93+
image: elasticsearch:${STACK_VERSION}
94+
container_name: es3
95+
hostname: es3
96+
restart: unless-stopped
97+
healthcheck:
98+
test: "exit 0"
99+
environment:
100+
- "node.store.allow_mmap=false"
101+
- "node.name=es3"
102+
- "bootstrap.memory_lock=true"
103+
- "cluster.name=es-cluster"
104+
- "discovery.seed_hosts=es2,es1"
105+
- "cluster.initial_master_nodes=es1,es2,es3"
106+
- "ES_JAVA_OPTS=-Xms512m -Xmx512m -Des.index.number_of_replicas=0"
107+
- "xpack.security.enabled=false"
108+
- "xpack.security.http.ssl.enabled=false"
109+
- "xpack.security.transport.ssl.enabled=false"
110+
- "xpack.ml.enabled=false"
111+
- "xpack.graph.enabled=false"
112+
- "xpack.watcher.enabled=false"
113+
ulimits:
114+
memlock:
115+
soft: -1
116+
hard: -1
117+
ports:
118+
- 9202:9200
119+
# volumes:
120+
# - ./es/es3/data:/usr/share/elasticsearch/data
121+
# - ./es/es3/log:/usr/share/elasticsearch/log
122+
123+
kibana:
124+
image: kibana:${STACK_VERSION}
125+
healthcheck:
126+
test: "exit 0"
127+
environment:
128+
- "ELASTICSEARCH_HOSTS=http://es1:9200"
129+
- "SERVER_NAME=127.0.0.1"
130+
ports:
131+
- 5601:5601
132+
depends_on:
133+
- es1
134+
- es2
135+
- es3

‎install.sh‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
#!/usr/bin/env bash
2+
basefile="install"
3+
logfile="general.log"
4+
timestamp=`date '+%Y-%m-%d %H:%M:%S'`
5+
6+
if [ "$#" -ne 1 ]; then
7+
msg="[ERROR]: $basefile failed to receive enough args"
8+
echo "$msg"
9+
echo "$msg" >> $logfile
10+
exit 1
11+
fi
12+
13+
function setup-logging(){
14+
scope="setup-logging"
15+
info_base="[$timestamp INFO]: $basefile::$scope"
16+
17+
echo "$info_base started" >> $logfile
18+
19+
echo "$info_base removing old logs" >> $logfile
20+
21+
rm -f $logfile
22+
23+
echo "$info_base ended" >> $logfile
24+
25+
echo "================" >> $logfile
26+
}
27+
28+
function root-check(){
29+
scope="root-check"
30+
info_base="[$timestamp INFO]: $basefile::$scope"
31+
32+
echo "$info_base started" >> $logfile
33+
34+
#Make sure the script is running as root.
35+
if [ "$UID" -ne "0" ]; then
36+
echo "[$timestamp ERROR]: $basefile::$scope you must be root to run $0" >> $logfile
37+
echo "==================" >> $logfile
38+
echo "You must be root to run $0. Try the following"
39+
echo "sudo $0"
40+
exit 1
41+
fi
42+
43+
echo "$info_base ended" >> $logfile
44+
echo "================" >> $logfile
45+
}
46+
47+
function docker-check() {
48+
scope="docker-check"
49+
info_base="[$timestamp INFO]: $basefile::$scope"
50+
cmd=`docker -v`
51+
52+
echo "$info_base started" >> $logfile
53+
54+
if [ -z "$cmd" ]; then
55+
echo "$info_base docker not installed"
56+
echo "$info_base docker not installed" >> $logfile
57+
fi
58+
59+
echo "$info_base ended" >> $logfile
60+
echo "================" >> $logfile
61+
62+
}
63+
64+
function docker-compose-check() {
65+
scope="docker-compose-check"
66+
info_base="[$timestamp INFO]: $basefile::$scope"
67+
cmd=`docker-compose -v`
68+
69+
echo "$info_base started" >> $logfile
70+
71+
if [ -z "$cmd" ]; then
72+
echo "$info_base docker-compose not installed"
73+
echo "$info_base docker-compose not installed" >> $logfile
74+
fi
75+
76+
echo "$info_base ended" >> $logfile
77+
echo "================" >> $logfile
78+
79+
}
80+
function usage() {
81+
echo ""
82+
echo "Usage: "
83+
echo ""
84+
echo "-u: start."
85+
echo "-d: tear down."
86+
echo "-h: Display this help and exit."
87+
echo ""
88+
}
89+
function start-up(){
90+
91+
local scope="start-up"
92+
local docker_img_name=`head -n 1 README.md | sed 's/# //'`
93+
local info_base="[$timestamp INFO]: $basefile::$scope"
94+
95+
echo "$info_base started" >> $logfile
96+
97+
echo "$info_base starting services" >> $logfile
98+
99+
sudo docker-compose up --build
100+
101+
echo "$info_base ended" >> $logfile
102+
103+
echo "================" >> $logfile
104+
}
105+
function tear-down(){
106+
107+
scope="tear-down"
108+
info_base="[$timestamp INFO]: $basefile::$scope"
109+
110+
echo "$info_base started" >> $logfile
111+
112+
echo "$info_base starting services" >> $logfile
113+
114+
sudo docker-compose down
115+
116+
echo "$info_base ended" >> $logfile
117+
118+
echo "================" >> $logfile
119+
}
120+
121+
root-check
122+
docker-check
123+
docker-compose-check
124+
125+
while getopts ":udh" opts; do
126+
case $opts in
127+
u)
128+
setup-logging
129+
start-up ;;
130+
d)
131+
tear-down ;;
132+
h)
133+
usage
134+
exit 0 ;;
135+
/?)
136+
usage
137+
exit 1 ;;
138+
esac
139+
done

0 commit comments

Comments
 (0)