Skip to content

Commit d137157

Browse files
bearddan2000bearddan2000
committed
initial commit
0 parents  commit d137157

15 files changed

+502
-0
lines changed

‎README.md

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# python-cli-sqlserver-to-elasticsearch-client-simple
2+
3+
## Description
4+
Reads a single node for data in `animal-demo` document.
5+
6+
Uses `dog` table then covverts it to json for
7+
elasticsearch to use.
8+
9+
## Tech stack
10+
- python
11+
- pymssql
12+
- sqlalchemy
13+
- elasticsearch
14+
- kibana
15+
- mssql
16+
17+
## Docker stack
18+
- python
19+
- elasticsearch
20+
- kibana
21+
- mcr.microsoft.com/mssql/server:2017-CU17-ubuntu
22+
23+
## To run
24+
`sudo ./install.sh -u`
25+
26+
## To stop (optional)
27+
`sudo ./install.sh -d`
28+
29+
## For help
30+
`sudo ./install.sh -h`
31+
32+
## Credit
33+
- [Docker setup](https://lynn-kwong.medium.com/all-you-need-to-know-about-using-elasticsearch-in-python-b9ed00e0fdf0)
34+
- [Search setup](https://www.elastic.co/guide/en/elasticsearch/client/python-api/master/examples.html)

‎db/Dockerfile

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# We choose exact tag (not 'latest'), to be sure that new version won't break creating image
2+
FROM mcr.microsoft.com/mssql/server:2017-CU17-ubuntu
3+
4+
# Create app directory
5+
RUN mkdir -p /usr/src/app
6+
WORKDIR /usr/src/app
7+
8+
# Copy initialization scripts
9+
COPY . /usr/src/app
10+
# COPY sql /usr/src/app
11+
12+
# Grant permissions for the run-initialization script to be executable
13+
RUN chmod +x /usr/src/app/run-initialization.sh
14+
15+
# Set environment variables, not to have to write them with docker run command
16+
# Note: make sure that your password matches what is in the run-initialization script
17+
ENV SA_PASSWORD z!x<?oB1ab
18+
ENV ACCEPT_EULA Y
19+
ENV MSSQL_PID Express
20+
21+
# Expose port 1433 in case accesing from other container
22+
EXPOSE 1433
23+
24+
# Run Microsoft SQl Server and initialization script (at the same time)
25+
# Note: If you want to start MsSQL only (without initialization script) you can comment bellow line out, CMD entry from base image will be taken
26+
CMD /bin/bash ./entrypoint.sh

‎db/entrypoint.sh

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
/usr/src/app/run-initialization.sh & /opt/mssql/bin/sqlservr

‎db/run-initialization.sh

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
function do-sql()
2+
{
3+
local cmd=/opt/mssql-tools/bin/sqlcmd
4+
local server="localhost"
5+
local user="sa"
6+
local password="z!x<?oB1ab"
7+
local sql_file=$1
8+
9+
$cmd -S $server -U $user -P $password -i $sql_file
10+
}
11+
# Wait to be sure that SQL Server came up
12+
sleep 90s
13+
14+
# Run the setup script to create the DB and the schema in the DB
15+
# Note: make sure that your password matches what is in the Dockerfile
16+
17+
echo "processing file /usr/src/app/schema.sql"
18+
do-sql /usr/src/app/schema.sql

‎db/schema.sql

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
USE master;
2+
Go
3+
4+
CREATE DATABASE animal;
5+
Go
6+
7+
CREATE TABLE dbo.dog (
8+
id INT PRIMARY KEY IDENTITY(1,1),
9+
breed VARCHAR(20) NOT NULL,
10+
color VARCHAR(20) NOT NULL
11+
);
12+
Go
13+
14+
INSERT INTO dbo.dog (breed, color)
15+
VALUES
16+
('Am Bulldog', 'White'),
17+
('Blue Tick', 'Grey'),
18+
('Labrador', 'Black'),
19+
('Gr Shepard', 'Brown');
20+
Go

‎docker-compose.yml

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
version: "3.4"
2+
3+
services:
4+
5+
py-srv:
6+
build: py-srv
7+
healthcheck:
8+
test: "exit 0"
9+
command: sh -c "/wait && python app.py"
10+
environment:
11+
- WAIT_HOSTS=db:1433,elasticsearch:9200
12+
- WAIT_HOSTS_TIMEOUT=300
13+
- WAIT_SLEEP_INTERVAL=30
14+
- WAIT_HOST_CONNECT_TIMEOUT=30
15+
depends_on:
16+
- db
17+
- elasticsearch
18+
links:
19+
- "db:db"
20+
- "elasticsearch:elasticsearch"
21+
22+
db:
23+
build: db
24+
25+
elasticsearch:
26+
image: elasticsearch:7.12.0
27+
healthcheck:
28+
test: "exit 0"
29+
interval: 1m30s
30+
timeout: 30s
31+
retries: 5
32+
start_period: 30s
33+
environment:
34+
- discovery.type=single-node
35+
- bootstrap.memory_lock=true
36+
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
37+
ulimits:
38+
memlock:
39+
soft: -1
40+
hard: -1
41+
ports:
42+
- 9200:9200
43+
44+
kibana:
45+
image: kibana:7.12.0
46+
healthcheck:
47+
test: "exit 0"
48+
interval: 1m30s
49+
timeout: 30s
50+
retries: 5
51+
start_period: 30s
52+
environment:
53+
- "ELASTICSEARCH_URL=http://elasticsearch:9200"
54+
- "SERVER_NAME=127.0.0.1"
55+
ports:
56+
- 5601:5601
57+
depends_on:
58+
- elasticsearch
59+
60+
volumes:
61+
esdata:
62+
driver: local
63+

‎install.sh

+139
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
#!/usr/bin/env bash
2+
basefile="install"
3+
logfile="general.log"
4+
timestamp=`date '+%Y-%m-%d %H:%M:%S'`
5+
6+
if [ "$#" -ne 1 ]; then
7+
msg="[ERROR]: $basefile failed to receive enough args"
8+
echo "$msg"
9+
echo "$msg" >> $logfile
10+
exit 1
11+
fi
12+
13+
function setup-logging(){
14+
scope="setup-logging"
15+
info_base="[$timestamp INFO]: $basefile::$scope"
16+
17+
echo "$info_base started" >> $logfile
18+
19+
echo "$info_base removing old logs" >> $logfile
20+
21+
rm -f $logfile
22+
23+
echo "$info_base ended" >> $logfile
24+
25+
echo "================" >> $logfile
26+
}
27+
28+
function root-check(){
29+
scope="root-check"
30+
info_base="[$timestamp INFO]: $basefile::$scope"
31+
32+
echo "$info_base started" >> $logfile
33+
34+
#Make sure the script is running as root.
35+
if [ "$UID" -ne "0" ]; then
36+
echo "[$timestamp ERROR]: $basefile::$scope you must be root to run $0" >> $logfile
37+
echo "==================" >> $logfile
38+
echo "You must be root to run $0. Try the following"
39+
echo "sudo $0"
40+
exit 1
41+
fi
42+
43+
echo "$info_base ended" >> $logfile
44+
echo "================" >> $logfile
45+
}
46+
47+
function docker-check() {
48+
scope="docker-check"
49+
info_base="[$timestamp INFO]: $basefile::$scope"
50+
cmd=`docker -v`
51+
52+
echo "$info_base started" >> $logfile
53+
54+
if [ -z "$cmd" ]; then
55+
echo "$info_base docker not installed"
56+
echo "$info_base docker not installed" >> $logfile
57+
fi
58+
59+
echo "$info_base ended" >> $logfile
60+
echo "================" >> $logfile
61+
62+
}
63+
64+
function docker-compose-check() {
65+
scope="docker-compose-check"
66+
info_base="[$timestamp INFO]: $basefile::$scope"
67+
cmd=`docker-compose -v`
68+
69+
echo "$info_base started" >> $logfile
70+
71+
if [ -z "$cmd" ]; then
72+
echo "$info_base docker-compose not installed"
73+
echo "$info_base docker-compose not installed" >> $logfile
74+
fi
75+
76+
echo "$info_base ended" >> $logfile
77+
echo "================" >> $logfile
78+
79+
}
80+
function usage() {
81+
echo ""
82+
echo "Usage: "
83+
echo ""
84+
echo "-u: start."
85+
echo "-d: tear down."
86+
echo "-h: Display this help and exit."
87+
echo ""
88+
}
89+
function start-up(){
90+
91+
local scope="start-up"
92+
local docker_img_name=`head -n 1 README.md | sed 's/# //'`
93+
local info_base="[$timestamp INFO]: $basefile::$scope"
94+
95+
echo "$info_base started" >> $logfile
96+
97+
echo "$info_base starting services" >> $logfile
98+
99+
sudo docker-compose up --build
100+
101+
echo "$info_base ended" >> $logfile
102+
103+
echo "================" >> $logfile
104+
}
105+
function tear-down(){
106+
107+
scope="tear-down"
108+
info_base="[$timestamp INFO]: $basefile::$scope"
109+
110+
echo "$info_base started" >> $logfile
111+
112+
echo "$info_base starting services" >> $logfile
113+
114+
sudo docker-compose down
115+
116+
echo "$info_base ended" >> $logfile
117+
118+
echo "================" >> $logfile
119+
}
120+
121+
root-check
122+
docker-check
123+
docker-compose-check
124+
125+
while getopts ":udh" opts; do
126+
case $opts in
127+
u)
128+
setup-logging
129+
start-up ;;
130+
d)
131+
tear-down ;;
132+
h)
133+
usage
134+
exit 0 ;;
135+
/?)
136+
usage
137+
exit 1 ;;
138+
esac
139+
done

‎py-srv/Dockerfile

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
FROM python
2+
3+
RUN /usr/local/bin/python -m pip install --upgrade pip
4+
5+
COPY bin/ /app
6+
7+
WORKDIR /app
8+
9+
RUN pip install -r requirements.txt
10+
11+
ENV WAIT_VERSION 2.7.2
12+
13+
ADD https://github.com/ufoscout/docker-compose-wait/releases/download/$WAIT_VERSION/wait /wait
14+
15+
RUN chmod +x /wait
16+
17+
CMD ["python", "app.py"]

‎py-srv/bin/__init__.py

Whitespace-only changes.

‎py-srv/bin/app.py

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
from elasticsearch import Elasticsearch
2+
from elasticsearch.client import IndicesClient
3+
4+
from db_schema import configurations
5+
from data import DOC, INDEX_NAME
6+
7+
es = Elasticsearch(
8+
"elasticsearch:9200",
9+
http_auth=["elastic", "changeme"],
10+
)
11+
12+
def print_index():
13+
for record in DOC:
14+
resp=es.index(index=INDEX_NAME, id=record['id'], document=record)
15+
print("print_index: {}".format(resp['result']))
16+
17+
def print_get():
18+
resp=es.get(index=INDEX_NAME, id=1)
19+
print("print_get: {}".format(resp['_source']))
20+
21+
def print_search():
22+
SELECT_ALL = {"match_all": {}}
23+
es.indices.refresh(index=INDEX_NAME)
24+
resp=es.search(index=INDEX_NAME, query=SELECT_ALL, size=5)
25+
print("print_search_hits: {}".format(resp['hits']['total']['value']))
26+
for hit in resp['hits']['hits']:
27+
print("print_search: {}".format(hit["_source"]))
28+
29+
print_index()
30+
print_get()
31+
print_search()

0 commit comments

Comments
 (0)