Skip to content

Commit c52d5bd

Browse files
perdeltperdelt
authored
V0.4.3 (#40)
* Prepare next release * Docs: Monitoring * K8s: v1apps.list_namespaced_deployment (new K8s version in cluster) * K8s: Resultfolder as attribute of cluster object * Masterscript: K8s store number of requested gpus * Masterscript: K8s run shell scripts for loading data * Masterscript: Allow list of jars per dbms * Requirements: Allow current versions * TPC-H: New specifics * Masterscript: K8s monitoring as option * Masterscript: K8s optional UPPER parameter of db and schema * Masterscript: K8s DEPRECATED: we must know all jars upfront * TPC-H: DDL for MariaDB Columnstore * TPC-H: Bigint at MonetDB * TPC-H: SQL Server precision and DB * TPC-H: OmniSci sharding * TPC-H: OmniSci template
1 parent 2c4e56e commit c52d5bd

14 files changed

+1136
-60
lines changed

‎README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ This documentation
1414
* OmniSci
1515
* PostgreSQL
1616
* illustrates the deployment in [Kubernetes](docs/Deployments.md)
17+
* illustrates the usage of [Monitoring](docs/Monitoring.md)
1718
* provides [more detailed examples](docs/Examples.md)
1819
* [Example: TPC-H Benchmark for 3 DBMS on 1 Virtual Machine](docs/Examples.md#example-tpc-h-benchmark-for-3-dbms-on-1-virtual-machine)
1920
* [Example: TPC-H Benchmark for 1 DBMS on 3 Virtual Machines](docs/Examples.md#example-tpc-h-benchmark-for-1-dbms-on-3-virtual-machines)

‎bexhoma/masterAWS.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -679,7 +679,12 @@ def runBenchmarks(self, connection=None, code=None, info='', resultfolder='', co
679679
self.benchmark.connections.append(c)
680680
self.benchmark.dbms[c['name']] = tools.dbms(c, False)
681681
# we must know all jars upfront
682-
tools.dbms.jars = [d['template']['JDBC']['jar'] for c,d in self.config['dockers'].items()]
682+
tools.dbms.jars = []
683+
for c,d in self.config['dockers'].items():
684+
if isinstance(d['template']['JDBC']['jar'], list):
685+
tools.dbms.jars.extend(d['template']['JDBC']['jar'])
686+
else:
687+
tools.dbms.jars.append(d['template']['JDBC']['jar'])
683688
# write appended connection config
684689
filename = self.benchmark.path+'/connections.config'
685690
with open(filename, 'w') as f:

‎bexhoma/masterK8s.py

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ def __init__(self, clusterconfig='cluster.config', configfolder='experiments/',
4545
configfile=f.read()
4646
self.config = eval(configfile)
4747
self.configfolder = configfolder
48+
self.resultfolder = self.config['benchmarker']['resultfolder']
4849
self.queryfile = queryfile
4950
self.clusterconfig = clusterconfig
5051
self.timeLoading = 0
@@ -59,6 +60,7 @@ def __init__(self, clusterconfig='cluster.config', configfolder='experiments/',
5960
self.workload = {}
6061
self.host = 'localhost'
6162
self.port = self.config['credentials']['k8s']['port']
63+
self.monitoring_active = True
6264
# k8s:
6365
self.namespace = self.config['credentials']['k8s']['namespace']
6466
self.appname = self.config['credentials']['k8s']['appname']
@@ -255,6 +257,7 @@ def generateDeployment(self):
255257
limit_cpu = cpu
256258
req_mem = mem
257259
limit_mem = mem
260+
req_gpu = gpu
258261
node_cpu = ''
259262
node_gpu = node
260263
# should be overwritten by resources dict?
@@ -276,6 +279,7 @@ def generateDeployment(self):
276279
self.resources['requests'] = {}
277280
self.resources['requests']['cpu'] = req_cpu
278281
self.resources['requests']['memory'] = req_mem
282+
self.resources['requests']['gpu'] = req_gpu
279283
self.resources['limits'] = {}
280284
self.resources['limits']['cpu'] = limit_cpu
281285
self.resources['limits']['memory'] = limit_mem
@@ -299,7 +303,7 @@ def generateDeployment(self):
299303
if not 'nodeSelector' in dep['spec']['template']['spec']:
300304
dep['spec']['template']['spec']['nodeSelector'] = {}
301305
dep['spec']['template']['spec']['nodeSelector']['gpu'] = node_gpu
302-
dep['spec']['template']['spec']['containers'][0]['resources']['limits']['nvidia.com/gpu'] = int(gpu)
306+
dep['spec']['template']['spec']['containers'][0]['resources']['limits']['nvidia.com/gpu'] = int(req_gpu)
303307
# add resource cpu
304308
#if node_cpu:
305309
if not 'nodeSelector' in dep['spec']['template']['spec']:
@@ -326,7 +330,7 @@ def deleteDeployment(self, deployment):
326330
self.kubectl('kubectl delete deployment '+deployment)
327331
def getDeployments(self):
328332
try:
329-
api_response = self.v1beta.list_namespaced_deployment(self.namespace, label_selector='app='+self.appname)
333+
api_response = self.v1apps.list_namespaced_deployment(self.namespace, label_selector='app='+self.appname)
330334
#pprint(api_response)
331335
if len(api_response.items) > 0:
332336
return [p.metadata.name for p in api_response.items]
@@ -514,9 +518,14 @@ def loadData(self):
514518
print("loadData")
515519
self.timeLoadingStart = default_timer()
516520
scriptfolder = '/data/{experiment}/{docker}/'.format(experiment=self.configfolder, docker=self.d)
521+
shellcommand = 'sh {scriptname}'
517522
commands = self.initscript
518523
for c in commands:
519-
self.executeCTL(self.docker['loadData'].format(scriptname=scriptfolder+c))
524+
filename, file_extension = os.path.splitext(c)
525+
if file_extension.lower() == '.sql':
526+
self.executeCTL(self.docker['loadData'].format(scriptname=scriptfolder+c))
527+
elif file_extension.lower() == '.sh':
528+
self.executeCTL(shellcommand.format(scriptname=scriptfolder+c))
520529
self.timeLoadingEnd = default_timer()
521530
self.timeLoading = self.timeLoadingEnd - self.timeLoadingStart
522531
def getMemory(self):
@@ -688,7 +697,7 @@ def runBenchmarks(self, connection=None, code=None, info=[], resultfolder='', co
688697
c['connectionmanagement']['timeout'] = self.connectionmanagement['timeout']
689698
c['connectionmanagement']['singleConnection'] = self.connectionmanagement['singleConnection'] if 'singleConnection' in self.connectionmanagement else False
690699
c['monitoring'] = {}
691-
if 'monitor' in self.config['credentials']['k8s']:
700+
if self.monitoring_active and 'monitor' in self.config['credentials']['k8s']:
692701
if 'grafanatoken' in self.config['credentials']['k8s']['monitor']:
693702
c['monitoring']['grafanatoken'] = self.config['credentials']['k8s']['monitor']['grafanatoken']
694703
if 'grafanaurl' in self.config['credentials']['k8s']['monitor']:
@@ -708,7 +717,7 @@ def runBenchmarks(self, connection=None, code=None, info=[], resultfolder='', co
708717
for metricname, metricdata in self.config['credentials']['k8s']['monitor']['metrics'].items():
709718
c['monitoring']['metrics'][metricname] = metricdata.copy()
710719
c['monitoring']['metrics'][metricname]['query'] = c['monitoring']['metrics'][metricname]['query'].format(host=node, gpuid=gpuid)
711-
c['JDBC']['url'] = c['JDBC']['url'].format(serverip='localhost', dbname=self.v)
720+
c['JDBC']['url'] = c['JDBC']['url'].format(serverip='localhost', dbname=self.v, DBNAME=self.v.upper())
712721
if code is not None:
713722
resultfolder += '/'+str(code)
714723
self.benchmark = benchmarker.benchmarker(
@@ -730,8 +739,15 @@ def runBenchmarks(self, connection=None, code=None, info=[], resultfolder='', co
730739
else:
731740
self.benchmark.connections.append(c)
732741
self.benchmark.dbms[c['name']] = tools.dbms(c, False)
733-
# we must know all jars upfront
734-
tools.dbms.jars = [d['template']['JDBC']['jar'] for c,d in self.config['dockers'].items()]
742+
# DEPRECATED: we must know all jars upfront
743+
"""
744+
tools.dbms.jars = []
745+
for c,d in self.config['dockers'].items():
746+
if isinstance(d['template']['JDBC']['jar'], list):
747+
tools.dbms.jars.extend(d['template']['JDBC']['jar'])
748+
else:
749+
tools.dbms.jars.append(d['template']['JDBC']['jar'])
750+
"""
735751
# write appended connection config
736752
filename = self.benchmark.path+'/connections.config'
737753
with open(filename, 'w') as f:

‎docs/Monitoring.md

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
# Monitoring
2+
3+
To include monitoring you will need
4+
* a Prometheus server scraping a fixed IP / Port
5+
* a Grafana server collecting metrics from the Prometheus server
6+
* some [configuration](#configuration) what metrics to collect
7+
8+
This document contains information about the
9+
* [Concept](#concept)
10+
* [Installation](#installation)
11+
* [Configuration](#configuration)
12+
13+
## Concept
14+
15+
<p align="center">
16+
<img src="architecture.png" width="640">
17+
</p>
18+
19+
There is
20+
* an Experiment Host - this needs Prometheus exporters
21+
* a Monitor - this needs a Prometheus server and a Grafana server scraping the Experiment Host
22+
* a Manager - this needs a configuration (which metrics to collect and where from)
23+
24+
## Installation
25+
26+
To be documented
27+
28+
### Kubernetes
29+
30+
* Experiment Host: Exporters are part of the [deployments](Deployments.md)
31+
* Monitor: Servers are deployed using Docker images, fixed on a separate monitoring instance
32+
* Manager: See [configuration](#configuration)
33+
34+
### AWS
35+
36+
* Experiment Host: Exporters are deployed using Docker images, fixed on the benchmarked instance
37+
* Monitor: Servers are deployed using Docker images, fixed on a separate monitoring instance
38+
* Manager: See [configuration](#configuration)
39+
40+
## Configuration
41+
42+
We insert information about
43+
* the Grafana server
44+
* access token
45+
* URL
46+
* the collection
47+
* extension of measure intervals
48+
* time shift
49+
* metrics definitions
50+
51+
into the cluster configuration.
52+
This is handed over to the [DBMS configuration](https://github.com/Beuth-Erdelt/DBMS-Benchmarker/blob/master/docs/Options.md#connection-file) of the [benchmarker](https://github.com/Beuth-Erdelt/DBMS-Benchmarker/blob/master/docs/Concept.md#monitoring-hardware-metrics).
53+
54+
### Example
55+
56+
The details of the metrics correspond to the YAML configuration of the [deployments](Deployments.md):
57+
* `job="monitor-node"`
58+
* `container_name="dbms"`
59+
60+
61+
```
62+
'monitor': {
63+
'grafanatoken': 'Bearer ABCDE==',
64+
'grafanaurl': 'http://localhost:3000/api/datasources/proxy/1/api/v1/',
65+
'grafanaextend': 20,
66+
'grafanashift': 0,
67+
'metrics': {
68+
'total_cpu_memory': {
69+
'query': 'container_memory_working_set_bytes{{job="monitor-node", container_label_io_kubernetes_container_name="dbms"}}',
70+
'title': 'CPU Memory [MiB]'
71+
},
72+
'total_cpu_memory_cached': {
73+
'query': 'container_memory_usage_bytes{{job="monitor-node", container_label_io_kubernetes_container_name="dbms"}}',
74+
'title': 'CPU Memory Cached [MiB]'
75+
},
76+
'total_cpu_util': {
77+
'query': 'sum(irate(container_cpu_usage_seconds_total{{job="monitor-node", container_label_io_kubernetes_container_name="dbms"}}[1m]))',
78+
'title': 'CPU Util [%]'
79+
},
80+
'total_cpu_throttled': {
81+
'query': 'sum(irate(container_cpu_cfs_throttled_seconds_total{{job="monitor-node", container_label_io_kubernetes_container_name="dbms"}}[1m]))',
82+
'title': 'CPU Throttle [%]'
83+
},
84+
'total_cpu_util_others': {
85+
'query': 'sum(irate(container_cpu_usage_seconds_total{{job="monitor-node", container_label_io_kubernetes_container_name!="dbms",id!="/"}}[1m]))',
86+
'title': 'CPU Util Others [%]'
87+
},
88+
'total_cpu_util_s': {
89+
'query': 'sum(container_cpu_usage_seconds_total{{job="monitor-node", container_label_io_kubernetes_container_name="dbms"}})',
90+
'title': 'CPU Util [s]'
91+
},
92+
'total_cpu_throttled_s': {
93+
'query': 'sum(container_cpu_cfs_throttled_seconds_total{{job="monitor-node", container_label_io_kubernetes_container_name="dbms"}})',
94+
'title': 'CPU Throttle [s]'
95+
},
96+
'total_cpu_util_others_s': {
97+
'query': 'sum(container_cpu_usage_seconds_total{{job="monitor-node", container_label_io_kubernetes_container_name!="dbms",id!="/"}})',
98+
'title': 'CPU Util Others [s]'
99+
},
100+
'total_network_rx': {
101+
'query': 'sum(container_network_receive_bytes_total{{container_label_app="dbmsbenchmarker", job="monitor-node"}})',
102+
'title': 'Net Rx [b]'
103+
},
104+
'total_network_tx': {
105+
'query': 'sum(container_network_transmit_bytes_total{{container_label_app="dbmsbenchmarker", job="monitor-node"}})',
106+
'title': 'Net Tx [b]'
107+
},
108+
'total_fs_read': {
109+
'query': 'sum(container_fs_reads_bytes_total{{job="monitor-node", container_label_io_kubernetes_container_name="dbms"}})',
110+
'title': 'FS Read [b]'
111+
},
112+
'total_fs_write': {
113+
'query': 'sum(container_fs_writes_bytes_total{{job="monitor-node", container_label_io_kubernetes_container_name="dbms"}})',
114+
'title': 'FS Write [b]'
115+
},
116+
'total_gpu_util': {
117+
'query': 'sum(DCGM_FI_DEV_GPU_UTIL{{UUID=~"{gpuid}"}})',
118+
'title': 'GPU Util [%]'
119+
},
120+
'total_gpu_power': {
121+
'query': 'sum(DCGM_FI_DEV_POWER_USAGE{{UUID=~"{gpuid}"}})',
122+
'title': 'GPU Power Usage [W]'
123+
},
124+
'total_gpu_memory': {
125+
'query': 'sum(DCGM_FI_DEV_FB_USED{{UUID=~"{gpuid}"}})',
126+
'title': 'GPU Memory [MiB]'
127+
},
128+
}
129+
}
130+
```
131+
132+
#### Fine Tuning
133+
134+
If the Grafana server has metrics coming from general Prometheus server, that is it scrapes more exporters than just the bexhoma related, we will need to specify further which metrics we are interested in.
135+
136+
There is a placeholder `{gpuid}` that is substituted automatically by a list of GPUs present in the pod.
137+

‎experiments/tpch/MariaDBCS/initdata-tpch-SF1.sh

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
cpimport tpch nation /data/tpch/SF1/nation.tbl
2+
cpimport tpch region /data/tpch/SF1/region.tbl
3+
cpimport tpch part /data/tpch/SF1/part.tbl
4+
cpimport tpch supplier /data/tpch/SF1/supplier.tbl
5+
cpimport tpch partsupp /data/tpch/SF1/partsupp.tbl
6+
cpimport tpch customer /data/tpch/SF1/customer.tbl
7+
cpimport tpch orders /data/tpch/SF1/orders.tbl
8+
cpimport tpch lineitem /data/tpch/SF1/lineitem.tbl

‎experiments/tpch/MariaDBCS/initschema-tpch.sql

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
CREATE DATABASE tpch;
2+
CREATE USER 'benchmark'@'%';
3+
GRANT ALL PRIVILEGES ON *.* TO 'benchmark'@'%' WITH GRANT OPTION;
4+
5+
-- sccsid: @(#)dss.ddl 2.1.8.1
6+
create table tpch.nation ( n_nationkey integer not null,
7+
n_name char(25) not null,
8+
n_regionkey integer not null,
9+
n_comment varchar(152)) engine=columnstore;
10+
11+
create table tpch.region ( r_regionkey integer not null,
12+
r_name char(25) not null,
13+
r_comment varchar(152)) engine=columnstore;
14+
15+
create table tpch.part ( p_partkey integer not null,
16+
p_name varchar(55) not null,
17+
p_mfgr char(25) not null,
18+
p_brand char(10) not null,
19+
p_type varchar(25) not null,
20+
p_size integer not null,
21+
p_container char(10) not null,
22+
p_retailprice decimal(15,2) not null,
23+
p_comment varchar(23) not null ) engine=columnstore;
24+
25+
create table tpch.supplier ( s_suppkey integer not null,
26+
s_name char(25) not null,
27+
s_address varchar(40) not null,
28+
s_nationkey integer not null,
29+
s_phone char(15) not null,
30+
s_acctbal decimal(15,2) not null,
31+
s_comment varchar(101) not null) engine=columnstore;
32+
33+
create table tpch.partsupp ( ps_partkey integer not null,
34+
ps_suppkey integer not null,
35+
ps_availqty integer not null,
36+
ps_supplycost decimal(15,2) not null,
37+
ps_comment varchar(199) not null ) engine=columnstore;
38+
39+
create table tpch.customer ( c_custkey integer not null,
40+
c_name varchar(25) not null,
41+
c_address varchar(40) not null,
42+
c_nationkey integer not null,
43+
c_phone char(15) not null,
44+
c_acctbal decimal(15,2) not null,
45+
c_mktsegment char(10) not null,
46+
c_comment varchar(117) not null) engine=columnstore;
47+
48+
create table tpch.orders ( o_orderkey integer not null,
49+
o_custkey integer not null,
50+
o_orderstatus char(1) not null,
51+
o_totalprice decimal(15,2) not null,
52+
o_orderdate date not null,
53+
o_orderpriority char(15) not null,
54+
o_clerk char(15) not null,
55+
o_shippriority integer not null,
56+
o_comment varchar(79) not null) engine=columnstore;
57+
58+
create table tpch.lineitem ( l_orderkey integer not null,
59+
l_partkey integer not null,
60+
l_suppkey integer not null,
61+
l_linenumber integer not null,
62+
l_quantity decimal(15,2) not null,
63+
l_extendedprice decimal(15,2) not null,
64+
l_discount decimal(15,2) not null,
65+
l_tax decimal(15,2) not null,
66+
l_returnflag char(1) not null,
67+
l_linestatus char(1) not null,
68+
l_shipdate date not null,
69+
l_commitdate date not null,
70+
l_receiptdate date not null,
71+
l_shipinstruct char(25) not null,
72+
l_shipmode char(10) not null,
73+
l_comment varchar(44) not null) engine=columnstore;
74+

0 commit comments

Comments
 (0)