-
Notifications
You must be signed in to change notification settings - Fork 364
/
benchmark.sh
executable file
·381 lines (333 loc) · 14.1 KB
/
benchmark.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
#!/bin/sh
#/*******************************************************************************
# * Copyright (c) 2020 Bosch.IO GmbH and others.
# *
# * All rights reserved. This program and the accompanying materials
# * are made available under the terms of the Eclipse Public License v2.0
# * and Eclipse Distribution License v1.0 which accompany this distribution.
# *
# * The Eclipse Public License is available at
# * http://www.eclipse.org/legal/epl-v20.html
# * and the Eclipse Distribution License is available at
# * http://www.eclipse.org/org/documents/edl-v10.html.
# *
# * Contributors:
# * Achim Kraus (Bosch.IO GmbH) - initial script
# ******************************************************************************/
# create benchmark statistic
echo "Californium CoAP Benchmark"
echo
echo "Requires a cf-extplugtest-server for exchanging messages."
echo
echo "Please check the available RAM (e.g.: on linux use \"free -m\") and"
echo "adjust the \"-Xmx6g\" argument in \"CF_OPT\" to about 50% of the available RAM."
echo "For newer JVMs the \"-XX:MaxRAMPercentage=50\" argument in \"CF_OPT\" may be used instead."
echo
echo "The required server may be started using:"
echo "java -Xmx6g -XX:+UseG1GC -jar cf-extplugtest-server-3.10.0.jar --no-external --no-plugtest"
echo "Adjust the \"-Xmx6g\" argument also to about 50% of the available RAM."
echo "For newer JVMs the \"-XX:MaxRAMPercentage=50\" argument in \"CF_OPT\" may also be used instead."
echo "If the benchmark is mainly used with the loopback interface (localhost), use the --no-external as above."
echo "To use client and server on different hosts, provide --no-loopback instead."
echo
echo "If the cf-extplugtest-server reports:"
echo
echo " Maxium heap size: ????M 84% used."
echo " Heap may exceed! Enlarge the maxium heap size."
echo " Or consider to reduce the value of EXCHANGE_LIFETIME"
echo " in \"CaliforniumReceivetest.properties\" or set"
echo " DEDUPLICATOR to NO_DEDUPLICATOR there."
echo
echo "you have a too fast CPU for the available amount of RAM :-)."
echo "Try to adjust the \"-Xmx\" to a larger value than the 50%."
echo
echo "Depending on your OS and configuration, the maximum number of sockets or threads may be limited."
echo
echo "Some cloud-provider use containers instead of real (or virtual) machines, which results in many"
echo "cases in lower limits. Check, if \"/proc/user_beancounters\" is available, and if so, check the"
echo "number of \"numproc\". That is mostly enough for servers, but for the benchmark client this limits"
echo "currently the number of clients to less than the half of the value of \"numproc\"."
echo
echo "For real (or virtual) machines with linux, the maximum number may be increased, if the host has"
echo "enough resources (RAM and CPU) to execute it. On Ubuntu 18.04, please adjust the values"
echo "\"DefaultLimitNOFILE\" in \"/etc/systemd/user.conf\" and \"/etc/systemd/system.conf\" accordingly"
echo "to the number of wanted sockets, and uncomment it by removing the leading \"#\"."
echo "For plain coap, currently more threads are required. Adjust \"UserTasksMax\" in"
echo "\"/etc/systemd/logind.conf\" to twice the number of sockets plus 500 more. With that, up to 10000"
echo "clients my be used for the benchmark. It's not recommended to use that many clients from one process"
echo "and it's even less recommended to use more than that!"
echo
echo "Variables:"
echo " USE_TCP, USE_UDP, USE_PLAIN, USE_SECURE, USE_CON, USE_NON"
echo " UDP_CLIENTS, TCP_CLIENTS, OBS_CLIENTS"
echo " PLAIN_PORT, SECURE_PORT"
echo " USE_REVERSE, USE_OBSERVE"
echo " REQS, NOTIFIES"
echo " PAYLOAD, PAYLOAD_LARGE, CALI_AUTH"
echo " USE_HTTP (coap2http proxy only!)"
echo
echo "These variables maybe override in the calling shell by"
echo
echo "export USE_TCP=0"
echo "export REQS=10"
echo
echo "Note: sometimes the recommended default configuration is changed."
echo " Please delete therefore the \"Californium???.properties\" to apply the changes."
echo
# commands to check several limits
# cat /proc/sys/kernel/threads-max
# cat /proc/sys/kernel/pid_max
# cat /proc/sys/vm/max_map_count
# prlimit
CF_JAR=cf-extplugtest-client-3.11.0.jar
CF_JAR_FIND="cf-extplugtest-client-*.jar"
CF_EXEC="org.eclipse.californium.extplugtests.BenchmarkClient"
#CF_OPT="-XX:+UseG1GC -Xmx6g -Xverify:none"
CF_OPT="-XX:+UseZGC -Xmx10g"
export CALIFORNIUM_STATISTIC="3.11.0"
if [ -z "$1" ] ; then
CF_HOST=localhost
else
CF_HOST=$1
shift
fi
if [ -z "$1" ] ; then
CLIENTS_MULTIPLIER=10
else
CLIENTS_MULTIPLIER=$1
shift
fi
if [ -z "$1" ] ; then
CF_SEC=
else
CF_SEC=$1
shift
fi
: "${PLAIN_PORT:=5783}"
: "${SECURE_PORT:=5784}"
: "${RESOURCE_PATH:=benchmark}"
# 0 := disable, 1 := enable
: "${USE_TCP:=1}"
: "${USE_UDP:=1}"
: "${USE_PLAIN:=1}"
: "${USE_SECURE:=1}"
: "${USE_CON:=3}" # 0:= not used, 1 := used with piggybacked response, 2 := with separate response, 3 := both variants
: "${USE_NON:=1}"
: "${USE_LARGE_BLOCK1:=1}"
: "${USE_REQUEST:=1}"
: "${USE_OBSERVE:=1}"
: "${USE_REVERSE:=1}"
: "${USE_REVERSE_OBSERVE:=1}"
: "${USE_HANDSHAKES:=1}"
: "${USE_PROXY:=0}"
: "${USE_HTTP:=0}"
: "${USE_NONESTOP:=--no-stop}"
# may be "misused" for other arguments, e.g. "--no-stop --handshakes-full=5" for full handshakes after 5 requests.
: "${USE_NSTART:=--nstart 1}"
# export EXECUTER_REMOVE_ON_CANCEL=true
# export EXECUTER_LOGGING_QUEUE_SIZE_DIFF=1000
# adjust the multiplier according the speed of your CPU
MULTIPLIER=10
: "${REQS:=$((1000 * $MULTIPLIER))}"
REQS_EXTRA=$(($REQS + ($REQS/10)))
REV_REQS=$((2 * $REQS))
: "${NOTIFIES:=$((350 * $MULTIPLIER))}"
: "${REV_NOTIFIES:=$NOTIFIES}"
: "${PAYLOAD:=40}"
: "${PAYLOAD_MEDIUM:=400}"
: "${PAYLOAD_LARGE:=5000}"
: "${REQS_LARGE:=$(($PAYLOAD * $REQS / $PAYLOAD_LARGE))}"
: "${UDP_CLIENTS:=$((200 * $CLIENTS_MULTIPLIER))}"
: "${TCP_CLIENTS:=$((100 * $CLIENTS_MULTIPLIER))}"
: "${OBS_CLIENTS:=$((100 * $CLIENTS_MULTIPLIER))}"
: "${CALI_AUTH:=--psk-store cali.psk}"
if [ ! -s "${CF_JAR}" ] ; then
echo "search for ${CF_JAR}"
# search for given version
CF_JAR_TEST=`find -name ${CF_JAR} | head -n 1`
if [ -z "${CF_JAR_TEST}" ] ; then
CF_JAR_TEST=`find .. -name ${CF_JAR} | head -n 1`
fi
if [ -n "${CF_JAR_TEST}" ] ; then
CF_JAR=${CF_JAR_TEST}
fi
fi
if [ ! -s "${CF_JAR}" ] ; then
echo "search for ${CF_JAR_FIND}"
# search for alternative available version
CF_JAR_TEST=`find -name "${CF_JAR_FIND}" ! -name "*sources.jar" | sort -r | head -n 1`
if [ -z "${CF_JAR_TEST}" ] ; then
CF_JAR_TEST=`find .. -name "${CF_JAR_FIND}" ! -name "*sources.jar" | sort -r | head -n 1`
fi
echo "found ${CF_JAR_TEST}"
if [ -n "${CF_JAR_TEST}" ] ; then
echo "Found different version."
echo "Using ${CF_JAR_TEST}"
echo "instead ${CF_JAR}"
CF_JAR=${CF_JAR_TEST}
fi
fi
if [ ! -s "${CF_JAR}" ] ; then
echo "Missing ${CF_JAR}"
exit -1
fi
echo ${CF_JAR}
benchmark_udp()
{
if [ ${USE_UDP} -eq 0 ] ; then return; fi
if [ ${USE_PLAIN} -ne 0 ] ; then
java ${CF_OPT} -cp ${CF_JAR} ${CF_EXEC} coap://${CF_HOST}:${PLAIN_PORT}/$@
if [ ! $? -eq 0 ] ; then exit $?; fi
sleep 5
fi
if [ ${USE_SECURE} -ne 0 ] ; then
java ${CF_OPT} -cp ${CF_JAR} ${CF_EXEC} ${CF_SEC} ${CALI_AUTH} coaps://${CF_HOST}:${SECURE_PORT}/$@
if [ ! $? -eq 0 ] ; then exit $?; fi
sleep 5
fi
}
benchmark_tcp()
{
if [ ${USE_TCP} -eq 0 ] ; then return; fi
if [ ${USE_PLAIN} -ne 0 ] ; then
java ${CF_OPT} -cp ${CF_JAR} ${CF_EXEC} coap+tcp://${CF_HOST}:${PLAIN_PORT}/$@
if [ ! $? -eq 0 ] ; then exit $?; fi
sleep 5
fi
if [ ${USE_SECURE} -ne 0 ] ; then
java ${CF_OPT} -cp ${CF_JAR} ${CF_EXEC} coaps+tcp://${CF_HOST}:${SECURE_PORT}/$@
if [ ! $? -eq 0 ] ; then exit $?; fi
sleep 5
fi
}
benchmark()
{
benchmark_udp $@
benchmark_tcp $@
}
benchmark_all()
{
# POST
if [ ${USE_REQUEST} -eq 1 ] ; then
# Large Payload
if [ ${USE_LARGE_BLOCK1} -ne 0 ] ; then
if [ ${USE_CON} -eq 1 ] || [ ${USE_CON} -eq 3 ] ; then
benchmark_udp "${RESOURCE_PATH}?rlen=${PAYLOAD}" --clients ${UDP_CLIENTS} --requests ${REQS_LARGE} ${USE_NONESTOP} --payload-random ${PAYLOAD_LARGE} --blocksize 64
fi
if [ ${USE_NON} -ne 0 ] ; then
benchmark_udp "${RESOURCE_PATH}?rlen=${PAYLOAD}" --clients ${UDP_CLIENTS} --non --requests ${REQS_LARGE} ${USE_NONESTOP} --payload-random ${PAYLOAD_LARGE} --blocksize 64
fi
benchmark_tcp "${RESOURCE_PATH}?rlen=${PAYLOAD}" --clients ${TCP_CLIENTS} --requests ${REQS_LARGE} ${USE_NONESTOP} --payload-random ${PAYLOAD_LARGE} --bertblocks 4
fi
# Small Payload
if [ ${USE_CON} -eq 1 ] || [ ${USE_CON} -eq 3 ] ; then
benchmark_udp "${RESOURCE_PATH}?rlen=${PAYLOAD}" --clients ${UDP_CLIENTS} --requests ${REQS} ${USE_NONESTOP} ${USE_NSTART} --payload-random ${PAYLOAD}
fi
if [ ${USE_NON} -ne 0 ] ; then
benchmark_udp "${RESOURCE_PATH}?rlen=${PAYLOAD}" --clients ${UDP_CLIENTS} --non --requests ${REQS} ${USE_NONESTOP} ${USE_NSTART} --payload-random ${PAYLOAD}
fi
benchmark_tcp "${RESOURCE_PATH}?rlen=${PAYLOAD}" --clients ${TCP_CLIENTS} --requests ${REQS} ${USE_NONESTOP} --payload-random ${PAYLOAD}
# POST separate response
if [ ${USE_CON} -eq 2 ] || [ ${USE_CON} -eq 3 ] ; then
benchmark_udp "${RESOURCE_PATH}?rlen=${PAYLOAD}&ack" --clients ${UDP_CLIENTS} --requests ${REQS} ${USE_NONESTOP} ${USE_NSTART} --payload-random ${PAYLOAD}
fi
fi
if [ ${USE_OBSERVE} -eq 1 ] ; then
benchmark_udp "${RESOURCE_PATH}?rlen=${PAYLOAD}" --clients ${OBS_CLIENTS} --requests 1 ${USE_NONESTOP} --notifies ${NOTIFIES} --reregister 25 --register 75
benchmark_udp "${RESOURCE_PATH}?rlen=${PAYLOAD}" --clients ${OBS_CLIENTS} --requests 1 ${USE_NONESTOP} --notifies ${NOTIFIES} --reregister 25 --register 75 --cancel-proactive
benchmark_tcp "${RESOURCE_PATH}?rlen=${PAYLOAD}" --clients ${OBS_CLIENTS} --requests 1 ${USE_NONESTOP} --notifies ${NOTIFIES} --reregister 25 --register 75
benchmark_tcp "${RESOURCE_PATH}?rlen=${PAYLOAD}" --clients ${OBS_CLIENTS} --requests 1 ${USE_NONESTOP} --notifies ${NOTIFIES} --reregister 25 --register 75 --cancel-proactive
fi
if [ ${USE_REVERSE} -eq 1 ] ; then
# reverse GET
if [ ${USE_CON} -ne 0 ] ; then
benchmark_udp "reverse-request?req=${REQS_EXTRA}&res=feed-CON&rlen=${PAYLOAD}" --clients ${UDP_CLIENTS} --requests 2 ${USE_NONESTOP} --reverse ${REV_REQS}
fi
if [ ${USE_NON} -ne 0 ] ; then
benchmark_udp "reverse-request?req=${REQS_EXTRA}&res=feed-CON&rlen=${PAYLOAD}" --clients ${UDP_CLIENTS} --non --requests 2 ${USE_NONESTOP} --reverse ${REV_REQS}
fi
benchmark_tcp "reverse-request?req=${REQS_EXTRA}&res=feed-CON&rlen=${PAYLOAD}" --clients ${TCP_CLIENTS} --requests 2 ${USE_NONESTOP} --reverse ${REV_REQS}
fi
if [ ${USE_REVERSE_OBSERVE} -eq 1 ] ; then
# reverse observe CON
if [ ${USE_CON} -ne 0 ] ; then
benchmark_udp "reverse-observe?obs=25000&res=feed-CON&rlen=${PAYLOAD_MEDIUM}" --clients ${OBS_CLIENTS} --requests 1 ${USE_NONESTOP} --reverse ${REV_NOTIFIES} --min -200 --max 200 --blocksize 64
fi
# reverse observe NON
if [ ${USE_CON} -ne 0 ] ; then
benchmark_udp "reverse-observe?obs=25000&res=feed-NON&rlen=${PAYLOAD_MEDIUM}" --clients ${OBS_CLIENTS} --requests 1 ${USE_NONESTOP} --reverse ${REV_NOTIFIES} --min -200 --max 200 --blocksize 64
fi
benchmark_tcp "reverse-observe?obs=25000&res=feed-CON&rlen=${PAYLOAD_MEDIUM}" --clients ${OBS_CLIENTS} --requests 1 ${USE_NONESTOP} --reverse ${REV_NOTIFIES} --min -200 --max 200 --blocksize 64
fi
}
benchmark_dtls_handshake()
{
if [ ${USE_UDP} -eq 0 ] ; then return; fi
if [ ${USE_SECURE} -ne 0 ] ; then
START_HS=`date +%s`
i=0
while [ $i -lt $1 ] ; do
java ${CF_OPT} -cp ${CF_JAR} ${CF_EXEC} $2 "coaps://${CF_HOST}:${SECURE_PORT}/${RESOURCE_PATH}?rlen=${PAYLOAD}" --clients ${UDP_CLIENTS} --requests 10 ${USE_NONESTOP}
if [ ! $? -eq 0 ] ; then exit $?; fi
sleep 2
i=$(($i + 1))
done
END_HS=`date +%s`
TIME=$((${END_HS} - ${START_HS}))
return $TIME
fi
}
benchmark_dtls_handshakes()
{
if [ ${USE_HANDSHAKES} -eq 0 ] || [ ${USE_UDP} -eq 0 ] || [ ${USE_SECURE} -eq 0 ] ; then return; fi
old=$CALIFORNIUM_STATISTIC
export CALIFORNIUM_STATISTIC=
LOOPS=10
benchmark_dtls_handshake $LOOPS
TIME1=$?
benchmark_dtls_handshake $LOOPS --auth=ECDHE_PSK
TIME2=$?
benchmark_dtls_handshake $LOOPS --auth=RPK
TIME3=$?
benchmark_dtls_handshake $LOOPS --auth=X509
TIME4=$?
echo "PSK :" $TIME1
echo "PSK/ECDHE:" $TIME2
echo "RPK :" $TIME3
echo "X509 :" $TIME4
export CALIFORNIUM_STATISTIC=$old
}
longterm()
{
# in seconds
LONG_INTERVAL_S=$((60 * 5))
LONG_INTERVAL_MS=$(($LONG_INTERVAL_S * 1000))
LONG_INTERVAL_TIMEOUT_S=$(($LONG_INTERVAL_S + 30))
# long term observe NON
benchmark_udp "reverse-observe?obs=2500000&res=feed-NON&timeout=${LONG_INTERVAL_TIMEOUT_S}&rlen=${PAYLOAD}" --clients ${UDP_CLIENTS} -requests --requests 1 ${USE_NONESTOP} --reverse ${NOTIFIES} --min ${LONG_INTERVAL_MS}
}
proxy()
{
if [ ${USE_PROXY} -eq 0 ] || [ ${USE_UDP} -eq 0 ] ; then return; fi
if [ ${USE_PLAIN} -ne 0 ] ; then
if [ ${USE_HTTP} -ne 0 ] ; then
java ${CF_OPT} -cp ${CF_JAR} ${CF_EXEC} "coap://${CF_HOST}:8000/http-target" --clients ${UDP_CLIENTS} --requests ${REQS} --proxy "localhost:5683:http"
if [ ! $? -eq 0 ] ; then exit $?; fi
sleep 5
fi
java ${CF_OPT} -cp ${CF_JAR} ${CF_EXEC} "coap://${CF_HOST}:${PLAIN_PORT}/${RESOURCE_PATH}?rlen=${PAYLOAD}" --clients ${UDP_CLIENTS} --requests ${REQS} --proxy "localhost:5683:coap"
if [ ! $? -eq 0 ] ; then exit $?; fi
sleep 5
fi
if [ ${USE_SECURE} -ne 0 ] ; then
java ${CF_OPT} -cp ${CF_JAR} ${CF_EXEC} "coap://${CF_HOST}:${SECURE_PORT}/${RESOURCE_PATH}?rlen=${PAYLOAD}" --clients ${UDP_CLIENTS} --requests ${REQS} --proxy "localhost:5683:coaps"
if [ ! $? -eq 0 ] ; then exit $?; fi
sleep 5
fi
}
START_BENCHMARK=$(date +%s)
proxy
benchmark_all
benchmark_dtls_handshakes
END_BENCHMARK=$(date +%s)
echo "ALL:" $((${END_BENCHMARK} - ${START_BENCHMARK}))