forked from patinnc/60secs
-
Notifications
You must be signed in to change notification settings - Fork 0
/
gen_xlsx.sh
executable file
·1926 lines (1875 loc) · 61.6 KB
/
gen_xlsx.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/bin/bash
#SCR_DIR=`dirname $(readlink -e $0)`
#SCR_DIR=`dirname $0`
#SCR_DIR=`dirname "$(readlink -f "$0")"`
SCR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
echo "SCR_DIR= $SCR_DIR" > /dev/stderr
declare -a REGEX
declare -a SKU
DIR=
PHASE_FILE=
OPT_OPT_DEF=chart_new,dont_sum_sockets
XLSX_FILE=
END_TM=
SKIP_XLS=0
NUM_DIR=0
NUM_DIR_BEG=
AVERAGE_END=
MAX_VAL=
TS_INIT=
VERBOSE=0
CLIP=
G_SUM=()
OPTIONS=
INPUT_FILE_LIST=
AVG_DIR=
DESC_FILE=
OSTYP=$OSTYPE
NUM_CPUS=0
FAIL=0
IFS_SV=$IFS
PY_PID=
BK_PID=()
BK_DIR=()
if [[ "$OSTYP" == "linux-gnu"* ]]; then
NUM_CPUS=`grep -c processor /proc/cpuinfo`
elif [[ "$OSTYP" == "darwin"* ]]; then
# Mac OSX
NUM_CPUS=`sysctl -a | grep machdep.cpu.thread_count | awk '{v=$2+0;printf("%d\n", v);exit;}'`
elif [[ "$OSTYP" == "cygwin" ]]; then
# POSIX compatibility layer and Linux environment emulation for Windows
NUM_CPUS=`grep -c processor /proc/cpuinfo`
elif [[ "$OSTYP" == "msys" ]]; then
# Lightweight shell and GNU utilities compiled for Windows (part of MinGW)
NUM_CPUS=`grep -c processor /proc/cpuinfo`
elif [[ "$OSTYP" == "win32" ]]; then
# I'm not sure this can happen.
NUM_CPUS=4
elif [[ "$OSTYP" == "freebsd"* ]]; then
# ...
NUM_CPUS=`grep -c processor /proc/cpuinfo`
else
# Unknown.
NUM_CPUS=4
fi
# on macbook could do sysctl -a | grep machdep.cpu.thread_count
BACKGROUND=$(($NUM_CPUS+2)) # setting this to 0 turns off launching sys_2_tsv.sh in the background
echo "$0 ${@}"
echo "BACKGROUND= $BACKGROUND NUM_CPUS= $NUM_CPUS"
JOB_ID=0
while getopts "hvASa:b:B:c:D:d:e:F:g:I:j:m:N:o:P:r:s:X:x:" opt; do
case ${opt} in
A )
AVERAGE=1
;;
S )
SKIP_XLS=$(($SKIP_XLS+1))
;;
v )
VERBOSE=$((VERBOSE+1))
;;
a )
AVG_DIR=$OPTARG
;;
b )
BEG_TM_IN=$OPTARG
;;
B )
BACKGROUND_IN=$OPTARG
;;
c )
CLIP=$OPTARG
;;
D )
DEBUG_OPT=$OPTARG
;;
d )
DIR=$OPTARG
echo "$0.$LINENO input DIR= $DIR"
;;
e )
END_TM_IN=$OPTARG
;;
F )
DESC_FILE=$OPTARG
;;
g )
G_SUM+=("$OPTARG")
;;
I )
INPUT_FILE_LIST=$OPTARG
;;
j )
JOB_ID=$OPTARG
;;
m )
MAX_VAL=$OPTARG
;;
N )
NUM_DIR_IN=$OPTARG
;;
o )
OPTIONS=$OPTARG
;;
P )
PHASE_FILE=$OPTARG
;;
s )
SKU+=($OPTARG)
;;
r )
REGEX+=($OPTARG)
;;
X )
AXLSX_FILE=$OPTARG
;;
x )
XLSX_FILE=$OPTARG
;;
h )
echo "$0 split data files into columns"
echo "Usage: $0 [-h] -d sys_data_dir [-v] [ -p prefix ]"
echo " -a avg_dir requires -A. Average tsv files will be put in this dir"
echo " -A flag indicating you want to average the same file from multiple dirs into 1 sheet."
echo " The default is to create 1 sheet per file per directory"
echo " -b begin_timestamp exclude data until this timestamp (UTC timestamp)"
echo " -B background_processs_allowed max background processes allowed. if 0 then no background processes. default is $BACKGROUND"
echo " -d dir containing sys_XX_* files created by 60secs.sh"
echo " -D debug_opt_strings used for debugging"
echo " -F desc_file file containing 1 line of text describing the results dir. Currently this is just the gen_xls.sh cmdline."
echo " this file can be used to identify breaks in the chart_sheet rows of charts. All charts with the same desc_file will be put be put on the same line"
echo " -g key=val key value pairs to be added to summary sheet. use multiple -g k=v options to specify multiple key value pairs"
echo " -I file_with_list_of_input_files used for getting a specify list of file proccessed"
echo " -j job_id if you are doing more than 1 dir and running jobs in background then this id is used to create unique input filenames for tsv_2_xlsx.py"
echo " -m max_val any value in chart > this value will be replaced by 0.0"
echo " -N number_of_dirs | beg_dir_num,end_dir_num if you have more than 1 directories then you can limit the num of dirs with this option. Default process all"
echo " if you enter beg_dir_num,end_dir_num then dirs numbering from beg_dir_num to end_dir_num are selected."
echo " -o options comma separated options."
echo " 'do_sum_sockets' if the perf stat data is per-socket then sum per-socket data to the system level"
echo " 'dont_sum_sockets' if the perf stat data is per-socket then don't sum per-socket data to the system level"
echo " 'line_for_scatter' substitute line charts for the scatter plots"
echo " 'drop_summary' don't add a sheet for each summary sheet (if you are doing more than 1 dir). Just do the sum_all sheet"
echo " 'chart_sheet' put all the charts on a separate sheet"
echo " 'all_charts_one_row' put all the charts for a workbook on one row"
echo " 'match_itp_muttley_interval' if you have itp/perf stat and muttley data, try to match the muttley interval to the perf interval"
echo " say the perf stat interval is 30 seconds and the muttley interval is 10 seconds. You might want to have the same number of rows of data"
echo " in the muttley tables as in perf stat data. So there are 3 muttley records for every 1 perf stat record. So only use the 3rd muttley record."
echo " this requires getting the perf stat interval from the run_itp.log file"
echo " 'pidstat_dont_add_pid' don't add the pid to process name. Allows better matching if doing multple servers"
echo " 'sum_file_no_formula' for non-combined summary sheets, don't use the excel formula to compute the average on the summary sheet"
echo " this can be useful if you are using compare_summary_table.sh to create a comparison of multiple summary sheets"
echo " 'get_max_val' when consolidating values for spreadsheet, don't get the avareage value, get the max value"
echo " 'get_perf_stat_max_val' when consolidating values for spreadsheet, don't get the avareage value, get the max value, this one is for perf_stat_scatter.awk"
echo " -P phase_file"
echo " -r regex regex expression to select directories"
echo " -S skip creating detail xlsx file, just do the summary all spreadsheet"
echo " -s sku_list select dirs with sku. have to be able to figure host from path and must have crane list of host info"
echo " -x xlsx_filename This is passed to tsv_2_xlsx.py as the name of the xlsx. (you need to add the .xlsx)"
echo " The default is chart_line.xlsx"
echo " -X xlsx_filename like above but assume path relative to current dir"
echo " -e ending_timestamp cut off data files at this timestamp"
echo " useful for runs that mess up before the expected end time"
echo " -v verbose mode"
exit
;;
: )
echo "$0.$LINENO Invalid option: $OPTARG requires an argument ${@}" 1>&2
exit
;;
\? )
echo "$0.$LINENO Invalid option: $OPTARG ${@}" 1>&2
exit
;;
esac
done
shift $((OPTIND -1))
remaining_args="$@"
ck_last_rc() {
local RC=$1
local FROM=$2
if [ $RC -gt 0 ]; then
echo "$0: got non-zero RC=$RC at $LINENO. called from line $FROM" > /dev/stderr
exit $RC
fi
}
if [ "$remaining_args" != "" ]; then
echo "$0.$LINENO remaining args= $remaining_args"
echo "$0.$LINENO got args leftover. Usually due to * in -d dir_name option"
exit
fi
if [ "$BACKGROUND_IN" != "" ]; then
BACKGROUND=$BACKGROUND_IN
fi
SUM_ALL_AVG_BY_METRIC=
if [ "$OPTIONS" != "" ]; then
lkfor="sum_all_avg_by_metric{"
if [[ $OPTIONS == *"$lkfor"* ]]; then
rest=${OPTIONS#*$lkfor}
echo $(( ${#OPTIONS} - ${#rest} - ${#lkfor} ))
echo "$0.$LINENO got $lkfor, rest= $rest"
lkfor="}"
#rest2=${rest#*$lkfor}
pfx=${rest%%$lkfor*}
echo "$0.$LINENO options: sum_all_avg_by_metric=\"${pfx}\""
SUM_ALL_AVG_BY_METRIC="$pfx"
fi
fi
if [ "$NUM_DIR_IN" != "" ]; then
NUM_DIR_ARR=()
IFS=',' read -r -a NUM_DIR_ARR <<< "$NUM_DIR_IN"
IFS=$IFS_SV
echo "NUM_DIR_ARR= ${NUM_DIR_ARR[@]}, BM0= ${NUM_DIR_ARR[0]}, BM1=${NUM_DIR_ARR[1]}" > /dev/stderr
NUM_DIR_BEG=${NUM_DIR_ARR[0]}
NUM_DIR_END=${NUM_DIR_ARR[1]}
if [ "$NUM_DIR_END" == "" ]; then
# only 1 value entered. Treat it as a 'read this many files'
NUM_DIR=$NUM_DIR_BEG
NUM_DIR_BEG=
fi
echo "NUM_DIR_ARR= ${NUM_DIR_ARR[@]}, ND0= ${NUM_DIR_ARR[0]}, ND1=${NUM_DIR_ARR[1]} NUM_DIR_BEG= $NUM_DIR_BEG NUM_DIR_END= $NUM_DIR_END" > /dev/stderr
fi
REGEX_LEN=${#REGEX[@]}
SKU_LEN=${#SKU[@]}
SKIP_SYS_2_TSV=0
if [ "$OPTIONS" != "" ]; then
if [[ $OPTIONS == *"skip_sys_2_tsv"* ]]; then
SKIP_SYS_2_TSV=1
fi
fi
INPUT_DIR=$DIR
if [ $VERBOSE -gt 0 ]; then
echo "$0.$LINENO SKIP_XLS= $SKIP_XLS"
fi
get_abs_filename() {
# $1 : relative filename
echo "$(cd "$(dirname "$1")" && pwd)/$(basename "$1")"
}
export AWKPATH=$SCR_DIR
get_hostname_from_path() {
if [ "$1" != "" ]; then
USEDIR=$1
else
USEDIR=`pwd`
fi
HOSTNM=`awk -v script="$0" -v lineno="$LINENO" -v usedir="$USEDIR" '
BEGIN{
n = split(usedir, arr, "/");
for (i=n; i > 2; i--) {
if (arr[i] == arr[i-2] && index(arr[i-1], "-") > 0) {
printf("%s\n", arr[i-1]);
exit;
}
}
printf("%s.%s: missed_hostnm in %s\n", script, lineno, usedir) > "/dev/stderr";
exit 1;
}'`
ck_last_rc $? $LINENO
}
get_grail_info_for_hostname() {
if [ "$1" != "" ]; then
UHOSTNM=$1
else
UHOSTNM=$HOSTNM
fi
if [ "$2" != "" ]; then
USUM_FILE=$2
fi
if [ "$UHOSTNM" != "" ]; then
CKFL=grail_cpu_info.txt
if [ ! -e $CKFL ]; then
CKFL=$BASE_DIR/grail_cpu_info.txt
fi
if [ $VERBOSE -gt 0 ]; then
echo "_____ck grail file $CKFL" > /dev/stderr
fi
if [ -e $CKFL ]; then
if [ $VERBOSE -gt 0 ]; then
echo "_____got grail hst= $UHOSTNM file $CKFL" > /dev/stderr
fi
#awk -v hst="$UHOSTNM" 'BEGIN{FS=";";} $1 == hst {printf("%s\n", $0); exit;}'
SKU_NCPU_CPU_BOX_DISK=(`awk -v hst="$UHOSTNM" -v FS=";" '
$1 == hst {
n=split($0,a,";");
sku =a[2];
cpus =a[3]+0;
if (cpus == 0) { cpus = ""; }
brand =a[4];
model =a[5];
diskTB=a[6]+0;
if (diskTB == 0) {diskTB = ""; }
owner =a[7];
printf("\"%s\"\n%s\n\"%s\"\n\"%s\"\n%s\n\"%s\"\n", sku, cpus, brand, model,diskTB, owner); exit 0;}
' $CKFL`)
ck_last_rc $? $LINENO
#echo "_____got grail sku= ${SKU_NCPU_CPU_BOX_DISK[@]}" > /dev/stderr
if [ "$USUM_FILE" != "" ]; then
V=${SKU_NCPU_CPU_BOX_DISK[0]}
if [ "$V" == "" ]; then
V="unknown"
fi
#printf "host\tSKU\t\"%s\"\tSKU\n" "$V" >> $USUM_FILE;
printf "host\tSKU\t%s\tSKU\n" $V >> $USUM_FILE;
V=${SKU_NCPU_CPU_BOX_DISK[4]}
if [ "$V" == "" ]; then
V=0
fi
printf "host\tdisk_GBs\t%s\tdisk_GBs\n" "$V" >> $USUM_FILE;
V=${SKU_NCPU_CPU_BOX_DISK[3]}
if [ "$V" == "" ]; then
V="unknown"
fi
#printf "host\thost_make\t\"%s\"\thost_make\n" "$V" >> $USUM_FILE;
printf "host\thost_make\t%s\thost_make\n" "$V" >> $USUM_FILE;
V=${SKU_NCPU_CPU_BOX_DISK[2]}
if [ "$V" == "" ]; then
V="unknown"
fi
#printf "host\tcpu_string\t\"%s\"\tcpu_string\n" "$V" >> $USUM_FILE;
printf "host\tcpu_string\t%s\tcpu_string\n" "$V" >> $USUM_FILE;
V=${SKU_NCPU_CPU_BOX_DISK[5]}
if [ "$V" == "" ]; then
V="unknown"
fi
#printf "host\towner\t\"%s\"\towner\n" "$V" >> $USUM_FILE;
printf "host\towner\t%s\towner\n" "$V" >> $USUM_FILE;
fi
fi
fi
}
get_dir_list() {
local CKF=$1
DIR=$2
local RESP
echo "$0.$LINENO get_dir_list: found $RESP $CKF file(s) under dir $DIR. Using the dir of first one if more than one."
#RESP=`find $DIR -name $CKF -print0 | sort -z | xargs -0 cat`
if [ "$REGEX_LEN" != "0" ]; then
RESP2=`find $DIR -name $CKF -print | sort`
RESP=$RESP2
local ii
for ii in ${REGEX[@]}; do
RESP=`echo -e "$RESP" | grep "$ii"`
done
mydir=`echo -e "$RESP" | wc -l`
echo "$0.$LINENO mydir count= $mydir"
else
RESP=`find $DIR -name $CKF -print | sort | xargs `
fi
echo "$0.$LINENO get_dir_list: found $CKF file in dir $DIR:"
local STR=
local j=0
for ii in $RESP; do
NM=$(dirname $ii)
j=$((j+1))
if [ "$NUM_DIR" != "" -a "$NUM_DIR" != "0" -a $NUM_DIR -gt 0 -a $j -ge $NUM_DIR ]; then
echo "$0.$LINENO job_id= $JOB_ID limit number of dirs with $CKF due to -N $NUM_DIR option"
break
fi
if [ "$NUM_DIR_BEG" != "" ]; then
if [ "$j" -lt "$NUM_DIR_BEG" ]; then
echo "$0.$LINENO job_id= $JOB_ID skip dir $j due to -N $NUM_DIR_IN option" > /dev/stderr
continue
fi
fi
if [ "$NUM_DIR_END" != "" ]; then
if [ "$j" -gt "$NUM_DIR_END" ]; then
echo "$0.$LINENO job_id= $JOB_ID skip dir $j due to -N $NUM_DIR_IN option" > /dev/stderr
continue
fi
fi
STR="$STR $NM"
done
DIR=$STR
echo "$0.$LINENO +___________-- get_dir_list: j= $j DIR= $DIR" > /dev/stderr
#exit 1
}
OPT_a=
if [ "$AVG_DIR" != "" ]; then
if [ "$AVERAGE" == "0" ]; then
echo "$0: cmdline options has -a $AVG_DIR but you didn't specify -A option. Bye" > /dev/stderr
exit
fi
if [ -d $AVG_DIR ]; then
mkdir -p "$AVG_DIR"
fi
AVG_DIR=$(get_abs_filename "$AVG_DIR")
OPT_a=" -a $AVG_DIR "
fi
if [ "$DESC_FILE" != "" ]; then
if [ $VERBOSE -gt 0 ]; then
echo "$0.$LINENO DESC_FILE= $DESC_FILE" > /dev/stderr
fi
DESC_FILE=$(get_abs_filename "$DESC_FILE")
if [ $VERBOSE -gt 0 ]; then
echo "$0.$LINENO DESC_FILE= $DESC_FILE" > /dev/stderr
fi
fi
if [ "$INPUT_FILE_LIST" != "" ]; then
if [ -e $INPUT_FILE_LIST ]; then
echo "$0.$LINENO got input_file_list= $INPUT_FILE_LIST"
fi
else
DIR_ORIG=$DIR
CKF=60secs.log
GOT_DIR=0
RESP=`find $DIR -name $CKF | wc -l | awk '{$1=$1;print}'`
if [ $VERBOSE -gt 0 ]; then
echo "$0.$LINENO RESP= 60secs.log = $RESP"
fi
if [ $RESP -gt 0 ]; then
get_dir_list $CKF $DIR
echo "$0.$LINENO found $RESP $CKF file(s) under dir $DIR. Using the dir of first one if more than one."
GOT_DIR=1
fi
if [ $RESP -eq 0 ]; then
if [ $VERBOSE -gt 0 ]; then
echo "$0.$LINENO didn't find 60secs.log file under dir $DIR. Bye"
fi
CKF="metric_out"
RESP=`find $DIR -name $CKF | wc -l | awk '{$1=$1;print}'`
if [ "$RESP" == "0" ]; then
CKF="metric_out.tsv"
RESP=`find $DIR -name $CKF | wc -l | awk '{$1=$1;print}'`
#echo "found $RESP $CKF file(s) under dir $DIR. Using the dir of first one if more than one."
fi
if [ "$RESP" == "0" ]; then
CKF="metric_out.csv"
RESP=`find $DIR -name $CKF | wc -l | awk '{$1=$1;print}'`
#echo "found $RESP $CKF file(s) under dir $DIR. Using the dir of first one if more than one."
fi
if [ "$RESP" == "0" ]; then
CKF="metric_out.csv.tsv"
RESP=`find $DIR -name $CKF | wc -l | awk '{$1=$1;print}'`
fi
if [ "$RESP" != "0" ]; then
echo "$0.$LINENO found $RESP $CKF file(s) under dir $DIR. Using the dir of first one if more than one."
get_dir_list $CKF $DIR
GOT_DIR=1
echo "$0.$LINENO using1 DIR= $DIR, orig DIR= $DIR_ORIG"
fi
if [ "$GOT_DIR" == "0" ]; then
CKF="sys_*_perf_stat.txt"
RESP=`find $DIR -name "$CKF" | wc -l | awk '{$1=$1;print}'`
if [ $VERBOSE -gt 0 ]; then
echo "$0.$LINENO got29 $RESP $CKF file(s) under dir $DIR. Using the dir of first one if more than one." > /dev/stderr
fi
if [ "$RESP" == "0" ]; then
CKF="sys_*_perf_stat.txt*"
RESP=`find $DIR -name "$CKF" | wc -l | awk '{$1=$1;print}'`
if [ $VERBOSE -gt 0 ]; then
echo "$0.$LINENO got $RESP $CKF file(s) under dir $DIR. Using the dir of first one if more than one." > /dev/stderr
fi
fi
if [ "$RESP" != "0" ]; then
if [ $VERBOSE -gt 0 ]; then
echo "$0.$LINENO found $RESP $CKF file(s) under dir $DIR. Using the dir of first one if more than one." > /dev/stderr
fi
RESP=`find $DIR -name "$CKF"|sort`
if [ "$REGEX_LEN" != "0" ]; then
RESP=`find $DIR -name "$CKF" | sort`
for ii in ${REGEX[@]}; do
RESP=`echo -e "$RESP" | grep "$ii"`
done
RESP3=`echo "$RESP" | wc -l`
if [ $VERBOSE -gt 0 ]; then
echo "$0.$LINENO mydir count= $mydir, resplines= $RESP3"
fi
fi
if [ $VERBOSE -gt 0 ]; then
echo "$0.$LINENO found51 $CKF file in dir $DIR"
fi
STR=
j=0
GOT_TS_INIT=0
for ii in $RESP; do
NM=$(dirname $ii)
NUM_LINES=`head -10 $ii | wc -l |awk '{$1=$1;printf("%d\n", $1);}'`
if [ $NUM_LINES -lt 10 ]; then
# the sys_*_perf_stat.txt file can get created with just one line in it if perf isn't present
# so don't include dirs with empty data
if [ -e $NM/run.log ]; then
#20201203_201403 1607055243.064233658 start -p /mnt/root/output/onprem_rome_combo_32cpus_v10 -I 0 -i 1 -d 7200
#20201203_204129 1607056889.971113020 end elapsed_secs 1646.906879
TS_INIT=`awk '/start/{if ($3 == "start") {print $2;exit;}}' $NM/run.log`
if [ "$TS_INIT" != "" ]; then
GOT_TS_INIT=1
fi
echo "$0.$LINENO: ck $NM/run.log got TS_INIT= $TS_INIT"
fi
if [ "$GOT_TS_INIT" != "1" ]; then
if [ $VERBOSE -gt 0 ]; then
echo "$0.$LINENO skip $CKF file in dir $DIR due to too few lines= $NUM_LINES"
fi
continue
fi
fi
if [ $GOT_TS_INIT == 0 -a $j -eq 0 ]; then
TS_INIT=`awk '
function dt_to_epoch(offset) {
if (date_str == "") {
return 0.0;
}
months=" JanFebMarAprMayJunJulAugSepOctNovDec";
n=split(date_str, darr, /[ :]+/);
mnth_num = sprintf("%d", index(months, darr[1])/3);
dt_str = darr[6] " " mnth_num " " darr[2] " " darr[3] " " darr[4] " " darr[5];
epoch = mktime(dt_str);
return epoch + offset;
}
/epoch\tts\trel_ts\tinterval/ {
getline;
ts=$1 - $2;
printf("%f\n", ts);
exit;
}
/^# started on / {
# started on Fri Jun 12 14:36:31 UTC 2020 1591972591.618156223
# started on Fri Jun 12 14:36:31 2020
pos = index($0, " on ")+8;
date_str = substr($0, pos);
if ($8 == "UTC") {
if (NF == 10) {
ts_initial = $10+0;
printf("%s\n", $10);
exit;
}
date_str = $5 " " $6 " " $7 " " $9;
}
tst_epoch = dt_to_epoch(0.0);
printf("%s\n", tst_epoch);
exit
}' $ii`
#TS_INIT=
if [ $VERBOSE -gt 0 ]; then
echo "$0.$LINENO first sys_*_perf_stat.txt TS_INIT= $TS_INIT" > /dev/stderr
echo "$0.$LINENO first sys_*_perf_stat.txt TS_INIT= $TS_INIT"
fi
fi
if [ "$NUM_DIR_BEG" != "" ]; then
if [ "$j" -lt "$NUM_DIR_BEG" ]; then
#echo "$0.$LINENO job_id= $JOB_ID skip dir $j due to -N $NUM_DIR_IN option" > /dev/stderr
j=$((j+1))
continue
fi
fi
if [ "$NUM_DIR_END" != "" ]; then
if [ "$j" -gt "$NUM_DIR_END" ]; then
#echo "$0.$LINENO job_id= $JOB_ID skip dir $j due to -N $NUM_DIR_IN option" > /dev/stderr
j=$((j+1))
continue
fi
fi
j=$((j+1))
STR="$STR $NM"
if [ "$NUM_DIR" != "" -a $NUM_DIR -gt 0 -a $j -ge $NUM_DIR ]; then
echo "$0.$LINENO job_id= $JOB_ID limit number of dirs due to -N $NUM_DIR option"
break
fi
done
DIR=$STR
#echo "$0.$LINENO: -__________- using2 DIR= $DIR, orig DIR= $DIR_ORIG" > /dev/stderr
else
echo "$0.$LINENO: didn't find 60secs.log nor metric_out nor sys_*_perf_stat.txt file under dir $DIR at line $LINENO Bye"
exit
fi
fi
else
if [ "$GOT_DIR" == "0" ]; then
echo "$0.$LINENO found54 $RESP 60secs.log file(s) under dir $DIR. Using the dir of first one if more than one."
RESP=`find $DIR -name 60secs.log | head -1`
echo "$0.$LINENO found 60secs.log file in dir $RESP"
DIR=$(dirname $RESP)
echo "$0.$LINENO using3 DIR= $DIR, orig DIR= $DIR_ORIG"
fi
fi
fi
LST=$DIR
if [ $VERBOSE -gt 0 ]; then
echo "$0.$LINENO DIR: $DIR"
fi
#exit
if [ "$SKU_LEN" != "0" ]; then
echo "$0.$LINENO SKU= ${SKU[@]}"
if [ "$SKU_LEN" != "0" ]; then
itot=0
ii=0
RESP=
for idir in $DIR; do
itot=$((itot+1))
if [ $VERBOSE -gt 1 ]; then
echo "__________sku try dir= $idir"
fi
get_hostname_from_path $idir
if [ "$HOSTNM" != "" ]; then
get_grail_info_for_hostname $HOSTNM
if [ "${#SKU_NCPU_CPU_BOX_DISK[@]}" != 0 ]; then
V=${SKU_NCPU_CPU_BOX_DISK[0]}
if [ $VERBOSE -gt 0 ]; then
echo "__________HOSTNM= $HOSTNM , sku= $V"
fi
for ij in ${SKU[@]}; do
if [ "$V" == "\"$ij\"" ]; then
if [ $VERBOSE -gt 0 ]; then
echo "__match __HOSTNM= $HOSTNM , sku= $V"
fi
RESP="$RESP $idir"
ii=$((ii+1))
break
fi
done
fi
fi
done
DIR=$RESP
LST=$DIR
echo "+__________$0.$LINENO input dirs= $itot. got matches= $ii for skus= ${SKU[@]}" > /dev/stderr
fi
fi
CDIR=`pwd`
ALST=$CDIR/tsv_2_xlsx_${JOB_ID}.inp
#echo "ALST= $ALST"
if [ -e $ALST ]; then
rm $ALST
fi
OXLS=tmp.xlsx
if [ "$AXLSX_FILE" != "" ]; then
OXLS=${AXLSX_FILE}_all.xlsx
fi
shopt -s nullglob
echo -e "-o\t$OXLS" >> $ALST
FCTRS=
SVGS=
SUM_FILE=sum.tsv
if [ $VERBOSE -gt 0 ]; then
echo "$0.$LINENO LST= $LST" > /dev/stderr
fi
DIR_1ST_DIR=
if [ "$INPUT_FILE_LIST" == "" ]; then
NUM_DIRS=0
for i in $LST; do
if [ $NUM_DIRS -eq 0 ]; then
DIR_1ST_DIR=$i
fi
#echo "$0.$LINENO dir_num= $NUM_DIRS dir= $i"
NUM_DIRS=$((NUM_DIRS+1))
done
fi
TS_BEG=`date +%s`
if [ "$INPUT_FILE_LIST" == "" ]; then
oIFS=$IFS
DIR_NUM_MX=0
for i in $LST; do
DIR_NUM_MX=$(($DIR_NUM_MX+1))
done
DIR_NUM=0
for i in $LST; do
OPT_DESC_FILE=
if [ "$DESC_FILE" == "" ]; then
if [ -e $i/desc.txt ]; then
FLS=$(get_abs_filename "$i/desc.txt")
if [ -e $FLS ]; then
OPT_DESC_FILE="$FLS"
fi
fi
fi
if [ $VERBOSE -gt 0 ]; then
pushd $i
else
pushd $i > /dev/null
fi
pd=$(pwd)
str=`echo "${pd##*/}"`
if [ "$str" != "60secs" ]; then
XLS=$str
fi
if [ $VERBOSE -gt 0 ]; then
echo "$0.$LINENO XLS= $XLS" > /dev/stderr
fi
RPS=`echo $i | sed 's/rps_v/rpsv/' | sed 's/rps.*_.*/rps/' | sed 's/.*_//' | sed 's/\/.*//'`
RPS="${RPS}"
if [ "$RPS" == "" ]; then
RPS="1rps"
fi
if [ "$SUM_FILE" != "" ]; then
if [ -e $SUM_FILE ]; then
rm $SUM_FILE
fi
if [ -e "$SUM_FILE.dist" ]; then
rm "$SUM_FILE.dist"
fi
fi
FCTR=`echo $RPS | sed 's/rps//'`
FCTR=`awk -v fctr="$FCTR" 'BEGIN{fctr += 0.0; mby=1.0; if (fctr >= 100.0) {mby=0.001;} if (fctr == 0.0) {fctr=1.0;mby=1.0;} printf("%.3f\n", mby*fctr); exit;}'`
if [ $VERBOSE -gt 0 ]; then
echo "$0.$LINENO rps= $RPS, fctr= $FCTR"
fi
if [ "$XLSX_FILE" != "" ]; then
XLS=$XLSX_FILE
fi
if [ "$AXLSX_FILE" != "" ]; then
XLS=$CDIR/$AXLSX_FILE
fi
OPT_PH=
if [ "$PHASE_FILE" != "" ]; then
OPT_PH=" -P $PHASE_FILE "
fi
OPT_BEG_TM=
if [ "$TS_INIT" != "" ]; then
OPT_BEG_TM=" -b $TS_INIT "
fi
if [ "$BEG_TM_IN" != "" ]; then
OPT_BEG_TM=" -b $BEG_TM_IN "
echo "$0.$LINENO: BEG_TM= $BEG_TM_IN"
echo "$0.$LINENO ____got BEG_TM_IN=\"$BEG_TM_IN\"" > /dev/stderr
fi
OPT_END_TM=
if [ "$END_TM_IN" != "" ]; then
OPT_END_TM=" -e $END_TM_IN "
fi
OPT_SKIP=
if [ "$SKIP_XLS" -gt "0" ]; then
OPT_SKIP=" -S "
fi
OPT_M=
if [ "$MAX_VAL" != "" ]; then
OPT_M=" -m $MAX_VAL "
fi
OPT_CLIP=
if [ "$CLIP" != "" ]; then
OPT_CLIP=" -c $CLIP "
fi
OPT_DEBUG=
if [ "$DEBUG_OPT" != "" ]; then
OPT_DEBUG=" -D $DEBUG_OPT "
fi
OPT_OPT=
if [ "$OPTIONS" != "" ]; then
OPT_OPT="$OPTIONS "
else
OPT_OPT=$OPT_OPT_DEF
fi
OPT_A=
if [ "$AVERAGE" != "0" ]; then
if [ $NUM_DIRS -gt 1 ]; then
OPT_A=" -A "
fi
fi
OPT_G=
if [ "$G_SUM" != "" ]; then
for g in "${G_SUM[@]}"; do
OPT_G+=" -g $g "
#echo "$0.$LINENO build g opt= $OPT_G"
done
echo "$0.$LINENO new g opt= $OPT_G"
fi
if [ -e $CDIR/job_${JOB_ID}.stop ]; then
RESP=`head -1 $CDIR/job_${JOB_ID}.stop`
echo "$0: got job_$JOB_ID.stop pid= $RESP and bashpid= $$" > /dev/stderr
if [ "$RESP" == "$$" ]; then
echo "$0: quitting at line $LINENO due to job_$JOB_ID.stop having value PID= $$"
exit 1
fi
fi
if [ "$SKIP_SYS_2_TSV" == "0" ]; then
SYS_2_TSV_STDOUT_FILE=tmp.jnk
if [ $VERBOSE -gt 0 ]; then
echo "$SCR_DIR/sys_2_tsv.sh -B $CDIR $OPT_a $OPT_A $OPT_G -j $JOB_ID -p \"$RPS\" $OPT_DEBUG $OPT_SKIP $OPT_M -d . $OPT_CLIP $OPT_BEG_TM $OPT_END_TM -i \"*.png\" -s $SUM_FILE -x $XLS.xlsx -o $OPT_OPT $OPT_PH -t $DIR &> $SYS_2_TSV_STDOUT_FILE" &
fi
if [ "$BACKGROUND" -le "0" ]; then
$SCR_DIR/sys_2_tsv.sh -B $CDIR $OPT_a $OPT_A $OPT_G -j $JOB_ID -p "$RPS" $OPT_DEBUG $OPT_SKIP $OPT_M -d . $OPT_CLIP $OPT_BEG_TM $OPT_END_TM -i "*.png" -s $SUM_FILE -x $XLS.xlsx -o $OPT_OPT $OPT_PH -t $DIR &> $SYS_2_TSV_STDOUT_FILE
RC=$?
ck_last_rc $RC $LINENO
else
$SCR_DIR/sys_2_tsv.sh -B $CDIR $OPT_a $OPT_A $OPT_G -j $JOB_ID -p "$RPS" $OPT_DEBUG $OPT_SKIP $OPT_M -d . $OPT_CLIP $OPT_BEG_TM $OPT_END_TM -i "*.png" -s $SUM_FILE -x $XLS.xlsx -o $OPT_OPT $OPT_PH -t $DIR &> $SYS_2_TSV_STDOUT_FILE &
LPID=$!
RC=$?
BK_DIR[$LPID]=$i
BK_OUT[$LPID]=$SYS_2_TSV_STDOUT_FILE
if [ $VERBOSE -gt 0 ]; then
echo "$0.$LINENO LPID= $LPID, RC= $RC"
fi
fi
LOAD=`uptime | awk '{printf("%.0f\n", $(NF-2)+0.5);}'`
jbs=0
for job in `jobs -p`
do
#echo $job
jbs=$((jbs+1))
done
#echo "$0.$LINENO job_id= $JOB_ID jbs= $jbs LOAD= $LOAD BACKGROUND= $BACKGROUND" > /dev/stderr
jbs=$(($jbs+$LOAD))
if [ "$jbs" -gt "$BACKGROUND" ]; then
#jbs=0
for job in `jobs -p`
do
TS_CUR=`date +%s`
TS_DFF=$(($TS_CUR-$TS_BEG))
echo "$0.$LINENO: job_id= $JOB_ID wait for jobs (jbs= $jbs) pid= $job, dir_num= $DIR_NUM of $DIR_NUM_MX, elap_secs= $TS_DFF, load= $LOAD"
jbs=$((jbs-1))
wait $job
RC=$?
if [ "$RC" != "0" ]; then
echo "$0: sys_2_tsv.sh got error! bye at $LINENO" > /dev/stderr
echo "$0: look at ${BK_OUT[$LPID]} file in last data dir for error messages" > /dev/stderr
echo "$0: dir= ${BK_DIR[$job]}"
exit 1
fi
if [ "$jbs" -lt "$BACKGROUND" ]; then
break
fi
done
fi
fi
TS_CUR=`date +%s`
TS_DFF=$(($TS_CUR-$TS_BEG))
if [ $VERBOSE -gt 0 ]; then
echo "$0.$LINENO FLS: job_id= $JOB_ID dir_num= ${DIR_NUM} of ${DIR_NUM_MX}, elap_tm= $TS_DFF secs, ${FLS}" > /dev/stderr
fi
DIR_NUM=$(($DIR_NUM+1))
if [ $VERBOSE -gt 0 ]; then
popd
else
popd > /dev/null
fi
done
wait_for_all() {
jbs=0
for job in `jobs -p`
do
echo "$0.$LINENO wait for jobs (jbs= $jbs) pid= $job"
wait $job
RC=$?
#if [ "$RC" == "1" -o "$RC" == "2" ]; then
if [ "$RC" != "0" ]; then
echo "$0: sys_2_tsv.sh got error RC= \"$RC\"! at $LINENO. bye. called by line $1" > /dev/stderr
echo "$0: look at ${BK_OUT[$job]} in last data dir for error messages" > /dev/stderr
echo "$0: dir= ${BK_DIR[$job]}"
tail -20 ${BK_DIR[$job]}/${BK_OUT[$job]}
exit 1
fi
done
}
wait_for_all $LINENO
MUTT_ARR=()
for i in $LST; do
if [ $VERBOSE -gt 0 ]; then
pushd $i
else
pushd $i > /dev/null
fi
if [ "$PHASE_FILE" == "" ]; then
RESP=phase_cpu2017.txt
if [ $VERBOSE -gt 0 ]; then
echo "$0.$LINENO phase blank"
fi
if [ -e $RESP ]; then
echo "$0.$LINENO phase $RESP"
#OPT_PH=" -P $i/$RESP "
echo -e "-P\t\"$i/$RESP\"" >> $ALST
echo "$0.$LINENO phase $OPT_PH"
fi
fi
SM_FL=
#if [ ! -e $SUM_FILE ]; then
SM_FL=$i/$SUM_FILE
#fi
if [ $VERBOSE -gt 0 ]; then
echo "$0 SM_FL= $SM_FL SUM_FILE= $SUM_FILE"
fi
echo -e "-p\t\"$RPS\"" >> $ALST
echo -e "-s\t2,2" >> $ALST
if [ "$AVERAGE" == "1" ]; then
echo -e "-A" >> $ALST
fi
if [ "$CLIP" != "" ]; then
echo -e "-c\t$CLIP" >> $ALST
fi
if [ "$DESC_FILE" != "" ]; then
echo -e "-d\t\"$DESC_FILE\"" >> $ALST
fi
if [ "$DESC_FILE" == "" ]; then
if [ -e desc.txt ]; then
FLS=$(get_abs_filename "desc.txt")
OPT_DESC_FILE="$FLS"
fi
fi
if [ "$OPT_DESC_FILE" != "" ]; then
echo -e "-d\t\"$OPT_DESC_FILE\"" >> $ALST
fi
echo -e "-i\t\"$i/*.png\"" >> $ALST
#echo -e "-x\t$i.xlsx" >> $ALST
#echo -e "-o\tchart_new,dont_sum_sockets" >> $ALST
# itp files
# yab_cmd files might be in same dir or up 1 level
if [ -e yab_cmds.json.tsv ]; then
FLS=$(get_abs_filename yab_cmds.json.tsv)
#FLS=`ls -1 $i/metric_out.tsv`
echo -e "${FLS}" >> $ALST
else
if [ -e ../yab_cmds.json.tsv ]; then
FLS=$(get_abs_filename ../yab_cmds.json.tsv)
#FLS=`ls -1 $i/metric_out.tsv`
echo -e "${FLS}" >> $ALST
fi
fi
if [ -e metric_out.tsv ]; then
FLS=$(get_abs_filename metric_out.tsv)
#FLS=`ls -1 $i/metric_out.tsv`
echo -e "${FLS}" >> $ALST
fi
if [ -e metric_out.csv.tsv ]; then
FLS=$(get_abs_filename metric_out.csv.tsv)
#FLS=`ls -1 $i/metric_out.tsv`
echo -e "${FLS}" >> $ALST
fi
#if [ -e infra_cputime.txt.tsv ]; then
# FLS=$(get_abs_filename infra_cputime.txt.tsv)
# echo -e "${FLS}" >> $ALST
#fi
if [ $VERBOSE -gt 0 ]; then
popd
else
popd > /dev/null
fi
#FLS=`ls -1 $SM_FL $i/*txt.tsv | grep -v infra_cputime`
CKNM=$i/muttley_host_calls.tsv
if [ -e $CKNM ]; then
MUTT_ARR+=($CKNM)
fi
FLS=`ls -1 $SM_FL $i/*txt.tsv`
FLS_IC=`ls -1 $i/*txt.tsv | grep infra_cputime`
FLS_PS=`ls -1 $i/*txt.tsv | grep perf_stat`
echo -e "${FLS}" >> $ALST
MYA=($i/*log.tsv)
if [ "${#MYA}" != "0" ]; then
FLS=`ls -1 $i/*log.tsv`
echo -e "${FLS}" >> $ALST
fi
MYSVG=($i/*.svg)
if [ "${#MYSVG}" != "0" ]; then
SVG=`ls -1 $i/*.svg`
fi
MYA=($i/*current.tsv)
if [ "${#MYA}" != "0" ]; then
FLS=`ls -1 $i/*current.tsv`
echo -e "${FLS}" >> $ALST
fi
MYA=($i/muttley*.json.tsv)
if [ "${#MYA}" != "0" ]; then
FLS=`ls -1 $i/muttley*.json.tsv`
echo -e "${FLS}" >> $ALST
fi
# MYA=($i/sum_all.tsv)
# if [ "${#MYA}" != "0" ]; then
# FLS=`ls -1 $i/sum_all.tsv`
# echo -e "${FLS}" >> $ALST