forked from antelopeusersgroup/antelope_contrib
/
q330logs2db.xpl
executable file
·429 lines (329 loc) · 11.3 KB
/
q330logs2db.xpl
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
#
# Searches for important user messages from q330 log files
# and saves them to a dlevent database table
#
# This is one step up the chain from logs2dlevent which
# greps through the logs that were written to disk with
# orb2logs
#
#
#
# J.Eakins
# 11/19/2008
#
# Added ability to send messages direct to Slack
# 11/16/2015
use strict ;
# use diagnostics ;
use Datascope ;
use archive;
use orb;
use Cwd;
use File::Find;
use File::Basename;
use Getopt::Std ;
elog_init ( $0, @ARGV) ;
our ( $opt_l, $opt_p, $opt_q, $opt_s, $opt_S, $opt_v, $opt_V, $opt_X );
my ($logsource, $orbname, $dbname, $orb, $pktid, $srcname, $net, $sta, $chan, $loc );
my ($nbytes,$result,$pkt,$packet,$subcode,$desc,$type,$suffix,$pf,$ref);
my ($dlname, $dlsta, $dlevtype, $comment);
my ($evtime, $evdate_hr, $evmin, $evsec, $event_type, $event );
my (%dlevents, $key, $value, %event_phrase, %convert_umsg, @match, @reject);
my ($target, @dlevent_record, @dbl, @db);
my ($dbout, $statefile);
my ($search_pattern, $reject_pattern, $url);
my ($com1, $com2, $com3, $com4, $com5, $com6);
my ($now, $t, $n, $pktstart, $time, $log) ;
my ($prog_name, $cmdline);
my ($stop, $pkttime ) ;
# -l is the pattern of logfile to match
# -p is the pf file containing the phrases to match
# -s where to start reviewing log packets. Default is only look at new packets.
# -S state file.
# -S overrides -s; if no -S or -s, starts with newest packet.
# -X pipes messages to Slack (needs to have slack_hook_url defined in pf file))
# -q don't write out to database (useful if only want Slack messages)
if (! getopts('qp:l:s:S:vVX') || (@ARGV < 1 || @ARGV > 2 ) ) {
elog_complain("\nGetopts or number of arguments failure.\n");
&usage;
} else {
$orbname = $ARGV[0] ;
$dbout = $ARGV[1] if !$opt_q ;
}
elog_die("Must use '-X' if using '-q'\n") if ($opt_q && !$opt_X) ;
if (!$opt_q && (@ARGV < 2)) {
elog_complain("Need to specify a database\n") ;
&usage;
}
$prog_name = $0 ;
$prog_name =~ s".*/"";
$statefile = $opt_S ? $opt_S : "state/q330logs2db" ;
elog_notify("Using statefile: $statefile\n") if $opt_v ;;
if (! -e $statefile ) {
my $cmd = sprintf "mkdir -p %s", dirname($statefile) ;
run ($cmd) ;
run ("touch $statefile") ;
}
$now = time();
$t = strtime($now);
elog_notify(0,"\nStarting program at: $t");
$logsource = $opt_l || ".*/log" ;
#
# read pf file
#
if ($opt_p) {
$pf = $opt_p;
} else {
$pf = $prog_name ;
}
elog_notify(0,"Reading pf file: $pf \n") if ($opt_v || $opt_V) ;
&get_pf ;
$search_pattern = join "|", @match ;
$reject_pattern = join "|", @reject;
# open orb and reap/unstuff all input source packets
&cmdline;
elog_notify($cmdline);
$orb = orbopen ($orbname, "r&" );
elog_notify("orbname: $orbname \n") if $opt_V ;
if ($orb == -1) {
elog_die("Can't open $orbname\n") ;
}
#
# determine where to position orb pointer
#
# if state file exists, override $opt_s
#
elog_notify("Positioning orb\n") if $opt_v ;
if ( $opt_s && ( ! $opt_S || ! -e $statefile ) ) {
elog_notify("opt_s is: $opt_s\n") if ($opt_v || $opt_V) ;
if ($opt_s =~ /OLDEST|oldest|Oldest|first|FIRST|First/) {
$n = orbposition($orb,"oldest");
$pktstart = orbtell($orb);
} else {
$n = orbafter($orb,str2epoch($opt_s));
$pktstart = orbtell($orb);
}
elog_notify("starting from pktid: $pktstart\n");
} elsif ($opt_s && $opt_S) { # implies there is an $opt_S too
elog_complain("Using state file for first pktid\n");
elog_complain("State file overrides -s $opt_s \n") if $opt_s ;
$stop = 0;
$pktid = 0;
exhume( $statefile, \$stop, 1 ) ;
if (orbresurrect( $orb, \$pktid, \$pkttime ) == 0) {
elog_notify( 0, "resurrection successful: pktid: $pktid \n" );
} else {
elog_complain( 0, "resurrection unsuccesful: pktid: $pktid \n");
}
$n = orbseek( $orb, "$pktid" );
$pktstart = orbtell($orb);
elog_notify("After exhume, orbresurrect, and orbseek, starting from pktid: $pktstart\n");
} else { # can't read state info and no start time, start at current packet
elog_notify("Couldn't exhume. Starting after current pktid\n");
$n = orbseek( $orb, "ORBNEWEST" );
$pktstart = orbtell($orb);
elog_notify(0, "starting from pktid: $pktstart\n");
}
$n = orbselect ($orb, "$logsource") ;
if ($n == -1) {
elog_die("No packets after orbselect\n");
}
elog_notify("npkts(?); $n \n") if ($opt_V) ;
for (; $stop == 0 ; ) {
($pktid, $srcname, $pkttime, $packet, $nbytes) = orbreap($orb) ;
if ( $opt_S ) {
eval( bury() ) if $pktid >= 0;
if( $@ ) {
elog_complain( "Unexpected failure of bury command! ");
}
}
eval {
($result, $pkt) = unstuffPkt($srcname, $time, $packet, $nbytes) ;
} ;
if ( $@ ) {
elog_notify("unstuffPkt failed: $@\n") ;
} elsif ($result eq "Pkt_ch") {
($net, $sta, $chan, $loc, $suffix, $subcode) = $pkt->parts() ;
($type, $desc) = $pkt->PacketType() ;
# if unstuffed source packet matches then open db and add record
$log = $pkt->string;
if (defined $log) {
if ( $log =~ /($search_pattern)/ ) {
if ( $log =~ /($reject_pattern)/) {
elog_notify(0,"Skipping this one: $log \n") if $opt_V;
next ;
} else {
elog_notify(0, "$log \n") if ($opt_v || $opt_V) ; # print contents of line
# get individual pieces - differs if it is a LOG or a UMSG
($evdate_hr,$evmin,$evsec,$target,$dlsta,$com1,$com2,$com3,$com4,$com5,$com6) = split /:/,&trim($log) ;
$evtime = "$evdate_hr".":".$evmin.":".$evsec;
$target = &trim($target);
$dlsta = &trim($dlsta);
$dlsta = "$dlsta";
elog_notify(0, "Dlname: $dlsta\n") if $opt_V ;
elog_notify(0, "Evtime: $evtime\n") if $opt_V ;
$evtime = str2epoch($evtime);
elog_notify(0, "Evtime: $evtime\n") if $opt_V ;
if ( !$dlsta =~ /[A-Z,0-9]{2}\_[A-Z,0-9]{3,6}/) {
elog_notify(0,"This does not look like a valid dlsta: $dlsta. Skipping\n") if $opt_v;
next;
}
$comment = "$com1";
if ($comment =~ /LOG/) { # Try to get correct info from Calibration message
# skip com2 - com4 as currently they are another date (earlier versions have this as log info, see elsif below)
if ($com5) {
$comment = $comment . $com5 ;
if ($com6) {
$comment = $comment . $com6;
}
} elsif ($com2 =~ /ip.*/) { # try to program for old log/umsg's from before 2006?
$comment = $comment . $com2;
if ($com3) {
$comment = $comment . $com3;
if ($com4) {
$comment = $comment . $com4;
}
}
}
} else {
if ( $com2 ) {
$comment = $comment . $com2;
if ($com3) {
$comment = $comment . $com3;
if ($com4) {
$comment = $comment . $com4;
}
}
}
}
foreach $value (values %dlevents) {
if ($comment =~ $value ) {
$dlevtype = $event_phrase{$value} ;
$dlname = &trim($dlsta) ;
elog_notify(0, " $dlname: $dlevtype: " . strtime($evtime) . " \n");
if ( $dlevtype =~ /service/) {
foreach $key (keys %convert_umsg) {
if ($comment =~ /$key/) {
$dlevtype = $convert_umsg{$key} ;
elog_notify(0, "Using converted dlevtype for: $comment\n") if ($opt_v || $opt_V);
elog_notify(0, "New dlevtype is:", $convert_umsg{$key},"\n") if ($opt_v || $opt_V);
}
}
}
last;
}
}
# add contents to databas, skip if -q
&post2db unless $opt_q ;
# add in Slack integration. Value for Slack URL needs to be specified in pf file
post2slack($dlsta,$comment,$dlevtype) if ($opt_X) ;
}
} else {
# print "Pattern does not match\n";
}
} # end of if defined $log
} else {
elog_notify(0, "Non-matching packet.\n") if $opt_V;
} # end of successful pkt unstuff
} # end of for loop keeping orb open
&bury() if $pktid >= 0 ;
elog_notify(0,"Finished.\n") ;
exit;
# start subs here
sub trim {
my @out = @_;
for (@out) {
s/^\s+//;
s/\s+$//;
}
return wantarray ? @out : $out[0];
}
sub run { # run system cmds safely
my ( $cmd ) = @_ ;
system ( $cmd ) ;
if ($?) {
elog_complain(0, "$cmd error $? \n") ;
exit(1);
}
}
sub cmdline { # &cmdline();
$cmdline = "command line: $0 " ;
$cmdline = $cmdline . " -p $opt_p " if $opt_p;
$cmdline = $cmdline . " -v " if $opt_v;
$cmdline = $cmdline . " -V " if $opt_V;
$cmdline = $cmdline . " -q " if $opt_q;
$cmdline = $cmdline . " -X " if $opt_X;
$cmdline = $cmdline . " -l '$opt_l' " if $opt_l;
$cmdline = $cmdline . " -S $opt_S " if $opt_S;
$cmdline = $cmdline . " -s $opt_s " if $opt_s;
$cmdline = $cmdline . " @ARGV \n" ;
\nUSAGE: $0 [-p pf] [-v] [-q] [-X] [-l match_logname] [-S state] [-s {start_time|OLDEST}] orb db
return;
}
sub get_pf {
my ( $ref );
$ref = pfget ($pf, 'dlevents' );
%dlevents = %$ref;
foreach $key (sort keys %dlevents) { # put events alpha-order
$event_type = $key;
$event = $dlevents{$event_type};
elog_notify(0, "Event type: ", $event_type, " Matching phrase is: ", $event,"\n") if ( $opt_v || $opt_V) ;
}
%event_phrase = reverse %dlevents;
$ref = pfget ($pf, 'convert_umsg' );
%convert_umsg = %$ref;
$ref = pfget($pf, "match");
@match = @$ref;
$ref = pfget($pf, "reject");
@reject = @$ref;
$url = pfget ($pf, "slack_hook_url") if $opt_X ;
}
sub post2db {
elog_notify(0,"Opening db: $dbout\n") if $opt_V;
@db = dbopen($dbout, "r+") ;
@dbl = dblookup(@db, "", "dlevent" , "", "") ;
@dlevent_record = ();
push(@dlevent_record, "dlname", $dlsta,
"time", $evtime,
"dlevtype", $dlevtype,
"dlcomment", $comment,
) ;
elog_notify(0, "Creating dlevent_record\n") if ($opt_V);
eval { dbaddv(@dbl,@dlevent_record) };
if ($@) {
warn $@;
elog_notify(0, "Duplicate comment... Dlsta: $dlsta, Time: $evtime, Evtype: $dlevtype\n");
elog_complain(0, "Duplicate comment. Will ignore.\n") if ($opt_v || $opt_V);
}
# close db, then look at next packet
dbfree(@dbl);
dbclose (@db) ;
}
sub post2slack {
my ($dlsta,$comment,$dlevtype) = (@_);
$evtime = strtime($evtime);
my $text = "$dlsta \| $dlevtype \| $evtime \| $comment" ;
$text =~ s/[^a-zA-Z0-9\.=\s\_\-!\@\$\%#><\[\]\|\)\(\^\/\\\?\:]//g ;
elog_notify ("Posting umsg to Slack: $text \n") if $opt_v ;
# Both of the below commands worked. Not sure which should be preferred
# my $mycmd = "curl -s -d \'payload={\"text\":\"$text\"}\' $url";
my $mycmd = "curl -X POST -s --data-urlencode \'payload={\"text\":\"$text\"}\' $url";
# must use "open" to get rid of "ok" response when a message posts to Slack
open (FH, "$mycmd |") || die "Failed: $!\n";
sleep 10; # attempt to bypass "(23) Failed writing body"
close (FH) || elog_complain "Curl command did not exit properly: $?" ;
}
sub usage {
print STDERR <<END;
\nUSAGE: $0 [-p pf] [-v] [-q] [-X] [-l match_logname] [-S state] [-s {start_time|OLDEST}] orb db
END
exit(1);
}
__DATA__
% cat q330logs2db.state
:usarray &Arr{
last_pktid 257604
last_pkttime &Literal{
1162577902.525 (307) 2006-11-03 18:18:22.525 UTC Friday}
orb_start 1161444453.37733
}