diff --git a/doc/releaseNotes.md b/doc/releaseNotes.md index 228c158..a392d9a 100644 --- a/doc/releaseNotes.md +++ b/doc/releaseNotes.md @@ -182,3 +182,7 @@ This file describes the main feature changes for each InfoLogger released versio - o2-infologger-alert service - o2-infologger-browser: - added some extra startup option, to preconfigure filters + +# v2.10.0 - 26/1/2026 +- o2-infologger-httpd: inject log messages from HTTP request +- o2-infologger-status: creates periodically a HTML report of recent messages received by infoLoggerServer diff --git a/src/InfoLoggerDispatchStats.cxx b/src/InfoLoggerDispatchStats.cxx index c70c1bf..6116318 100644 --- a/src/InfoLoggerDispatchStats.cxx +++ b/src/InfoLoggerDispatchStats.cxx @@ -29,6 +29,8 @@ #include "ConfigInfoLoggerServer.h" #include "infoLoggerMessage.h" +#define INDEX_SEPARATOR "_" + using namespace std::chrono; //////////////////////////////////////////////////////// @@ -90,7 +92,7 @@ InfoLoggerDispatchStats::InfoLoggerDispatchStats(ConfigInfoLoggerServer* config, return; // invalid index name } lix.push_back(i); - if (lix.size()>1) nix += "-"; + if (lix.size()>1) nix += INDEX_SEPARATOR; nix += name; } dPtr->ilgFieldsToIndex.push_back(std::pair(std::move(nix), std::move(lix))); @@ -234,7 +236,7 @@ int InfoLoggerDispatchStats::customMessageProcess(std::shared_ptrilgFieldsToIndex[id].second.size(); ii++) { if (debug) {printf(" get [%d]\n", dPtr->ilgFieldsToIndex[id].second[ii]);} if (ii) { - v += "-"; + v += INDEX_SEPARATOR; } std::string fv = getStringValue(lmsg, dPtr->ilgFieldsToIndex[id].second[ii]); if (debug) {printf(" = %s\n",fv.c_str());} @@ -434,7 +436,14 @@ int InfoLoggerDispatchStats::customLoop() }; std::string txt; + bool isFirst = 1; for (const auto& [timestamp, window] : dPtr->windows) { + if (isFirst) { + isFirst = 0; + } else { + txt += " "; + } + //txt += "{ " + std::to_string(timestamp) + " " + toTclList(window) + " }"; txt += toTclList(window); } diff --git a/src/o2-infologger-httpd b/src/o2-infologger-httpd new file mode 100755 index 0000000..645a8c7 --- /dev/null +++ b/src/o2-infologger-httpd @@ -0,0 +1,209 @@ +#!/usr/bin/tclsh + +# daemon to inject log messages from HTTP requests +# example request: http://ali-staging:8084/log?Message=This+is+a+test+error&Facility=test&Severity=E +# +# open port: firewall-cmd --zone=public --permanent --add-port=8084/tcp; firewall-cmd --reload +# start daemon manually: [root@alio2-cr1-hv-mvs00 ~]# nohup /root/o2-infologger-httpd > /tmp/o2-infologger-httpd.log 2>&1 & + +# v1.0 03/07/2025 - initial release + +set cfg(RunWithoutDeps) 0 +set cfg(Debug) 0 +set cfg(HTTPPort) 8084 + +set infoLoggerFields {Facility Role System Detector Partition Run Severity Level ErrorCode SourceFile SourceLine} + + +#################### + + +# function to log on stdout +proc doLog {msg} { + set t [clock format [clock seconds] -format "%d/%m/%Y %H:%M:%S"] + puts "$t\t$msg" +} + + +# function to log on stdout +proc doLogDebug {msg} { + global cfg + if {!$cfg(Debug)} {return} + set t [clock format [clock seconds] -format "%d/%m/%Y %H:%M:%S"] + #puts "$t\t$msg" + puts "$msg" +} + + +# function to log on InfoLogger +proc doLogIlg {errcode msg} { + global logHandle + global logContext + if {$logHandle == ""} { + return + } + + $logContext setField "ErrorCode" "$errcode" + $logHandle log $logContext "$msg" + doLog "$msg" + logResetFields +} + + +#################### +# check dependencies +#################### + +set dependenciesOk 1 + +# try to load infoLogger library +set defaultLevel 11 +set defaultSeverity "I" +proc logResetFields {} { + global logHandle + global logContext + global defaultLevel + global defaultSeverity + $logContext setField "Facility" "ilg/httpd" + $logContext setField "System" "FLP" + $logContext setField "Level" "$defaultLevel" + $logContext setField "Severity" "$defaultSeverity" + $logContext setField "ErrorCode" "" +} + +if {[catch { + set logHandle "" + load /opt/o2-InfoLogger/lib/infoLoggerForTcl.so + set logHandle [InfoLogger] + set logContext [InfoLoggerMetadata] + logResetFields +} err]} { + doLog "Failed to init infoLogger library: $err" + set dependenciesOk 0 +} + +# exit or continue +if {(!$dependenciesOk)} { + if ($cfg(RunWithoutDeps)) { + doLog "Dependencies failed, but continue running" + } else { + doLog "Dependencies failed, exiting" + exit 1 + } +} + +######################## +# end check dependencies +######################## + + +doLog "Starting infoLogger HTTPD bridge - pid [pid]" + + +# Procedure to handle incoming connections +proc handle_connection {sock addr port} { + doLogDebug "Connection from $addr:$port" + fconfigure $sock -blocking 0 -buffering line + fileevent $sock readable [list read_request $sock] +} + +# URL decode helper (converts %XX to character, + to space) +proc urldecode {str} { + set str [string map {+ " "} $str] + set result "" + set i 0 + while {$i < [string length $str]} { + if {[string index $str $i] eq "%"} { + set hex [string range $str [expr {$i+1}] [expr {$i+2}]] + append result [binary format c [scan $hex %x]] + incr i 3 + } else { + append result [string index $str $i] + incr i + } + } + return $result +} + +# Parse query parameters into a dict +proc parse_query {query} { + set params {} + foreach pair [split $query "&"] { + if {[regexp {([^=]+)=?(.*)} $pair -> key value]} { + set key [urldecode $key] + set value [urldecode $value] + dict set params $key $value + } + } + return $params +} + +# Request handling +proc read_request {sock} { + if {[eof $sock]} { + close $sock + return + } + + gets $sock line + doLogDebug "Received: $line" + + # First line: GET /path?key=value HTTP/1.1 + if {[regexp {^GET\s+([^?\s]+)\??([^ ]*)\s+HTTP/} $line -> path query]} { + doLogDebug "Requested path: $path" + if {$query ne ""} { + doLogDebug "Query string: $query" + set params [parse_query $query] + doLogDebug "Parsed parameters:" + foreach {key value} $params { + doLogDebug " $key = $value" + } + processQuery $path $params + } + } + + # End of headers: empty line + if {[string trim $line] eq ""} { + # Send empty HTTP response + puts $sock "HTTP/1.1 200 OK\r" + puts $sock "Content-Length: 0\r" + puts $sock "Connection: close\r" + puts $sock "\r" + flush $sock + close $sock + } +} + + +# process queries +proc processQuery {path params} { + # path is the path from HTTP request + # params is a dict with key/value pairs + if {$path == "/log"} { + global logHandle + if {$logHandle == ""} { return } + + if {[dict exists $params Message]} { + set msg [dict get $params Message] + + set logContext [InfoLoggerMetadata] + global infoLoggerFields + foreach key $infoLoggerFields { + if {[dict exists $params $key]} { + set value [dict get $params $key] + $logContext setField $key $value + doLog "$key = $value" + } + } + $logHandle log $logContext "$msg" + doLog "LOG: $msg" + } + } +} + + + +# Start listening +socket -server handle_connection $cfg(HTTPPort) +doLog "Listening on port $cfg(HTTPPort)..." +vwait forever diff --git a/src/o2-infologger-stats b/src/o2-infologger-stats new file mode 100755 index 0000000..de194ab --- /dev/null +++ b/src/o2-infologger-stats @@ -0,0 +1,471 @@ +#!/usr/bin/tclsh + +# toolkit to process online stats coming from infoLoggerServer +# v1.0.0 13/11/2025 - initial release + +set cfg(TelegrafSocket) "/tmp/telegraf.sock" +set cfg(TelegrafBucket) "InfologgerStats" +set cfg(RunWithoutDeps) 1 +set cfg(InfoLoggerConfigFile) "/etc/o2.d/infologger/infoLogger.cfg" +set cfg(Debug) 0 +set cfg(LogFacility) "ilg/stats" +set cfg(LoopTimeout) 5000 + +#################### + +set configFile $cfg(InfoLoggerConfigFile) + +catch { + set configFile $env(INFOLOGGER_CONFIG) +} + +# process command line arguments (pre-init) +set x 0 +while {[set opt [lindex $argv $x]] != ""} { + switch -exact -- $opt { + -z { + set configFile [lindex $argv [expr $x + 1]] + incr x + } + } + incr x +} + + +#################### + + +# function to log on stdout +proc doLog {msg} { + set t [clock format [clock seconds] -format "%d/%m/%Y %H:%M:%S"] + puts "$t\t$msg" +} + +# function to log on InfoLogger +proc doLogIlg {errcode msg} { + global logHandle + global logContext + if {$logHandle == ""} { + return + } + + $logHandle log $logContext "$msg" + doLog "$msg" +} + +#################### +# check dependencies +#################### + +set dependenciesOk 1 + +# try to load infoLogger library +set defaultLevel 11 +set defaultSeverity "I" +proc logResetFields {} { + global logHandle + if {$logHandle == ""} {return} + global logContext + global defaultLevel + global defaultSeverity + global cfg + $logContext setField "Facility" $cfg(LogFacility) + $logContext setField "System" "FLP" + $logContext setField "Level" "$defaultLevel" + $logContext setField "Severity" "$defaultSeverity" + $logContext setField "ErrorCode" "" + $logContext setField "Run" "" + $logContext setField "Partition" "" + $logContext setField "Detector" "" +} + +if {[catch { + set logHandle "" + load /opt/o2-InfoLogger/lib/infoLoggerForTcl.so + set logHandle [InfoLogger] + set logContext [InfoLoggerMetadata] + logResetFields + #doLog "InfoLogger library enabled" +} err]} { + doLog "Failed to init infoLogger library: $err" + set dependenciesOk 0 +} + + +# socat +if {[catch {exec socat -V} err]} { + doLog "Failed to find 'socat': $err" + set dependenciesOk 0 +} + +# telegraf +if {![file exists $cfg(TelegrafSocket)]} { + doLog "Failed to find Telegraf socket $cfg(TelegrafSocket)" + set dependenciesOk 0 +} + +# configFile +if {![file exists $configFile]} { + doLog "Failed to find InfoLogger configuration $configFile" + set dependenciesOk 0 +} + +# exit or continue +if {(!$dependenciesOk)} { + if ($cfg(RunWithoutDeps)) { + doLog "Dependencies failed, but continue running" + } else { + doLog "Dependencies failed, exiting" + exit 1 + } +} + +######################## +# end check dependencies +######################## + + +proc doLogBoth {msg} { + doLog "$msg" + doLogIlg "" "$msg" +} + + +doLog "Starting infoLoggerStatsReporter - pid [pid]" + + +# Define online server timeouts (seconds) +set onlineserver(retry) 5 +set onlineserver(max) 60 +set onlineserver(timeout) $onlineserver(retry) +set onlineserver(timer) "" + +set defaultDir "/tmp" +set infoDir "" +set configName "\[default config\]" + +set default_db_user "" +set default_db_pwd "" +set default_db_host "" +set default_db_db "" +set default_loghost "localhost" +set default_logport "6103" + +set configFileSection "\[infoStats\]" +set configFileSectionFound 0 +if {$configFile!=""} { + set fd [open $configFile "r"] + set keyfound {} + while {1} { + gets $fd line + if {[eof $fd]} {break} + # remove leading/trailing blanks + set line [string trim $line] + if {$line==$configFileSection} { + # entering infoBrowser section + set configFileSectionFound 1 + } elseif {[regexp {^\[.*\]} $line]} { + # entering another section + set configFileSectionFound 0 + } + if {!$configFileSectionFound} { + continue + } + set lv [split [string trim $line] "="] + set lkey [lindex $lv 0] + if {[string range $lkey 0 1]=="#"} {continue} + set lv [string trim [join [lrange $lv 1 end] "="]] + + lappend keyfound $lkey + set cfgvals($lkey) $lv + } + close $fd + + set cfgok 1 + foreach {keyname isoptionnal varname defval} [list \ + serverHost 1 loghost "$default_loghost" \ + serverPortStats 1 logport "$default_logport" \ + configName 1 configName "$configFile" \ + ] { + set $varname $defval + if {[catch { set $varname $cfgvals($keyname) }]} { + if {!$isoptionnal} { + doLog "Configuration variable $keyname undefined" + set cfgok 0 + } + } + } + if {!$cfgok} { + doLog "Wrong configuration in $configFile, exiting" + exit -1 + } + +} else { + + set envok 1 + foreach {envname isoptionnal varname defval} [list \ + INFOLOGGER_SERVER_HOST 1 loghost "${default_loghost}" \ + INFOLOGGER_SERVER_PORT_STATS 1 logport "${default_logport}" \ + ] { + set $varname $defval + if {[catch { set $varname $env($envname) }]} { + if {!$isoptionnal} { + doLog "Environment variable $envname undefined" + set envok 0 + } + } + } + + if {!$envok} { + doLog "Wrong environment, exiting" + exit -1 + } +} + + +################################# +# display online data +################################# +set server_fd -1 + +# process an event on the socket +proc server_event {} { + global cfg + + global server_fd + + fileevent $server_fd readable "" + set n_loop 0 + + while {1} { + + # connection closed? + if {[eof $server_fd]} { + close $server_fd + set server_fd -1 + global onlineserver + doLog "Connection closed. Reconnecting in $onlineserver(timeout) seconds" + update + set onlineserver(timer) [after [expr $onlineserver(timeout)*1000] {server_connect}] + break + } + + # read and decode message + if {[gets $server_fd data]==-1} {break} + + puts "rx: [string length $data]" + #puts "[string range $data 0 100]" + + if {1} { + set fixed "" + for {set i 0} {$i<[string length $data]} {incr i} { + set c [string index $data $i] + append fixed $c + if {$c == "\}"} { + catch { + if {[string index $data [expr $i + 1]] == "\{"} { + append fixed " " + } + } + } + } + set data $fixed + } + + + # Ensure the string is a proper list + set windows [list {*}$data] + + puts "number of windows: [llength $windows]" + + foreach w $windows { + foreach {k v} $w { + set W($k) $v + } + puts "[clock format $W(timeBegin)] $W(totalMessages)" + } + + break + + # Use only the last window + set lastWindow [lindex $windows 0] + + # Convert to dict + dict for {k v} $lastWindow { + puts "$k = $v" + set W($k) $v + } + + puts "last window: [clock format $W(timeBegin)]" + puts "number of indexes: [llength $W(fieldCounts)]" + + # Parse fieldCounts + array unset FC + dict for {fieldName counts} $W(fieldCounts) { + set FC($fieldName) {} + + dict for {valueName count} $counts { + puts $valueName + lappend FC($fieldName) [list $valueName $count] + } + } + +# Determine the top-N (e.g. top 3) per field +set hotTopics {} +set N 3 + +foreach field [array names FC] { + set pairs $FC($field) + # Sort descending by count + set sorted [lsort -index 1 -integer -decreasing $pairs] + set top [lrange $sorted 0 [expr {$N-1}]] + dict set hotTopics $field $top +} + + + set html "\n\n" +append html "Status - Hot Topics\n" +append html "\n" +append html "\n" + +append html "

Hot Topics

\n" +append html "

Window: [clock format $W(timeBegin)] - [clock format $W(timeEnd)]

\n" + +# Table for each field +dict for {fieldName topics} $hotTopics { + append html "

$fieldName

\n" + append html "\n\n" + + foreach pair $topics { + lassign $pair valueName count + append html "\n" + } + + append html "
ValueCount
$valueName$count
\n" +} + +append html "" + + +# Create HTML file +set filename "report.html" +set fh [open $filename w] +puts $fh $html +close $fh + +puts "* HTML report generated: $filename" + + + + + + + + + + + + incr n_loop + if {$n_loop==1000} {break} + } + + if {$server_fd!=-1} { + fileevent $server_fd readable server_event + } +} + + +# open socket to data server +proc server_connect {} { + global server_fd + global env + global onlineserver + + set onlinetimer(timer) "" + + # connect only if not connected yet + if {$server_fd!=-1} {return} + + global loghost + global logport + + doLog "Connecting $loghost ..." + update + + # open socket + if {[catch {set server_fd [socket $loghost $logport]} err]} { + doLog "while connecting $loghost:$logport - $err. Will retry in $onlineserver(timeout) seconds" + set onlineserver(timer) [after [expr $onlineserver(timeout)*1000] {server_connect}] + if {$onlineserver(timeout)<$onlineserver(max)} {set onlineserver(timeout) [expr $onlineserver(timeout) * 2]} + update + return + } + set onlineserver(timeout) $onlineserver(retry) + fconfigure $server_fd -blocking false + fconfigure $server_fd -buffersize 1000000 + fileevent $server_fd readable server_event + + doLog "Connected" + + update +} + +set online 1 +set busy 0 + +proc doOnline {} { + global online + global server_fd + + global busy + + if {$online} { + + if {$busy} {return} + set busy 1 + + # Go Online + + # update status display + doLog "Opening infoLoggerServer" + + # connect server + server_connect + + update + + } else { + # close server + if {$server_fd!=-1} { + close $server_fd + set server_fd -1 + } + + doLog "Closing infoLoggerServer" + + global onlineserver + if {"$onlineserver(timer)"!=""} { + after cancel $onlineserver(timer) + set onlineserver(timer) "" + } + + update + set busy 0 + } +} + + +doOnline + + +while {1} { + after $cfg(LoopTimeout) {set timeout 1} + vwait timeout + #doLog "loop" + set now [clock seconds] +} +