From b727b863402168587447cc2b82a52561189c8ce6 Mon Sep 17 00:00:00 2001 From: Alexandre Assouad Date: Fri, 17 May 2024 10:11:23 +0200 Subject: [PATCH 1/2] Fix parsing-csv-logs-with-lua.md example for postgres > 13 --- .../advanced/parsing-csv-logs-with-lua.md | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/website/content/en/guides/advanced/parsing-csv-logs-with-lua.md b/website/content/en/guides/advanced/parsing-csv-logs-with-lua.md index 6c67bbdfb40ac..7f5544cbda1ba 100644 --- a/website/content/en/guides/advanced/parsing-csv-logs-with-lua.md +++ b/website/content/en/guides/advanced/parsing-csv-logs-with-lua.md @@ -155,6 +155,10 @@ the whole transform: "query_pos", "location", "application_name", + -- available only in postgres > 13, to remove for postgres <= 13 + "backend_type", + "leader_pid", + "query_id" } """ hooks.process = """ @@ -162,8 +166,8 @@ the whole transform: fields = csv.openstring(event.log.message):lines()() -- parse the `message` field event.log.message = nil -- drop the `message` field - for column, value in ipairs(fields) do -- iterate over CSV columns - column_name = column_names[column] -- get column name + for column, column_name in ipairs(column_names) do -- iterate over column names + value = fields[column] -- get field value event.log[column_name] = value -- set the corresponding field in the event end @@ -176,9 +180,9 @@ the whole transform: Trying to run `vector --config vector.toml` with the same input file results in structured events being output: ```json -{"application_name":"","command_tag":"","connection_from":"","context":"","database_name":"","detail":"","error_severity":"LOG","file":"log.csv","hint":"Future log output will go to log destination \"csvlog\".","host":"localhost","internal_query":"","internal_query_pos":"","location":"","log_time":"2020-04-09 12:48:49.661 UTC","message":"ending log output to stderr","process_id":"1","query":"","query_pos":"","session_id":"localhost.1","session_line_num":"1","session_start_time":"2020-04-09 12:48:49 UTC","sql_state_code":"00000","timestamp":"2020-04-09T19:49:07Z","transaction_id":"0","user_name":"","virtual_transaction_id":""} -{"application_name":"","command_tag":"","connection_from":"","context":"","database_name":"","detail":"","error_severity":"LOG","file":"log.csv","hint":"","host":"localhost","internal_query":"","internal_query_pos":"","location":"","log_time":"2020-04-09 12:48:49.669 UTC","message":"database system was shut down at 2020-04-09 12:48:25 UTC","process_id":"27","query":"","query_pos":"","session_id":"localhost.1b","session_line_num":"1","session_start_time":"2020-04-09 12:48:49 UTC","sql_state_code":"00000","timestamp":"2020-04-09T19:49:07Z","transaction_id":"0","user_name":"","virtual_transaction_id":""} -{"application_name":"","command_tag":"","connection_from":"","context":"","database_name":"","detail":"","error_severity":"LOG","file":"log.csv","hint":"","host":"localhost","internal_query":"","internal_query_pos":"","location":"","log_time":"2020-04-09 12:48:49.683 UTC","message":"database system is ready to accept connections","process_id":"1","query":"","query_pos":"","session_id":"localhost.1","session_line_num":"2","session_start_time":"2020-04-09 12:48:49 UTC","sql_state_code":"00000","timestamp":"2020-04-09T19:49:07Z","transaction_id":"0","user_name":"","virtual_transaction_id":""} +{"application_name":"","backend_type":"not initialized","command_tag":"","connection_from":"","context":"","database_name":"","detail":"","error_severity":"LOG","file":"log.csv","hint":"Future log output will go to log destination \"csvlog\".","host":"localhost","internal_query":"","internal_query_pos":"","leader_pid":"","location":"","log_time":"2020-04-09 12:48:49.661 UTC","message":"ending log output to stderr","process_id":"1","query":"","query_id":"0","query_pos":"","session_id":"localhost.1","session_line_num":"1","session_start_time":"2020-04-09 12:48:49 UTC","sql_state_code":"00000","timestamp":"2020-04-09T19:49:07Z","transaction_id":"0","user_name":"","virtual_transaction_id":""} +{"application_name":"","backend_type":"not initialized","command_tag":"","connection_from":"","context":"","database_name":"","detail":"","error_severity":"LOG","file":"log.csv","hint":"","host":"localhost","internal_query":"","internal_query_pos":"","leader_pid":"","location":"","log_time":"2020-04-09 12:48:49.669 UTC","message":"database system was shut down at 2020-04-09 12:48:25 UTC","process_id":"27","query":"","query_id":"0","query_pos":"","session_id":"localhost.1b","session_line_num":"1","session_start_time":"2020-04-09 12:48:49 UTC","sql_state_code":"00000","timestamp":"2020-04-09T19:49:07Z","transaction_id":"0","user_name":"","virtual_transaction_id":""} +{"application_name":"","backend_type":"not initialized","command_tag":"","connection_from":"","context":"","database_name":"","detail":"","error_severity":"LOG","file":"log.csv","hint":"","host":"localhost","internal_query":"","internal_query_pos":"","leader_pid":"","location":"","log_time":"2020-04-09 12:48:49.683 UTC","message":"database system is ready to accept connections","process_id":"1","query":"","query_id":"0","query_pos":"","session_id":"localhost.1","session_line_num":"2","session_start_time":"2020-04-09 12:48:49 UTC","sql_state_code":"00000","timestamp":"2020-04-09T19:49:07Z","transaction_id":"0","user_name":"","virtual_transaction_id":""} ``` Or, applying pretty formatting to one of the output events: @@ -186,6 +190,7 @@ Or, applying pretty formatting to one of the output events: ```json { "application_name": "", + "backend_type":"not initialized", "command_tag": "", "connection_from": "", "context": "", @@ -197,11 +202,13 @@ Or, applying pretty formatting to one of the output events: "host": "localhost", "internal_query": "", "internal_query_pos": "", + "leader_pid":"", "location": "", "log_time": "2020-04-09 12:48:49.661 UTC", "message": "ending log output to stderr", "process_id": "1", "query": "", + "query_id":"0", "query_pos": "", "session_id": "localhost.1", "session_line_num": "1", From 8d4048637413be80a05391afd07710b4c535d58d Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Tue, 28 May 2024 16:38:43 -0400 Subject: [PATCH 2/2] remove whitespace Signed-off-by: Jesse Szwedko --- website/content/en/guides/advanced/parsing-csv-logs-with-lua.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/content/en/guides/advanced/parsing-csv-logs-with-lua.md b/website/content/en/guides/advanced/parsing-csv-logs-with-lua.md index 7f5544cbda1ba..7034f75a4dee6 100644 --- a/website/content/en/guides/advanced/parsing-csv-logs-with-lua.md +++ b/website/content/en/guides/advanced/parsing-csv-logs-with-lua.md @@ -156,7 +156,7 @@ the whole transform: "location", "application_name", -- available only in postgres > 13, to remove for postgres <= 13 - "backend_type", + "backend_type", "leader_pid", "query_id" }