From 3d7553b9f5bab733303596fd3ce2a4393e028e51 Mon Sep 17 00:00:00 2001 From: Lorenzo Fontana Date: Mon, 2 Sep 2019 18:38:41 +0200 Subject: [PATCH 01/76] build: setup grpc and proto for falco outputs Signed-off-by: Lorenzo Fontana Co-Authored-by: Leonardo Di Donato --- userspace/falco/CMakeLists.txt | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/userspace/falco/CMakeLists.txt b/userspace/falco/CMakeLists.txt index fb5b5355fa8..2ef08e8cead 100644 --- a/userspace/falco/CMakeLists.txt +++ b/userspace/falco/CMakeLists.txt @@ -20,6 +20,15 @@ if(NOT SYSDIG_DIR) endif() configure_file("${SYSDIG_DIR}/userspace/sysdig/config_sysdig.h.in" config_sysdig.h) +add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/falco_output.grpc.pb.cc + ${CMAKE_CURRENT_BINARY_DIR}/falco_output.grpc.pb.h + ${CMAKE_CURRENT_BINARY_DIR}/falco_output.pb.cc + ${CMAKE_CURRENT_BINARY_DIR}/falco_output.pb.h + COMMENT "Generate gRPC code for falco_output" + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/falco_output.proto + COMMAND ${PROTOC} -I ${CMAKE_CURRENT_SOURCE_DIR} --cpp_out=. ${CMAKE_CURRENT_SOURCE_DIR}/falco_output.proto + COMMAND ${PROTOC} -I ${CMAKE_CURRENT_SOURCE_DIR} --grpc_out=. --plugin=protoc-gen-grpc=${GRPC_CPP_PLUGIN} ${CMAKE_CURRENT_SOURCE_DIR}/falco_output.proto + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) add_executable(falco configuration.cpp @@ -29,7 +38,10 @@ add_executable(falco statsfilewriter.cpp falco.cpp "${SYSDIG_DIR}/userspace/sysdig/fields_info.cpp" - webserver.cpp) + webserver.cpp + grpc_server.cpp + ${CMAKE_CURRENT_BINARY_DIR}/falco_output.grpc.pb.cc + ${CMAKE_CURRENT_BINARY_DIR}/falco_output.pb.cc) target_include_directories(falco PUBLIC "${SYSDIG_DIR}/userspace/sysdig" @@ -38,10 +50,18 @@ target_include_directories(falco PUBLIC "${PROJECT_BINARY_DIR}/driver/src" "${YAMLCPP_INCLUDE_DIR}" "${CIVETWEB_INCLUDE_DIR}" + "${GRPC_INCLUDE}" + "${PROTOBUF_INCLUDE}" + "${CMAKE_CURRENT_BINARY_DIR}" "${DRAIOS_DEPENDENCIES_DIR}/yaml-${DRAIOS_YAML_VERSION}/target/include") + + target_link_libraries(falco falco_engine sinsp) target_link_libraries(falco + "${GRPCPP_LIB}" + "${GRPC_LIB}" + "${PROTOBUF_LIB}" "${LIBYAML_LIB}" "${YAMLCPP_LIB}" "${CIVETWEB_LIB}") From ec5f974a7d8b505edae977a785524868f84e1a09 Mon Sep 17 00:00:00 2001 From: Lorenzo Fontana Date: Mon, 2 Sep 2019 18:39:39 +0200 Subject: [PATCH 02/76] new(userspace/falco): falco output protocol definition Co-Authored-By: Leonardo Di Donato Signed-off-by: Lorenzo Fontana --- userspace/falco/falco_output.proto | 35 ++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 userspace/falco/falco_output.proto diff --git a/userspace/falco/falco_output.proto b/userspace/falco/falco_output.proto new file mode 100644 index 00000000000..f9ba464a179 --- /dev/null +++ b/userspace/falco/falco_output.proto @@ -0,0 +1,35 @@ +syntax = "proto3"; + +import "google/protobuf/timestamp.proto"; + +service FalcoOutputService { + rpc Subscribe(FalcoOutputRequest) returns (stream FalcoOutputResponse); +} + +message FalcoOutputRequest { +} + +message FalcoOutputResponse { + string id = 1; + google.protobuf.Timestamp time = 2; + enum Priority { + EMERGENCY = 0; + ALERT = 1; + CRITICAL = 2; + ERROR = 3; + WARNING = 4; + NOTICE = 5; + INFORMATIONAL = 6; + DEBUG = 7; + } + Priority priority = 3; + enum Source { + SYSCALL = 0; + K8S_AUDIT = 1; + } + Source source = 4; + string rule = 5; + string format = 6; + string output = 7; + map output_fields = 8; +} From 0e00fe30e2d150b56e993a6e8f0ca8bb415019c0 Mon Sep 17 00:00:00 2001 From: Lorenzo Fontana Date: Mon, 2 Sep 2019 18:40:38 +0200 Subject: [PATCH 03/76] new(userspace/falco): initial grpc server implementation Co-Authored-By: Leonardo Di Donato Signed-off-by: Lorenzo Fontana --- userspace/falco/falco.cpp | 5 ++ userspace/falco/grpc_server.cpp | 119 ++++++++++++++++++++++++++++++++ userspace/falco/grpc_server.h | 31 +++++++++ 3 files changed, 155 insertions(+) create mode 100644 userspace/falco/grpc_server.cpp create mode 100644 userspace/falco/grpc_server.h diff --git a/userspace/falco/falco.cpp b/userspace/falco/falco.cpp index 0f136499205..ce43ec197db 100644 --- a/userspace/falco/falco.cpp +++ b/userspace/falco/falco.cpp @@ -47,6 +47,7 @@ limitations under the License. #include "config_falco.h" #include "statsfilewriter.h" #include "webserver.h" +#include "grpc_server.h" typedef function open_t; @@ -1162,6 +1163,10 @@ int falco_init(int argc, char **argv) webserver.start(); } + // grpc server + // TODO: this is blocking now, not what we want, falco must go on. Just an experiment for now. + start_grpc_server(5060, 1); + if(!trace_filename.empty() && !trace_is_scap) { read_k8s_audit_trace_file(engine, diff --git a/userspace/falco/grpc_server.cpp b/userspace/falco/grpc_server.cpp new file mode 100644 index 00000000000..1a7c5d8024b --- /dev/null +++ b/userspace/falco/grpc_server.cpp @@ -0,0 +1,119 @@ +/* +Copyright (C) 2016-2019 The Falco Authors + +This file is part of falco. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#include +#include +#include +#include + +#ifdef GRPC_INCLUDE_IS_GRPCPP +#include +#else +#include +#endif + +#include "grpc_server.h" +#include "falco_output.grpc.pb.h" +#include "falco_output.pb.h" + +bool grpc_server_impl::is_running() +{ + // TODO: this must act as a switch to shut down the server + return true; +} + +class grpc_server : public grpc_server_impl +{ +public: + grpc_server(const char* server_addr, int threadiness): + server_addr(server_addr), + threadiness(threadiness) + { + } + + virtual ~grpc_server() = default; + + // Run() is blocked. It doesn't return until Stop() is called from another thread. + void Run(); + + void thread_process(int threadIndex) + { + // TODO: is this right? That's what we want? + // Tell pthread to not handle termination signals in the current thread + sigset_t set; + sigemptyset(&set); + sigaddset(&set, SIGHUP); + sigaddset(&set, SIGINT); + pthread_sigmask(SIG_BLOCK, &set, nullptr); + + void* tag = nullptr; + bool eventReadSuccess = false; + while(completion_queue->Next(&tag, &eventReadSuccess)) + { + if(tag == nullptr) + { + // TODO: empty tag returned, log, what to do? + continue; + } + } + } + + // There is no shutdown handling in this code. + void run() + { + grpc::ServerBuilder builder; + // Listen on the given address without any authentication mechanism. + builder.AddListeningPort(server_addr, grpc::InsecureServerCredentials()); + // builder.RegisterService(&falco_output_svc); // TODO: enable this when we do the impl + + completion_queue = builder.AddCompletionQueue(); + server = builder.BuildAndStart(); + std::cout << "Server listening on " << server_addr << std::endl; + + // int context_count = threadiness * 10; + + threads.resize(threadiness); + + int thread_idx = 0; + for(std::thread& thread : threads) + { + thread = std::thread(&grpc_server::thread_process, this, thread_idx++); + } + + while(is_running()) + { + } + } + +private: + // FalcoOutputService::AsyncService falco_output_svc; + std::unique_ptr server; + std::string server_addr; + int threadiness = 0; + std::unique_ptr completion_queue; + std::vector threads; +}; + +bool start_grpc_server(unsigned short port, int threadiness) +{ + // TODO: make bind address configurable + std::string server_addr = "0.0.0.0:" + std::to_string(port); + grpc_server srv(server_addr.c_str(), threadiness); + srv.run(); + return true; +} diff --git a/userspace/falco/grpc_server.h b/userspace/falco/grpc_server.h new file mode 100644 index 00000000000..35587f4c872 --- /dev/null +++ b/userspace/falco/grpc_server.h @@ -0,0 +1,31 @@ +/* +Copyright (C) 2016-2019 The Falco Authors + +This file is part of falco. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#pragma once + +class grpc_server_impl +{ +public: + grpc_server_impl() = default; + ~grpc_server_impl() = default; + +protected: + bool is_running(); +}; + +bool start_grpc_server(unsigned short port, int threadiness); From 400c1be59d7e63b7689693a26d88ac9935c7538c Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Tue, 3 Sep 2019 10:26:13 +0000 Subject: [PATCH 04/76] update(userspace/falco): mvoing proto enum defs outside Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/falco_output.proto | 44 ++++++++++++++++-------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/userspace/falco/falco_output.proto b/userspace/falco/falco_output.proto index f9ba464a179..655dc797595 100644 --- a/userspace/falco/falco_output.proto +++ b/userspace/falco/falco_output.proto @@ -2,34 +2,36 @@ syntax = "proto3"; import "google/protobuf/timestamp.proto"; -service FalcoOutputService { - rpc Subscribe(FalcoOutputRequest) returns (stream FalcoOutputResponse); +service falco_output_service { + rpc Subscribe(falco_output_request) returns (stream falco_output_response); } -message FalcoOutputRequest { +message falco_output_request { } -message FalcoOutputResponse { +enum priority { + EMERGENCY = 0; + ALERT = 1; + CRITICAL = 2; + ERROR = 3; + WARNING = 4; + NOTICE = 5; + INFORMATIONAL = 6; + DEBUG = 7; +} + +enum source { + SYSCALL = 0; + K8S_AUDIT = 1; +} + +message falco_output_response { string id = 1; google.protobuf.Timestamp time = 2; - enum Priority { - EMERGENCY = 0; - ALERT = 1; - CRITICAL = 2; - ERROR = 3; - WARNING = 4; - NOTICE = 5; - INFORMATIONAL = 6; - DEBUG = 7; - } - Priority priority = 3; - enum Source { - SYSCALL = 0; - K8S_AUDIT = 1; - } - Source source = 4; + priority priority = 3; + source source = 4; string rule = 5; string format = 6; string output = 7; map output_fields = 8; -} +} \ No newline at end of file From bbb39361af02b26791b389ff00553c0d052cb41c Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Tue, 3 Sep 2019 10:26:56 +0000 Subject: [PATCH 05/76] new(userspace/falco): grpc context and stream context Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/grpc_context.cpp | 35 ++++++++++++++++++++ userspace/falco/grpc_context.h | 56 ++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 userspace/falco/grpc_context.cpp create mode 100644 userspace/falco/grpc_context.h diff --git a/userspace/falco/grpc_context.cpp b/userspace/falco/grpc_context.cpp new file mode 100644 index 00000000000..f8559755224 --- /dev/null +++ b/userspace/falco/grpc_context.cpp @@ -0,0 +1,35 @@ +/* +Copyright (C) 2016-2019 The Falco Authors + +This file is part of falco. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#include "grpc_context.h" + +context::context(grpc::ServerContext* ctx): + m_ctx(ctx) +{ + // todo: set thread-specific prefix "[session id][request id]" +} + +void context::get_metadata(std::string key, std::string& val) +{ + const std::multimap& client_metadata = m_ctx->client_metadata(); + auto it = client_metadata.find(key); + if(it != client_metadata.end()) + { + val.assign(it->second.data(), it->second.size()); + } +} \ No newline at end of file diff --git a/userspace/falco/grpc_context.h b/userspace/falco/grpc_context.h new file mode 100644 index 00000000000..dc471ccf5e4 --- /dev/null +++ b/userspace/falco/grpc_context.h @@ -0,0 +1,56 @@ +/* +Copyright (C) 2016-2019 The Falco Authors + +This file is part of falco. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#pragma once + +#ifdef GRPC_INCLUDE_IS_GRPCPP +#include +#else +#include +#endif + +class context +{ +public: + context(grpc::ServerContext* ctx); + ~context() = default; + + void get_metadata(std::string key, std::string& val); + +private: + grpc::ServerContext* m_ctx = nullptr; +}; + +class stream_context : public context +{ +public: + stream_context(grpc::ServerContext* ctx): context(ctx) {} + ~stream_context() = default; + +private: + enum : char + { + STREAMING = 1, + SUCCESS, + ERROR + } m_status = STREAMING; + // Request-specific stream data + mutable void* m_stream = nullptr; + // Are there more responses to stream? + mutable bool m_has_more = false; +}; From 413701e80f48a748719aaa3a2b301067dc7667be Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Tue, 3 Sep 2019 10:27:47 +0000 Subject: [PATCH 06/76] update(userspace/falco): reorganize grpc server Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/grpc_server.cpp | 109 ++++++++++++-------------------- userspace/falco/grpc_server.h | 34 +++++++++- 2 files changed, 74 insertions(+), 69 deletions(-) diff --git a/userspace/falco/grpc_server.cpp b/userspace/falco/grpc_server.cpp index 1a7c5d8024b..5530bb4b0ea 100644 --- a/userspace/falco/grpc_server.cpp +++ b/userspace/falco/grpc_server.cpp @@ -17,9 +17,6 @@ limitations under the License. */ #include -#include -#include -#include #ifdef GRPC_INCLUDE_IS_GRPCPP #include @@ -28,8 +25,7 @@ limitations under the License. #endif #include "grpc_server.h" -#include "falco_output.grpc.pb.h" -#include "falco_output.pb.h" +#include "grpc_context.h" bool grpc_server_impl::is_running() { @@ -37,83 +33,60 @@ bool grpc_server_impl::is_running() return true; } -class grpc_server : public grpc_server_impl +void grpc_server::thread_process(int thread_index) { -public: - grpc_server(const char* server_addr, int threadiness): - server_addr(server_addr), - threadiness(threadiness) + // TODO: is this right? That's what we want? + // Tell pthread to not handle termination signals in the current thread + sigset_t set; + sigemptyset(&set); + sigaddset(&set, SIGHUP); + sigaddset(&set, SIGINT); + pthread_sigmask(SIG_BLOCK, &set, nullptr); + + void* tag = nullptr; + bool event_read_success = false; + while(m_completion_queue->Next(&tag, &event_read_success)) { - } - - virtual ~grpc_server() = default; - - // Run() is blocked. It doesn't return until Stop() is called from another thread. - void Run(); - - void thread_process(int threadIndex) - { - // TODO: is this right? That's what we want? - // Tell pthread to not handle termination signals in the current thread - sigset_t set; - sigemptyset(&set); - sigaddset(&set, SIGHUP); - sigaddset(&set, SIGINT); - pthread_sigmask(SIG_BLOCK, &set, nullptr); - - void* tag = nullptr; - bool eventReadSuccess = false; - while(completion_queue->Next(&tag, &eventReadSuccess)) + if(tag == nullptr) { - if(tag == nullptr) - { - // TODO: empty tag returned, log, what to do? - continue; - } + // TODO: empty tag returned, log, what to do? + continue; } } +} - // There is no shutdown handling in this code. - void run() - { - grpc::ServerBuilder builder; - // Listen on the given address without any authentication mechanism. - builder.AddListeningPort(server_addr, grpc::InsecureServerCredentials()); - // builder.RegisterService(&falco_output_svc); // TODO: enable this when we do the impl +void grpc_server::run() +{ + grpc::ServerBuilder builder; + // Listen on the given address without any authentication mechanism. + builder.AddListeningPort(m_server_addr, grpc::InsecureServerCredentials()); + // builder.RegisterService(&falco_output_svc); // TODO: enable this when we do the impl - completion_queue = builder.AddCompletionQueue(); - server = builder.BuildAndStart(); - std::cout << "Server listening on " << server_addr << std::endl; + m_completion_queue = builder.AddCompletionQueue(); + m_server = builder.BuildAndStart(); + std::cout << "Server listening on " << m_server_addr << std::endl; - // int context_count = threadiness * 10; + // int context_count = threadiness * 10; - threads.resize(threadiness); + m_threads.resize(m_threadiness); - int thread_idx = 0; - for(std::thread& thread : threads) - { - thread = std::thread(&grpc_server::thread_process, this, thread_idx++); - } + int thread_idx = 0; + for(std::thread& thread : m_threads) + { + thread = std::thread(&grpc_server::thread_process, this, thread_idx++); + } - while(is_running()) - { - } + while(is_running()) + { } +} -private: - // FalcoOutputService::AsyncService falco_output_svc; - std::unique_ptr server; - std::string server_addr; - int threadiness = 0; - std::unique_ptr completion_queue; - std::vector threads; -}; +void grpc_server::subscribe_handler(const stream_context& ctx, falco_output_request req, falco_output_response res) +{ +} -bool start_grpc_server(unsigned short port, int threadiness) +void start_grpc_server(std::string server_address, int threadiness) { - // TODO: make bind address configurable - std::string server_addr = "0.0.0.0:" + std::to_string(port); - grpc_server srv(server_addr.c_str(), threadiness); + grpc_server srv(server_address, threadiness); srv.run(); - return true; } diff --git a/userspace/falco/grpc_server.h b/userspace/falco/grpc_server.h index 35587f4c872..df1ff86adcc 100644 --- a/userspace/falco/grpc_server.h +++ b/userspace/falco/grpc_server.h @@ -18,6 +18,13 @@ limitations under the License. #pragma once +#include +#include + +#include "falco_output.grpc.pb.h" +#include "falco_output.pb.h" +#include "grpc_context.h" + class grpc_server_impl { public: @@ -26,6 +33,31 @@ class grpc_server_impl protected: bool is_running(); + + void subscribe_handler(const stream_context& ctx, falco_output_request req, falco_output_response res); +}; + +class grpc_server : public grpc_server_impl +{ +public: + grpc_server(std::string server_addr, int threadiness): + m_server_addr(server_addr), + m_threadiness(threadiness) + { + } + virtual ~grpc_server() = default; + + void thread_process(int thread_index); + void run(); + void subscribe_handler(const stream_context& ctx, falco_output_request req, falco_output_response res); + +private: + // falco_output_service::AsyncService falco_output_svc; + std::unique_ptr m_server; + std::string m_server_addr; + int m_threadiness = 0; + std::unique_ptr m_completion_queue; + std::vector m_threads; }; -bool start_grpc_server(unsigned short port, int threadiness); +void start_grpc_server(std::string server_address, int threadiness); From b454f49ef93b9b747baa3b27e8a120fea9a9ea44 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Tue, 3 Sep 2019 10:28:21 +0000 Subject: [PATCH 07/76] update(userspace/falco): change the way the grpc server is started Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/falco.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/userspace/falco/falco.cpp b/userspace/falco/falco.cpp index ce43ec197db..46315466201 100644 --- a/userspace/falco/falco.cpp +++ b/userspace/falco/falco.cpp @@ -1165,7 +1165,7 @@ int falco_init(int argc, char **argv) // grpc server // TODO: this is blocking now, not what we want, falco must go on. Just an experiment for now. - start_grpc_server(5060, 1); + start_grpc_server("0.0.0.0:5060", 1); if(!trace_filename.empty() && !trace_is_scap) { From 54906e89cc84ea5051fae919c59a1a03ecaa3fe6 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Tue, 3 Sep 2019 14:47:08 +0000 Subject: [PATCH 08/76] build(userspace/falco): add grpc context impl to cmake Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/userspace/falco/CMakeLists.txt b/userspace/falco/CMakeLists.txt index 2ef08e8cead..93f60427602 100644 --- a/userspace/falco/CMakeLists.txt +++ b/userspace/falco/CMakeLists.txt @@ -39,6 +39,7 @@ add_executable(falco falco.cpp "${SYSDIG_DIR}/userspace/sysdig/fields_info.cpp" webserver.cpp + grpc_context.cpp grpc_server.cpp ${CMAKE_CURRENT_BINARY_DIR}/falco_output.grpc.pb.cc ${CMAKE_CURRENT_BINARY_DIR}/falco_output.pb.cc) From 81beee7f02ac92c6550c9ccd748893e6f7cfac33 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Tue, 3 Sep 2019 14:47:57 +0000 Subject: [PATCH 09/76] fix(userspace/falco): service name for the proto falco output svc Signed-off-by: Leonardo Di Donato --- userspace/falco/falco_output.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/userspace/falco/falco_output.proto b/userspace/falco/falco_output.proto index 655dc797595..1add401e3fb 100644 --- a/userspace/falco/falco_output.proto +++ b/userspace/falco/falco_output.proto @@ -3,7 +3,7 @@ syntax = "proto3"; import "google/protobuf/timestamp.proto"; service falco_output_service { - rpc Subscribe(falco_output_request) returns (stream falco_output_response); + rpc subscribe(falco_output_request) returns (stream falco_output_response); } message falco_output_request { From 092dd0536ed6b3c57aec17b3d65cbb0a9f0d9d48 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Tue, 3 Sep 2019 14:49:08 +0000 Subject: [PATCH 10/76] chore(userspace/falco): grpc context format Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/grpc_context.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/userspace/falco/grpc_context.h b/userspace/falco/grpc_context.h index dc471ccf5e4..07021792bcf 100644 --- a/userspace/falco/grpc_context.h +++ b/userspace/falco/grpc_context.h @@ -39,10 +39,10 @@ class context class stream_context : public context { public: - stream_context(grpc::ServerContext* ctx): context(ctx) {} + stream_context(grpc::ServerContext* ctx): + context(ctx){}; ~stream_context() = default; -private: enum : char { STREAMING = 1, @@ -53,4 +53,4 @@ class stream_context : public context mutable void* m_stream = nullptr; // Are there more responses to stream? mutable bool m_has_more = false; -}; +}; \ No newline at end of file From 8d0336a051a26274ab907020ea233ef7650bae8d Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Tue, 3 Sep 2019 14:50:18 +0000 Subject: [PATCH 11/76] chore(userspace/falco): rename grpc server into falco grpc server Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/grpc_server.h | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/userspace/falco/grpc_server.h b/userspace/falco/grpc_server.h index df1ff86adcc..888164576e5 100644 --- a/userspace/falco/grpc_server.h +++ b/userspace/falco/grpc_server.h @@ -25,39 +25,42 @@ limitations under the License. #include "falco_output.pb.h" #include "grpc_context.h" -class grpc_server_impl +class falco_grpc_server_impl { public: - grpc_server_impl() = default; - ~grpc_server_impl() = default; + falco_grpc_server_impl() = default; + ~falco_grpc_server_impl() = default; protected: bool is_running(); - void subscribe_handler(const stream_context& ctx, falco_output_request req, falco_output_response res); + void subscribe(const stream_context& ctx, const falco_output_request& req, falco_output_response& res); }; -class grpc_server : public grpc_server_impl +class falco_grpc_server : public falco_grpc_server_impl { public: - grpc_server(std::string server_addr, int threadiness): + falco_grpc_server(std::string server_addr, int threadiness): m_server_addr(server_addr), m_threadiness(threadiness) { } - virtual ~grpc_server() = default; + virtual ~falco_grpc_server() = default; void thread_process(int thread_index); void run(); - void subscribe_handler(const stream_context& ctx, falco_output_request req, falco_output_response res); + + falco_output_service::AsyncService m_svc; + std::unique_ptr m_completion_queue; private: - // falco_output_service::AsyncService falco_output_svc; std::unique_ptr m_server; std::string m_server_addr; int m_threadiness = 0; - std::unique_ptr m_completion_queue; std::vector m_threads; + }; void start_grpc_server(std::string server_address, int threadiness); + + From 183e92c40981dd381ead695ebdfad94449a3b649 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Tue, 3 Sep 2019 14:51:41 +0000 Subject: [PATCH 12/76] new(userspace/falco): introduce request context base and request stream context classes Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/grpc_server.h | 45 +++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/userspace/falco/grpc_server.h b/userspace/falco/grpc_server.h index 888164576e5..560b8d97c35 100644 --- a/userspace/falco/grpc_server.h +++ b/userspace/falco/grpc_server.h @@ -63,4 +63,49 @@ class falco_grpc_server : public falco_grpc_server_impl void start_grpc_server(std::string server_address, int threadiness); +class request_context_base +{ +public: + request_context_base() = default; + ~request_context_base() = default; + + std::unique_ptr m_srv_ctx; + enum : char + { + UNKNOWN = 0, + REQUEST, + WRITE, + FINISH + } m_state = UNKNOWN; + virtual void start(falco_grpc_server* srv) = 0; + virtual void process(falco_grpc_server* srv) = 0; + virtual void end(falco_grpc_server* srv, bool isError) = 0; +}; + +// +// Template class to handle streaming responses +// +template +class request_stream_context : public request_context_base +{ +public: + request_stream_context(): + m_process_func(nullptr), + m_request_func(nullptr){}; + ~request_stream_context() = default; + + // Pointer to function that does actual processing + void (falco_grpc_server::*m_process_func)(const stream_context&, const Request&, Response&); + // Pointer to function that requests the system to start processing given requests + void (falco_output_service::AsyncService::*m_request_func)(grpc::ServerContext*, Request*, grpc::ServerAsyncWriter*, grpc::CompletionQueue*, grpc::ServerCompletionQueue*, void*); + + void start(falco_grpc_server* srv); + void process(falco_grpc_server* srv); + void end(falco_grpc_server* srv, bool isError); + +private: + std::unique_ptr> m_res_writer; + std::unique_ptr m_stream_ctx; + Request m_req; +}; \ No newline at end of file From d80ac96d17845c5a641205dd935c45329d9961d7 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Tue, 3 Sep 2019 14:52:54 +0000 Subject: [PATCH 13/76] new(usperspace/falco): request stream context specialization and process stream macro Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/grpc_server.cpp | 73 +++++++++++++++++++++++++++------ 1 file changed, 61 insertions(+), 12 deletions(-) diff --git a/userspace/falco/grpc_server.cpp b/userspace/falco/grpc_server.cpp index 5530bb4b0ea..1b904293fb2 100644 --- a/userspace/falco/grpc_server.cpp +++ b/userspace/falco/grpc_server.cpp @@ -27,13 +27,54 @@ limitations under the License. #include "grpc_server.h" #include "grpc_context.h" -bool grpc_server_impl::is_running() +template<> +void request_stream_context::start(falco_grpc_server* srv) +{ + m_state = request_context_base::REQUEST; + m_srv_ctx.reset(new grpc::ServerContext); + auto srvctx = m_srv_ctx.get(); + m_res_writer.reset(new grpc::ServerAsyncWriter(srvctx)); + m_stream_ctx.reset(); + m_req.Clear(); + + auto cq = srv->m_completion_queue.get(); + (srv->m_svc.*m_request_func)(srvctx, &m_req, m_res_writer.get(), cq, cq, this); +} + +template<> +void request_stream_context::process(falco_grpc_server* srv) +{ +} + +template<> +void request_stream_context::end(falco_grpc_server* srv, bool isError) +{ +} + +bool falco_grpc_server_impl::is_running() { // TODO: this must act as a switch to shut down the server return true; } -void grpc_server::thread_process(int thread_index) +void falco_grpc_server_impl::subscribe(const stream_context& ctx, const falco_output_request& req, falco_output_response& res) +{ + if(ctx.m_status == stream_context::SUCCESS || ctx.m_status == stream_context::ERROR) + { + // todo > logic + + ctx.m_stream = nullptr; + } + else + { + // Start (or continue) streaming + // ctx.m_status == stream_context::STREAMING + } + + // todo > print/store statistics +} + +void falco_grpc_server::thread_process(int thread_index) { // TODO: is this right? That's what we want? // Tell pthread to not handle termination signals in the current thread @@ -55,7 +96,19 @@ void grpc_server::thread_process(int thread_index) } } -void grpc_server::run() +// +// Create array of contexts and start processing streaming RPC request. +// +#define PROCESS_STREAM(REQ, RESP, RPC, IMPL, CONTEXT_COUNT) \ + std::vector> RPC##_contexts(CONTEXT_COUNT); \ + for(request_stream_context & ctx : RPC##_contexts) \ + { \ + ctx.m_process_func = &falco_grpc_server::IMPL; \ + ctx.m_request_func = &falco_output_service::AsyncService::Request##RPC; \ + ctx.start(this); \ + } + +void falco_grpc_server::run() { grpc::ServerBuilder builder; // Listen on the given address without any authentication mechanism. @@ -66,14 +119,14 @@ void grpc_server::run() m_server = builder.BuildAndStart(); std::cout << "Server listening on " << m_server_addr << std::endl; - // int context_count = threadiness * 10; + int context_count = m_threadiness * 10; + PROCESS_STREAM(falco_output_request, falco_output_response, subscribe, subscribe, context_count) m_threads.resize(m_threadiness); - int thread_idx = 0; for(std::thread& thread : m_threads) { - thread = std::thread(&grpc_server::thread_process, this, thread_idx++); + thread = std::thread(&falco_grpc_server::thread_process, this, thread_idx++); } while(is_running()) @@ -81,12 +134,8 @@ void grpc_server::run() } } -void grpc_server::subscribe_handler(const stream_context& ctx, falco_output_request req, falco_output_response res) -{ -} - void start_grpc_server(std::string server_address, int threadiness) { - grpc_server srv(server_address, threadiness); + falco_grpc_server srv(server_address, threadiness); srv.run(); -} +} \ No newline at end of file From a563b1c4b6d1eecf0c8d85b941d5118c4066d609 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Tue, 3 Sep 2019 16:19:51 +0000 Subject: [PATCH 14/76] new(userspace/falco): request stream context process and end handling Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/grpc_server.cpp | 45 +++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/userspace/falco/grpc_server.cpp b/userspace/falco/grpc_server.cpp index 1b904293fb2..529ab71215d 100644 --- a/userspace/falco/grpc_server.cpp +++ b/userspace/falco/grpc_server.cpp @@ -44,11 +44,56 @@ void request_stream_context::start( template<> void request_stream_context::process(falco_grpc_server* srv) { + // When it is the 1st process call + if(m_state == request_context_base::REQUEST) + { + m_state = request_context_base::WRITE; + m_stream_ctx.reset(new stream_context(m_srv_ctx.get())); + } + + // Processing + falco_output_response res; + (srv->*m_process_func)(*m_stream_ctx, m_req, res); + + // When there still are more responses to stream + if(m_stream_ctx->m_has_more) + { + m_res_writer->Write(res, this); + } + // No more responses to stream + else + { + // Communicate to the gRPC runtime that we have finished. + // The memory address of `this` instance uniquely identifies the event. + m_state = request_context_base::FINISH; + m_res_writer->Finish(grpc::Status::OK, this); + } } template<> void request_stream_context::end(falco_grpc_server* srv, bool isError) { + if(m_stream_ctx) + { + m_stream_ctx->m_status = stream_context::SUCCESS; + if(isError) + { + m_stream_ctx->m_status = stream_context::ERROR; + // todo > log error + } + + // Complete the processing + falco_output_response res; + (srv->*m_process_func)(*m_stream_ctx, m_req, res); + } + else + { + // Handle the edge case when `m_request_func` event failed + // which means `m_stream_ctx` was not set + // todo > log error + } + + start(srv); } bool falco_grpc_server_impl::is_running() From 6d61ca5d8c4eea9d320258bac84204b6531e8e54 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Tue, 3 Sep 2019 16:47:55 +0000 Subject: [PATCH 15/76] new(userspace/falco): falco outputs grpc server stop Signed-off-by: Leonardo Di Donato --- userspace/falco/grpc_server.cpp | 37 ++++++++++++++++++++++++++++++++- userspace/falco/grpc_server.h | 2 +- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/userspace/falco/grpc_server.cpp b/userspace/falco/grpc_server.cpp index 529ab71215d..5b54617957b 100644 --- a/userspace/falco/grpc_server.cpp +++ b/userspace/falco/grpc_server.cpp @@ -24,6 +24,8 @@ limitations under the License. #include #endif +#include // sleep + #include "grpc_server.h" #include "grpc_context.h" @@ -113,9 +115,16 @@ void falco_grpc_server_impl::subscribe(const stream_context& ctx, const falco_ou else { // Start (or continue) streaming + // ctx.m_status == stream_context::STREAMING - } + // todo > do we want batching? + + res.set_source(source::SYSCALL); + res.set_rule("regola 1"); + + ctx.m_has_more = false; + } // todo > print/store statistics } @@ -175,6 +184,32 @@ void falco_grpc_server::run() } while(is_running()) + { + sleep(1); + } + + stop(); +} + +void falco_grpc_server::stop() +{ + m_server->Shutdown(); + m_completion_queue->Shutdown(); + + // todo > log "waiting for the server threads to complete" + + for(std::thread& t : m_threads) + { + t.join(); + } + m_threads.clear(); + + // todo > log "all server threads complete" + + // Ignore remaining events + void* ignore_tag = nullptr; + bool ignore_ok = false; + while(m_completion_queue->Next(&ignore_tag, &ignore_ok)) { } } diff --git a/userspace/falco/grpc_server.h b/userspace/falco/grpc_server.h index 560b8d97c35..cf5445f8722 100644 --- a/userspace/falco/grpc_server.h +++ b/userspace/falco/grpc_server.h @@ -49,6 +49,7 @@ class falco_grpc_server : public falco_grpc_server_impl void thread_process(int thread_index); void run(); + void stop(); falco_output_service::AsyncService m_svc; std::unique_ptr m_completion_queue; @@ -58,7 +59,6 @@ class falco_grpc_server : public falco_grpc_server_impl std::string m_server_addr; int m_threadiness = 0; std::vector m_threads; - }; void start_grpc_server(std::string server_address, int threadiness); From 3e1f48d7562e7d6b716113a835a4f750d747afab Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Tue, 3 Sep 2019 18:20:16 +0000 Subject: [PATCH 16/76] fix: grpc service must be registered and grpc context state must be handled for threads Signed-off-by: Leonardo Di Donato --- userspace/falco/grpc_server.cpp | 63 ++++++++++++++++++++++++++------- userspace/falco/grpc_server.h | 4 +++ 2 files changed, 55 insertions(+), 12 deletions(-) diff --git a/userspace/falco/grpc_server.cpp b/userspace/falco/grpc_server.cpp index 5b54617957b..0848cf92656 100644 --- a/userspace/falco/grpc_server.cpp +++ b/userspace/falco/grpc_server.cpp @@ -16,16 +16,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -#include - #ifdef GRPC_INCLUDE_IS_GRPCPP #include #else #include #endif - #include // sleep +#include "logger.h" #include "grpc_server.h" #include "grpc_context.h" @@ -100,7 +98,10 @@ void request_stream_context::end(fa bool falco_grpc_server_impl::is_running() { - // TODO: this must act as a switch to shut down the server + if(m_stop) + { + return false; + } return true; } @@ -115,15 +116,14 @@ void falco_grpc_server_impl::subscribe(const stream_context& ctx, const falco_ou else { // Start (or continue) streaming - // ctx.m_status == stream_context::STREAMING // todo > do we want batching? - + sleep(15); res.set_source(source::SYSCALL); - res.set_rule("regola 1"); + res.set_rule("rule X"); - ctx.m_has_more = false; + ctx.m_has_more = true; } // todo > print/store statistics } @@ -144,10 +144,46 @@ void falco_grpc_server::thread_process(int thread_index) { if(tag == nullptr) { - // TODO: empty tag returned, log, what to do? + // TODO: empty tag returned, log "completion queue with empty tag" continue; } + + // Obtain the context for a given tag + request_context_base* ctx = static_cast(tag); + + // When event has not been read successfully + if(!event_read_success) + { + if(ctx->m_state != request_context_base::REQUEST) + { + // todo > log "server completion queue failed to read event for tag `tag`" + // End the context with error + ctx->end(this, true); + } + continue; + } + + // Process the event + switch(ctx->m_state) + { + case request_context_base::REQUEST: + // Completion of m_request_func + case request_context_base::WRITE: + // Completion of ServerAsyncWriter::Write() + ctx->process(this); + break; + case request_context_base::FINISH: + // Completion of ServerAsyncWriter::Finish() + ctx->end(this, false); + + default: + // todo > log "unkown completion queue event" + // todo > abort? + break; + } } + + // todo > log "thread `thread_index` complete" } // @@ -164,16 +200,19 @@ void falco_grpc_server::thread_process(int thread_index) void falco_grpc_server::run() { + // Setup server grpc::ServerBuilder builder; // Listen on the given address without any authentication mechanism. builder.AddListeningPort(m_server_addr, grpc::InsecureServerCredentials()); - // builder.RegisterService(&falco_output_svc); // TODO: enable this when we do the impl + builder.RegisterService(&m_svc); + // builder.SetMaxSendMessageSize(GRPC_MAX_MESSAGE_SIZE); // testing max message size? + // builder.SetMaxReceiveMessageSize(GRPC_MAX_MESSAGE_SIZE); // testing max message size? m_completion_queue = builder.AddCompletionQueue(); m_server = builder.BuildAndStart(); - std::cout << "Server listening on " << m_server_addr << std::endl; + falco_logger::log(LOG_INFO, "Starting gRPC webserver at " + m_server_addr + "\n"); - int context_count = m_threadiness * 10; + int context_count = m_threadiness * 1; // todo > 10 or 100? PROCESS_STREAM(falco_output_request, falco_output_response, subscribe, subscribe, context_count) m_threads.resize(m_threadiness); diff --git a/userspace/falco/grpc_server.h b/userspace/falco/grpc_server.h index cf5445f8722..26254a8390e 100644 --- a/userspace/falco/grpc_server.h +++ b/userspace/falco/grpc_server.h @@ -20,6 +20,7 @@ limitations under the License. #include #include +#include #include "falco_output.grpc.pb.h" #include "falco_output.pb.h" @@ -35,6 +36,9 @@ class falco_grpc_server_impl bool is_running(); void subscribe(const stream_context& ctx, const falco_output_request& req, falco_output_response& res); + +private: + std::atomic m_stop{false}; }; class falco_grpc_server : public falco_grpc_server_impl From cb5c5c55531c8a9dd7c641150800377dd9e891a0 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Wed, 4 Sep 2019 10:37:01 +0000 Subject: [PATCH 17/76] chore(userspace/falco): gRPC server send rule and source Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/grpc_server.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/userspace/falco/grpc_server.cpp b/userspace/falco/grpc_server.cpp index 0848cf92656..a23cdc62c9d 100644 --- a/userspace/falco/grpc_server.cpp +++ b/userspace/falco/grpc_server.cpp @@ -119,9 +119,16 @@ void falco_grpc_server_impl::subscribe(const stream_context& ctx, const falco_ou // ctx.m_status == stream_context::STREAMING // todo > do we want batching? - sleep(15); + std::stringstream ss; + int c = 0; + int i = 9; + while(c < i) + { + ss << std::to_string(c); + c++; + } res.set_source(source::SYSCALL); - res.set_rule("rule X"); + res.set_rule(ss.str()); ctx.m_has_more = true; } From 7015b68309ab041d6626c20d6fc86aa200f50b8d Mon Sep 17 00:00:00 2001 From: Lorenzo Fontana Date: Wed, 4 Sep 2019 18:34:05 +0200 Subject: [PATCH 18/76] new(userspace/falco): grpc server event bus queue Co-Authored-By: Leonardo Di Donato Signed-off-by: Lorenzo Fontana --- userspace/falco/grpc_server.cpp | 14 +++++++++++--- userspace/falco/grpc_server.h | 22 +++++++++++++++++++--- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/userspace/falco/grpc_server.cpp b/userspace/falco/grpc_server.cpp index a23cdc62c9d..39efea78524 100644 --- a/userspace/falco/grpc_server.cpp +++ b/userspace/falco/grpc_server.cpp @@ -119,6 +119,7 @@ void falco_grpc_server_impl::subscribe(const stream_context& ctx, const falco_ou // ctx.m_status == stream_context::STREAMING // todo > do we want batching? +<<<<<<< HEAD std::stringstream ss; int c = 0; int i = 9; @@ -129,7 +130,13 @@ void falco_grpc_server_impl::subscribe(const stream_context& ctx, const falco_ou } res.set_source(source::SYSCALL); res.set_rule(ss.str()); +======= +>>>>>>> new(userspace/falco): grpc server event bus queue + if(!m_event_queue.try_pop(res)) + { + // TODO: log that we've not been able to pop? + } ctx.m_has_more = true; } // todo > print/store statistics @@ -260,8 +267,9 @@ void falco_grpc_server::stop() } } -void start_grpc_server(std::string server_address, int threadiness) +bool start_grpc_server(std::string server_address, int threadiness, falco_output_response_cq& output_event_queue) { - falco_grpc_server srv(server_address, threadiness); + falco_grpc_server srv(server_address, threadiness, output_event_queue); srv.run(); -} \ No newline at end of file + return true; +} diff --git a/userspace/falco/grpc_server.h b/userspace/falco/grpc_server.h index 26254a8390e..37f5c3c3a41 100644 --- a/userspace/falco/grpc_server.h +++ b/userspace/falco/grpc_server.h @@ -21,17 +21,31 @@ limitations under the License. #include #include #include +#include + +#include "tbb/concurrent_queue.h" #include "falco_output.grpc.pb.h" #include "falco_output.pb.h" #include "grpc_context.h" +using namespace tbb; + +typedef concurrent_queue falco_output_response_cq; + class falco_grpc_server_impl { public: falco_grpc_server_impl() = default; ~falco_grpc_server_impl() = default; + falco_output_response_cq& m_event_queue; + + falco_grpc_server_impl(falco_output_response_cq& event_queue): + m_event_queue(event_queue) + { + } + protected: bool is_running(); @@ -44,11 +58,13 @@ class falco_grpc_server_impl class falco_grpc_server : public falco_grpc_server_impl { public: - falco_grpc_server(std::string server_addr, int threadiness): + falco_grpc_server(std::string server_addr, int threadiness, falco_output_response_cq& m_event_queue): + falco_grpc_server_impl(m_event_queue), m_server_addr(server_addr), m_threadiness(threadiness) { } + virtual ~falco_grpc_server() = default; void thread_process(int thread_index); @@ -65,7 +81,7 @@ class falco_grpc_server : public falco_grpc_server_impl std::vector m_threads; }; -void start_grpc_server(std::string server_address, int threadiness); +bool start_grpc_server(std::string server_address, int threadiness, falco_output_response_cq& output_event_queue); class request_context_base { @@ -112,4 +128,4 @@ class request_stream_context : public request_context_base std::unique_ptr> m_res_writer; std::unique_ptr m_stream_ctx; Request m_req; -}; \ No newline at end of file +}; From b601d142dfc890a7521944d802e2010e9af7e612 Mon Sep 17 00:00:00 2001 From: Lorenzo Fontana Date: Wed, 4 Sep 2019 18:37:00 +0200 Subject: [PATCH 19/76] new(userspace/falco): initial grpc queuing logic Co-Authored-By: Leonardo Di Donato Signed-off-by: Lorenzo Fontana --- userspace/falco/falco.cpp | 32 +++++++++++++++++++++----------- userspace/falco/grpc_server.cpp | 13 ------------- 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/userspace/falco/falco.cpp b/userspace/falco/falco.cpp index 46315466201..617c986a828 100644 --- a/userspace/falco/falco.cpp +++ b/userspace/falco/falco.cpp @@ -34,6 +34,7 @@ limitations under the License. #include #include + #include #include "logger.h" @@ -221,15 +222,16 @@ static std::string read_file(std::string filename) // Event processing loop // uint64_t do_inspect(falco_engine *engine, - falco_outputs *outputs, - sinsp* inspector, - falco_configuration &config, - syscall_evt_drop_mgr &sdropmgr, - uint64_t duration_to_tot_ns, - string &stats_filename, - uint64_t stats_interval, - bool all_events, - int &result) + falco_outputs *outputs, + sinsp* inspector, + falco_configuration &config, + syscall_evt_drop_mgr &sdropmgr, + uint64_t duration_to_tot_ns, + string &stats_filename, + uint64_t stats_interval, + bool all_events, + falco_output_response_cq &output_event_queue, + int &result) { uint64_t num_evts = 0; int32_t rc; @@ -324,6 +326,11 @@ uint64_t do_inspect(falco_engine *engine, if(res) { outputs->handle_event(res->evt, res->rule, res->source, res->priority_num, res->format); + + // TODO(fntlnz): integrate this with the output handle event logic, for now we just want to test the threads + falco_output_response grpcres = falco_output_response(); + grpcres.set_rule(res->rule); + output_event_queue.push(grpcres); } num_evts++; @@ -1164,8 +1171,10 @@ int falco_init(int argc, char **argv) } // grpc server - // TODO: this is blocking now, not what we want, falco must go on. Just an experiment for now. - start_grpc_server("0.0.0.0:5060", 1); + falco_output_response_cq output_event_queue; + + // TODO(fntlnz): do any handling, make sure we handle signals in the GRPC server and we clean it gracefully + std::thread grpc_server_thread (start_grpc_server, "0.0.0.0:5060", 1, std::ref(output_event_queue)); if(!trace_filename.empty() && !trace_is_scap) { @@ -1186,6 +1195,7 @@ int falco_init(int argc, char **argv) stats_filename, stats_interval, all_events, + output_event_queue, result); duration = ((double)clock()) / CLOCKS_PER_SEC - duration; diff --git a/userspace/falco/grpc_server.cpp b/userspace/falco/grpc_server.cpp index 39efea78524..8672dfbebc9 100644 --- a/userspace/falco/grpc_server.cpp +++ b/userspace/falco/grpc_server.cpp @@ -119,19 +119,6 @@ void falco_grpc_server_impl::subscribe(const stream_context& ctx, const falco_ou // ctx.m_status == stream_context::STREAMING // todo > do we want batching? -<<<<<<< HEAD - std::stringstream ss; - int c = 0; - int i = 9; - while(c < i) - { - ss << std::to_string(c); - c++; - } - res.set_source(source::SYSCALL); - res.set_rule(ss.str()); -======= ->>>>>>> new(userspace/falco): grpc server event bus queue if(!m_event_queue.try_pop(res)) { From 318b0398f408c1ff0db6459b4a5687ae071e4576 Mon Sep 17 00:00:00 2001 From: Lorenzo Fontana Date: Thu, 5 Sep 2019 12:04:18 +0200 Subject: [PATCH 20/76] update(userspace/falco): keep the stream open Co-Authored-By: Leonardo Di Donato Signed-off-by: Lorenzo Fontana --- userspace/falco/grpc_server.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/userspace/falco/grpc_server.cpp b/userspace/falco/grpc_server.cpp index 8672dfbebc9..715ae56fcc3 100644 --- a/userspace/falco/grpc_server.cpp +++ b/userspace/falco/grpc_server.cpp @@ -84,7 +84,7 @@ void request_stream_context::end(fa // Complete the processing falco_output_response res; - (srv->*m_process_func)(*m_stream_ctx, m_req, res); + (srv->*m_process_func)(*m_stream_ctx, m_req, res); // subscribe() } else { @@ -117,16 +117,15 @@ void falco_grpc_server_impl::subscribe(const stream_context& ctx, const falco_ou { // Start (or continue) streaming // ctx.m_status == stream_context::STREAMING - // todo > do we want batching? - + dequeue: if(!m_event_queue.try_pop(res)) { - // TODO: log that we've not been able to pop? + // TODO: backoff mechanism that does has more false + goto dequeue; } ctx.m_has_more = true; } - // todo > print/store statistics } void falco_grpc_server::thread_process(int thread_index) From 440772cf6ffc0bf0aeb4795b2da18f407e7f0cec Mon Sep 17 00:00:00 2001 From: Lorenzo Fontana Date: Thu, 5 Sep 2019 13:42:47 +0200 Subject: [PATCH 21/76] new(userspace/falco): subscribe keepalive parameter Co-Authored-By: Leonardo Di Donato Signed-off-by: Lorenzo Fontana --- userspace/falco/falco_output.proto | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/userspace/falco/falco_output.proto b/userspace/falco/falco_output.proto index 1add401e3fb..7719f5275db 100644 --- a/userspace/falco/falco_output.proto +++ b/userspace/falco/falco_output.proto @@ -7,6 +7,8 @@ service falco_output_service { } message falco_output_request { + string duration = 1; // TODO(leodido, fntlnz): not handled yet but keeping for reference. + bool keepalive = 2; } enum priority { @@ -34,4 +36,4 @@ message falco_output_response { string format = 6; string output = 7; map output_fields = 8; -} \ No newline at end of file +} From b124b743fe22229ee13a19da7e3376ad48abefc9 Mon Sep 17 00:00:00 2001 From: Lorenzo Fontana Date: Thu, 5 Sep 2019 13:43:25 +0200 Subject: [PATCH 22/76] new(userspace/falco): handle subscribe events as streams based on keepalive Co-Authored-By: Leonardo Di Donato Signed-off-by: Lorenzo Fontana --- userspace/falco/grpc_server.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/userspace/falco/grpc_server.cpp b/userspace/falco/grpc_server.cpp index 715ae56fcc3..56f2c7d7b62 100644 --- a/userspace/falco/grpc_server.cpp +++ b/userspace/falco/grpc_server.cpp @@ -117,14 +117,16 @@ void falco_grpc_server_impl::subscribe(const stream_context& ctx, const falco_ou { // Start (or continue) streaming // ctx.m_status == stream_context::STREAMING - // todo > do we want batching? - dequeue: - if(!m_event_queue.try_pop(res)) + if(m_event_queue.try_pop(res) && !req.keepalive()) { - // TODO: backoff mechanism that does has more false - goto dequeue; + ctx.m_has_more = true; + return; } - ctx.m_has_more = true; + while(!m_event_queue.try_pop(res) && req.keepalive()) + { + } + + ctx.m_has_more = req.keepalive(); } } From 640bb3f0d23ec6ce16361dde60fb731e23af5d15 Mon Sep 17 00:00:00 2001 From: Lorenzo Fontana Date: Thu, 5 Sep 2019 13:44:22 +0200 Subject: [PATCH 23/76] new(userspace/falco): specify that we can have multiple client consuming mechanisms, only round robin fashion is implemented now Co-Authored-By: Leonardo Di Donato Signed-off-by: Lorenzo Fontana --- userspace/falco/falco.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/userspace/falco/falco.cpp b/userspace/falco/falco.cpp index 617c986a828..f3692aff84d 100644 --- a/userspace/falco/falco.cpp +++ b/userspace/falco/falco.cpp @@ -1173,8 +1173,12 @@ int falco_init(int argc, char **argv) // grpc server falco_output_response_cq output_event_queue; + // TODO(fntlnz,leodido): when we want to spawn multiple threads we need to have a queue per thread, or implement + // different queuing mechanisms, round robin, fanout? What we want to achieve? + int threadiness = 1; // TODO(fntlnz, leodido): make this configurable + // TODO(fntlnz): do any handling, make sure we handle signals in the GRPC server and we clean it gracefully - std::thread grpc_server_thread (start_grpc_server, "0.0.0.0:5060", 1, std::ref(output_event_queue)); + std::thread grpc_server_thread (start_grpc_server, "0.0.0.0:5060", threadiness, std::ref(output_event_queue)); if(!trace_filename.empty() && !trace_is_scap) { From 4898a1fff97960ee53cff1cd099a5888be97af56 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Fri, 6 Sep 2019 09:18:06 +0000 Subject: [PATCH 24/76] build(userspace/falco): prepare grpc server implementation to be moved Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/userspace/falco/CMakeLists.txt b/userspace/falco/CMakeLists.txt index 93f60427602..7b018446cba 100644 --- a/userspace/falco/CMakeLists.txt +++ b/userspace/falco/CMakeLists.txt @@ -40,6 +40,7 @@ add_executable(falco "${SYSDIG_DIR}/userspace/sysdig/fields_info.cpp" webserver.cpp grpc_context.cpp + grpc_server_impl.cpp grpc_server.cpp ${CMAKE_CURRENT_BINARY_DIR}/falco_output.grpc.pb.cc ${CMAKE_CURRENT_BINARY_DIR}/falco_output.pb.cc) From e281d179e4b272409331eb2fea55e9ba87bae0cc Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Fri, 6 Sep 2019 09:18:44 +0000 Subject: [PATCH 25/76] chore(usperspace/falco): move grpc server impl Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/grpc_server.cpp | 32 ----------------- userspace/falco/grpc_server.h | 35 ++---------------- userspace/falco/grpc_server_impl.cpp | 53 ++++++++++++++++++++++++++++ userspace/falco/grpc_server_impl.h | 50 ++++++++++++++++++++++++++ 4 files changed, 105 insertions(+), 65 deletions(-) create mode 100644 userspace/falco/grpc_server_impl.cpp create mode 100644 userspace/falco/grpc_server_impl.h diff --git a/userspace/falco/grpc_server.cpp b/userspace/falco/grpc_server.cpp index 56f2c7d7b62..bdaec631e47 100644 --- a/userspace/falco/grpc_server.cpp +++ b/userspace/falco/grpc_server.cpp @@ -96,39 +96,7 @@ void request_stream_context::end(fa start(srv); } -bool falco_grpc_server_impl::is_running() -{ - if(m_stop) - { - return false; - } - return true; -} - -void falco_grpc_server_impl::subscribe(const stream_context& ctx, const falco_output_request& req, falco_output_response& res) -{ - if(ctx.m_status == stream_context::SUCCESS || ctx.m_status == stream_context::ERROR) - { - // todo > logic - - ctx.m_stream = nullptr; - } - else - { - // Start (or continue) streaming - // ctx.m_status == stream_context::STREAMING - if(m_event_queue.try_pop(res) && !req.keepalive()) - { - ctx.m_has_more = true; - return; - } - while(!m_event_queue.try_pop(res) && req.keepalive()) - { - } - ctx.m_has_more = req.keepalive(); - } -} void falco_grpc_server::thread_process(int thread_index) { diff --git a/userspace/falco/grpc_server.h b/userspace/falco/grpc_server.h index 37f5c3c3a41..2c89c0e97fb 100644 --- a/userspace/falco/grpc_server.h +++ b/userspace/falco/grpc_server.h @@ -20,40 +20,10 @@ limitations under the License. #include #include -#include -#include - -#include "tbb/concurrent_queue.h" - -#include "falco_output.grpc.pb.h" -#include "falco_output.pb.h" -#include "grpc_context.h" - -using namespace tbb; - -typedef concurrent_queue falco_output_response_cq; - -class falco_grpc_server_impl -{ -public: - falco_grpc_server_impl() = default; - ~falco_grpc_server_impl() = default; - - falco_output_response_cq& m_event_queue; - falco_grpc_server_impl(falco_output_response_cq& event_queue): - m_event_queue(event_queue) - { - } - -protected: - bool is_running(); - - void subscribe(const stream_context& ctx, const falco_output_request& req, falco_output_response& res); +#include -private: - std::atomic m_stop{false}; -}; +#include "grpc_server_impl.h" class falco_grpc_server : public falco_grpc_server_impl { @@ -64,7 +34,6 @@ class falco_grpc_server : public falco_grpc_server_impl m_threadiness(threadiness) { } - virtual ~falco_grpc_server() = default; void thread_process(int thread_index); diff --git a/userspace/falco/grpc_server_impl.cpp b/userspace/falco/grpc_server_impl.cpp new file mode 100644 index 00000000000..f89082a6744 --- /dev/null +++ b/userspace/falco/grpc_server_impl.cpp @@ -0,0 +1,53 @@ +/* +Copyright (C) 2016-2019 The Falco Authors + +This file is part of falco. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#include "grpc_server_impl.h" + +bool falco_grpc_server_impl::is_running() +{ + if(m_stop) + { + return false; + } + return true; +} + +void falco_grpc_server_impl::subscribe(const stream_context& ctx, const falco_output_request& req, falco_output_response& res) +{ + if(ctx.m_status == stream_context::SUCCESS || ctx.m_status == stream_context::ERROR) + { + // todo > logic + + ctx.m_stream = nullptr; + } + else + { + // Start (or continue) streaming + // ctx.m_status == stream_context::STREAMING + if(m_event_queue.try_pop(res) && !req.keepalive()) + { + ctx.m_has_more = true; + return; + } + while(!m_event_queue.try_pop(res) && req.keepalive()) + { + } + + ctx.m_has_more = req.keepalive(); + } +} \ No newline at end of file diff --git a/userspace/falco/grpc_server_impl.h b/userspace/falco/grpc_server_impl.h new file mode 100644 index 00000000000..7b169b53aed --- /dev/null +++ b/userspace/falco/grpc_server_impl.h @@ -0,0 +1,50 @@ +/* +Copyright (C) 2016-2019 The Falco Authors + +This file is part of falco. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#pragma once + +#include + +#include "tbb/concurrent_queue.h" +#include "falco_output.grpc.pb.h" +#include "falco_output.pb.h" +#include "grpc_context.h" + +typedef tbb::concurrent_queue falco_output_response_cq; + +class falco_grpc_server_impl +{ +public: + falco_grpc_server_impl() = default; + ~falco_grpc_server_impl() = default; + + falco_output_response_cq& m_event_queue; + + falco_grpc_server_impl(falco_output_response_cq& event_queue): + m_event_queue(event_queue) + { + } + +protected: + bool is_running(); + + void subscribe(const stream_context& ctx, const falco_output_request& req, falco_output_response& res); + +private: + std::atomic m_stop{false}; +}; \ No newline at end of file From 004fa3b276a9980cec796a0204847d331d1002f9 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Fri, 6 Sep 2019 14:00:45 +0000 Subject: [PATCH 26/76] new: falco config for grpc server and outputs Co-Authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- falco.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/falco.yaml b/falco.yaml index af3b254d979..428fdd8c802 100644 --- a/falco.yaml +++ b/falco.yaml @@ -166,4 +166,11 @@ program_output: http_output: enabled: false - url: http://some.url \ No newline at end of file + url: http://some.url + +grpc: + enabled: true + bind_address: "0.0.0.0:5060" + +grpc_output: + enabled: true \ No newline at end of file From 08c8aed8deb3dec08e074846fce74d0478436531 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Fri, 6 Sep 2019 14:02:02 +0000 Subject: [PATCH 27/76] update(userspace/falco): grab grpc output configuration Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/engine/falco_common.h | 2 +- userspace/falco/configuration.cpp | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/userspace/engine/falco_common.h b/userspace/engine/falco_common.h index 99d4b3ce3b6..f304b6e5720 100644 --- a/userspace/engine/falco_common.h +++ b/userspace/engine/falco_common.h @@ -75,7 +75,7 @@ class falco_common void set_inspector(sinsp *inspector); - // Priority levels, as a vector of strings + // Priority levels, as a vector of strings static std::vector priority_names; // Same as numbers/indices into the above vector diff --git a/userspace/falco/configuration.cpp b/userspace/falco/configuration.cpp index 707727bbd4b..842ae9c90dc 100644 --- a/userspace/falco/configuration.cpp +++ b/userspace/falco/configuration.cpp @@ -148,6 +148,14 @@ void falco_configuration::init(string conf_filename, list &cmdline_optio m_outputs.push_back(http_output); } + falco_outputs::output_config grpc_output; + grpc_output.name = "grpc"; + if(m_config->get_scalar("grpc_output", "enabled", false)) + { + // todo > grpc_output is enabled but we should constraint it to the grpc server being enabled too + m_outputs.push_back(grpc_output); + } + if (m_outputs.size() == 0) { throw invalid_argument("Error reading config file (" + m_config_file + "): No outputs configured. Please configure at least one output file output enabled but no filename in configuration block"); From b531e0c786f94a04bebcd484430dd53abfcd0c14 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Fri, 6 Sep 2019 14:02:55 +0000 Subject: [PATCH 28/76] new(userspace/falco): introducing concurrent queue for falco outputs Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/falco_output_queue.h | 51 ++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 userspace/falco/falco_output_queue.h diff --git a/userspace/falco/falco_output_queue.h b/userspace/falco/falco_output_queue.h new file mode 100644 index 00000000000..09f9ec91046 --- /dev/null +++ b/userspace/falco/falco_output_queue.h @@ -0,0 +1,51 @@ +/* +Copyright (C) 2016-2019 The Falco Authors + +This file is part of falco. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#pragma once + +#include "falco_output.pb.h" +#include "tbb/concurrent_queue.h" + +typedef tbb::concurrent_queue falco_output_response_cq; + +class falco_output_queue +{ +public: + static falco_output_queue& get() + { + static falco_output_queue instance; + return instance; + } + + falco_output_response_cq queue() + { + return m_queue; + } + +private: + falco_output_queue() + { + } + + falco_output_response_cq m_queue; + + // We can use the better technique of deleting the methods we don't want. +public: + falco_output_queue(falco_output_queue const&) = delete; + void operator=(falco_output_queue const&) = delete; +}; \ No newline at end of file From 870f14759263000dd0c1523d3b2c724dd6f21209 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Fri, 6 Sep 2019 14:03:35 +0000 Subject: [PATCH 29/76] update(userspace/falco): remove format from grpc responses Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/falco_output.proto | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/userspace/falco/falco_output.proto b/userspace/falco/falco_output.proto index 7719f5275db..09b74902364 100644 --- a/userspace/falco/falco_output.proto +++ b/userspace/falco/falco_output.proto @@ -33,7 +33,6 @@ message falco_output_response { priority priority = 3; source source = 4; string rule = 5; - string format = 6; - string output = 7; - map output_fields = 8; -} + string output = 6; + map output_fields = 7; +} \ No newline at end of file From 76757c5e7ffca30de379e28b1645799c44fe9eda Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Fri, 6 Sep 2019 14:04:44 +0000 Subject: [PATCH 30/76] update(userspace/falco/lua): separate events output from messages outputs using different lua functions Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/lua/output.lua | 109 ++++++++++++++++++++++----------- 1 file changed, 73 insertions(+), 36 deletions(-) diff --git a/userspace/falco/lua/output.lua b/userspace/falco/lua/output.lua index e408ce53f8c..84fecfeb2bf 100644 --- a/userspace/falco/lua/output.lua +++ b/userspace/falco/lua/output.lua @@ -19,11 +19,15 @@ local mod = {} local outputs = {} -function mod.stdout(priority, priority_num, msg, options) +function mod.stdout(event, rule, source, priority, priority_num, msg, options) + mod.stdout_message(priority, priority_num, msg, outputs) +end + +function mod.stdout_message(priority, priority_num, msg, options) if options.buffered == 0 then - io.stdout:setvbuf 'no' + io.stdout:setvbuf "no" end - print (msg) + print(msg) end function mod.stdout_cleanup() @@ -35,28 +39,31 @@ function mod.stdout_reopen(options) end function mod.file_validate(options) - if (not type(options.filename) == 'string') then + if (not type(options.filename) == "string") then error("File output needs to be configured with a valid filename") end local file, err = io.open(options.filename, "a+") if file == nil then - error("Error with file output: "..err) + error("Error with file output: " .. err) end file:close() - end function mod.file_open(options) if ffile == nil then ffile = io.open(options.filename, "a+") if options.buffered == 0 then - ffile:setvbuf 'no' + ffile:setvbuf "no" end end end -function mod.file(priority, priority_num, msg, options) +function mod.file(event, rule, source, priority, priority_num, msg, options) + mod.file_message(priority, priority_num, msg, options) +end + +function mod.file_message(priority, priority_num, msg, options) if options.keep_alive == "true" then mod.file_open(options) else @@ -65,8 +72,7 @@ function mod.file(priority, priority_num, msg, options) ffile:write(msg, "\n") - if options.keep_alive == nil or - options.keep_alive ~= "true" then + if options.keep_alive == nil or options.keep_alive ~= "true" then ffile:close() ffile = nil end @@ -87,7 +93,11 @@ function mod.file_reopen(options) end end -function mod.syslog(priority, priority_num, msg, options) +function mod.syslog(event, rule, source, priority, priority_num, msg, options) + mod.syslog_message(priority, priority_num, msg, options) +end + +function mod.syslog_message(priority, priority_num, msg, options) falco.syslog(priority_num, msg) end @@ -101,12 +111,16 @@ function mod.program_open(options) if pfile == nil then pfile = io.popen(options.program, "w") if options.buffered == 0 then - pfile:setvbuf 'no' + pfile:setvbuf "no" end end end -function mod.program(priority, priority_num, msg, options) +function mod.program(event, rule, source, priority, priority_num, msg, options) + mod.program_message(priority, priority_num, msg, options) +end + +function mod.program_message(priority, priority_num, msg, options) -- XXX Ideally we'd check that the program ran -- successfully. However, the luajit we're using returns true even -- when the shell can't run the program. @@ -120,8 +134,7 @@ function mod.program(priority, priority_num, msg, options) pfile:write(msg, "\n") - if options.keep_alive == nil or - options.keep_alive ~= "true" then + if options.keep_alive == nil or options.keep_alive ~= "true" then pfile:close() pfile = nil end @@ -142,7 +155,11 @@ function mod.program_reopen(options) end end -function mod.http(priority, priority_num, msg, options) +function mod.http(event, rule, source, priority, priority_num, msg, options) + mod.http_message(priority, priority_num, msg, options) +end + +function mod.http_message(priority, priority_num, msg, options) c_outputs.handle_http(options.url, msg) end @@ -152,70 +169,84 @@ end function mod.http_reopen() end +function mod.grpc(event, rule, source, priority, priority_num, msg, options) + c_outputs.handle_grpc(event, rule, source, priority, priority_num, msg, options) +end + +function mod.grpc_message(priority, priority_num, msg, options) + -- todo +end + +function mod.grpc_cleanup() +end + +function mod.grpc_reopen() +end + function output_event(event, rule, source, priority, priority_num, format) -- If format starts with a *, remove it, as we're adding our own -- prefix here. - if format:sub(1,1) == "*" then + if format:sub(1, 1) == "*" then format = format:sub(2) end -- time_format_iso_8601 will be the same for all output channels time_format_iso_8601 = 0 - for index,o in ipairs(outputs) do + for index, o in ipairs(outputs) do time_format_iso_8601 = o.options.time_format_iso_8601 break end if source == "syscall" then if time_format_iso_8601 == 1 then - format = "*%evt.time.iso8601: "..priority.." "..format + format = "*%evt.time.iso8601: " .. priority .. " " .. format else - format = "*%evt.time: "..priority.." "..format + format = "*%evt.time: " .. priority .. " " .. format end else if time_format_iso_8601 == 1 then - format = "*%jevt.time.iso8601: "..priority.." "..format + format = "*%jevt.time.iso8601: " .. priority .. " " .. format else - format = "*%jevt.time: "..priority.." "..format + format = "*%jevt.time: " .. priority .. " " .. format end end msg = formats.format_event(event, rule, source, priority, format) - for index,o in ipairs(outputs) do - o.output(priority, priority_num, msg, o.options) + for index, o in ipairs(outputs) do + o.output(event, rule, source, priority, priority_num, msg, o.options) end end function output_msg(msg, priority, priority_num) - for index,o in ipairs(outputs) do - o.output(priority, priority_num, msg, o.options) + for index, o in ipairs(outputs) do + o.message(priority, priority_num, msg, o.options) end end function output_cleanup() formats.free_formatters() - for index,o in ipairs(outputs) do + for index, o in ipairs(outputs) do o.cleanup() end end function output_reopen() - for index,o in ipairs(outputs) do + for index, o in ipairs(outputs) do o.reopen(o.options) end end function add_output(output_name, buffered, time_format_iso_8601, options) - if not (type(mod[output_name]) == 'function') then - error("rule_loader.add_output(): invalid output_name: "..output_name) + if not (type(mod[output_name]) == "function") then + error("rule_loader.add_output(): invalid output_name: " .. output_name) end -- outputs can optionally define a validation function so that we don't -- find out at runtime (when an event finally matches a rule!) that the options are invalid - if (type(mod[output_name.."_validate"]) == 'function') then - mod[output_name.."_validate"](options) + if (type(mod[output_name .. "_validate"]) == "function") then + mod[output_name .. "_validate"](options) end if options == nil then @@ -225,10 +256,16 @@ function add_output(output_name, buffered, time_format_iso_8601, options) options.buffered = buffered options.time_format_iso_8601 = time_format_iso_8601 - table.insert(outputs, {output = mod[output_name], - cleanup = mod[output_name.."_cleanup"], - reopen = mod[output_name.."_reopen"], - options=options}) + table.insert( + outputs, + { + output = mod[output_name], + cleanup = mod[output_name .. "_cleanup"], + reopen = mod[output_name .. "_reopen"], + message = mod[output_name .. "_message"], + options = options + } + ) end return mod From d2753bc03ae968f25c11967f11cf275e9fbdd6a8 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Fri, 6 Sep 2019 14:05:57 +0000 Subject: [PATCH 31/76] update(userspace/falco): use concurrent output queue in grpc server impl Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/grpc_server_impl.cpp | 1 + userspace/falco/grpc_server_impl.h | 12 +----------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/userspace/falco/grpc_server_impl.cpp b/userspace/falco/grpc_server_impl.cpp index f89082a6744..c8138683fb7 100644 --- a/userspace/falco/grpc_server_impl.cpp +++ b/userspace/falco/grpc_server_impl.cpp @@ -37,6 +37,7 @@ void falco_grpc_server_impl::subscribe(const stream_context& ctx, const falco_ou } else { + auto m_event_queue = falco_output_queue::get().queue(); // Start (or continue) streaming // ctx.m_status == stream_context::STREAMING if(m_event_queue.try_pop(res) && !req.keepalive()) diff --git a/userspace/falco/grpc_server_impl.h b/userspace/falco/grpc_server_impl.h index 7b169b53aed..c64fbb23bd2 100644 --- a/userspace/falco/grpc_server_impl.h +++ b/userspace/falco/grpc_server_impl.h @@ -20,26 +20,16 @@ limitations under the License. #include -#include "tbb/concurrent_queue.h" +#include "falco_output_queue.h" #include "falco_output.grpc.pb.h" -#include "falco_output.pb.h" #include "grpc_context.h" -typedef tbb::concurrent_queue falco_output_response_cq; - class falco_grpc_server_impl { public: falco_grpc_server_impl() = default; ~falco_grpc_server_impl() = default; - falco_output_response_cq& m_event_queue; - - falco_grpc_server_impl(falco_output_response_cq& event_queue): - m_event_queue(event_queue) - { - } - protected: bool is_running(); From fcea8ff235128200b88c51d7169c014add1372e9 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Fri, 6 Sep 2019 14:06:53 +0000 Subject: [PATCH 32/76] update(userspace/falco): use concurrent output queue in grpc server Signed-off-by: Leonardo Di Donato Co-authored-by: Lorenzo Fontana --- userspace/falco/grpc_server.cpp | 6 +++--- userspace/falco/grpc_server.h | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/userspace/falco/grpc_server.cpp b/userspace/falco/grpc_server.cpp index bdaec631e47..bf54b3769ce 100644 --- a/userspace/falco/grpc_server.cpp +++ b/userspace/falco/grpc_server.cpp @@ -194,7 +194,7 @@ void falco_grpc_server::run() while(is_running()) { - sleep(1); + sleep(1); // todo > do we want to sleep here? } stop(); @@ -223,9 +223,9 @@ void falco_grpc_server::stop() } } -bool start_grpc_server(std::string server_address, int threadiness, falco_output_response_cq& output_event_queue) +bool start_grpc_server(std::string server_address, int threadiness) { - falco_grpc_server srv(server_address, threadiness, output_event_queue); + falco_grpc_server srv(server_address, threadiness); srv.run(); return true; } diff --git a/userspace/falco/grpc_server.h b/userspace/falco/grpc_server.h index 2c89c0e97fb..7aec3e73247 100644 --- a/userspace/falco/grpc_server.h +++ b/userspace/falco/grpc_server.h @@ -28,8 +28,7 @@ limitations under the License. class falco_grpc_server : public falco_grpc_server_impl { public: - falco_grpc_server(std::string server_addr, int threadiness, falco_output_response_cq& m_event_queue): - falco_grpc_server_impl(m_event_queue), + falco_grpc_server(std::string server_addr, int threadiness): m_server_addr(server_addr), m_threadiness(threadiness) { @@ -50,7 +49,7 @@ class falco_grpc_server : public falco_grpc_server_impl std::vector m_threads; }; -bool start_grpc_server(std::string server_address, int threadiness, falco_output_response_cq& output_event_queue); +bool start_grpc_server(std::string server_address, int threadiness); class request_context_base { From ce29da59a88860cdaaa94701ecf3a9eb6f101f17 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Fri, 6 Sep 2019 14:07:36 +0000 Subject: [PATCH 33/76] chore(userspace/falco): cleanup boot logic for grpc server Signed-off-by: Leonardo Di Donato --- userspace/falco/falco.cpp | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/userspace/falco/falco.cpp b/userspace/falco/falco.cpp index f3692aff84d..1402ccfe5aa 100644 --- a/userspace/falco/falco.cpp +++ b/userspace/falco/falco.cpp @@ -49,6 +49,7 @@ limitations under the License. #include "statsfilewriter.h" #include "webserver.h" #include "grpc_server.h" +#include "falco_output_queue.h" typedef function open_t; @@ -230,7 +231,6 @@ uint64_t do_inspect(falco_engine *engine, string &stats_filename, uint64_t stats_interval, bool all_events, - falco_output_response_cq &output_event_queue, int &result) { uint64_t num_evts = 0; @@ -326,11 +326,6 @@ uint64_t do_inspect(falco_engine *engine, if(res) { outputs->handle_event(res->evt, res->rule, res->source, res->priority_num, res->format); - - // TODO(fntlnz): integrate this with the output handle event logic, for now we just want to test the threads - falco_output_response grpcres = falco_output_response(); - grpcres.set_rule(res->rule); - output_event_queue.push(grpcres); } num_evts++; @@ -1171,14 +1166,13 @@ int falco_init(int argc, char **argv) } // grpc server - falco_output_response_cq output_event_queue; // TODO(fntlnz,leodido): when we want to spawn multiple threads we need to have a queue per thread, or implement // different queuing mechanisms, round robin, fanout? What we want to achieve? int threadiness = 1; // TODO(fntlnz, leodido): make this configurable // TODO(fntlnz): do any handling, make sure we handle signals in the GRPC server and we clean it gracefully - std::thread grpc_server_thread (start_grpc_server, "0.0.0.0:5060", threadiness, std::ref(output_event_queue)); + std::thread grpc_server_thread (start_grpc_server, "0.0.0.0:5060", threadiness); if(!trace_filename.empty() && !trace_is_scap) { @@ -1199,7 +1193,6 @@ int falco_init(int argc, char **argv) stats_filename, stats_interval, all_events, - output_event_queue, result); duration = ((double)clock()) / CLOCKS_PER_SEC - duration; From 642ca7cdc8ce5f8c8d1940d6b8df87db75bf7f9d Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Fri, 6 Sep 2019 14:08:19 +0000 Subject: [PATCH 34/76] new(userspace/falco): falco output handler to send events via grpc Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/falco_outputs.cpp | 39 ++++++++++++++++++++++++++++++- userspace/falco/falco_outputs.h | 5 ++-- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/userspace/falco/falco_outputs.cpp b/userspace/falco/falco_outputs.cpp index ff620581d9a..8540f5a8a9f 100644 --- a/userspace/falco/falco_outputs.cpp +++ b/userspace/falco/falco_outputs.cpp @@ -24,12 +24,14 @@ limitations under the License. #include "formats.h" #include "logger.h" +#include "falco_output_queue.h" using namespace std; const static struct luaL_reg ll_falco_outputs [] = { {"handle_http", &falco_outputs::handle_http}, + {"handle_grpc", &falco_outputs::handle_grpc}, {NULL,NULL} }; @@ -206,7 +208,7 @@ void falco_outputs::handle_msg(uint64_t now, bool first = true; sinsp_utils::ts_to_string(now, ×tr, false, true); - full_msg = timestr + ": " + falco_common::priority_names[LOG_CRIT] + " " + msg + "("; + full_msg = timestr + ": " + falco_common::priority_names[LOG_CRIT] + " " + msg + " ("; for(auto &pair : output_fields) { if(first) @@ -298,3 +300,38 @@ int falco_outputs::handle_http(lua_State *ls) } return 1; } + +int falco_outputs::handle_grpc(lua_State *ls) +{ + // fixme > check parameters later + // if(!lua_isstring(ls, -1) || + // !lua_isstring(ls, -2)) + // { + // lua_pushstring(ls, "Invalid arguments passed to handle_grpc()"); + // lua_error(ls); + // } + + enum source source; + if(!source_Parse((char *)lua_tostring(ls, 3), &source)) + { + lua_pushstring(ls, "Unknown source passed to to handle_grpc()"); + lua_error(ls); + } + + enum priority priority; + if(!priority_Parse((char *)lua_tostring(ls, 4), &priority)) + { + lua_pushstring(ls, "Unknown priority passed to to handle_grpc()"); + lua_error(ls); + } + + falco_output_response grpc_res = falco_output_response(); + grpc_res.set_rule((char *)lua_tostring(ls, 2)); + grpc_res.set_source(source); + grpc_res.set_priority(priority); + grpc_res.set_output((char *)lua_tostring(ls, 6)); + + falco_output_queue::get().queue().push(grpc_res); + + return 1; +} \ No newline at end of file diff --git a/userspace/falco/falco_outputs.h b/userspace/falco/falco_outputs.h index 709dc7c39ff..fb67055166b 100644 --- a/userspace/falco/falco_outputs.h +++ b/userspace/falco/falco_outputs.h @@ -46,8 +46,8 @@ class falco_outputs : public falco_common falco_outputs(falco_engine *engine); virtual ~falco_outputs(); - // The way to refer to an output (file, syslog, stdout, - // etc). An output has a name and set of options. + // The way to refer to an output (file, syslog, stdout, etc.) + // An output has a name and set of options. struct output_config { std::string name; @@ -78,6 +78,7 @@ class falco_outputs : public falco_common void reopen_outputs(); static int handle_http(lua_State *ls); + static int handle_grpc(lua_State *ls); private: From 54089cfbcbab4fe135c25d6d7e21eb4a292e90dd Mon Sep 17 00:00:00 2001 From: Lorenzo Fontana Date: Fri, 6 Sep 2019 18:16:35 +0200 Subject: [PATCH 35/76] new(userspace/falco): implement the output queue methods directly Co-Authored-By: Leonardo Di Donato Signed-off-by: Lorenzo Fontana --- userspace/falco/falco_output_queue.h | 11 ++++++++--- userspace/falco/grpc_server_impl.cpp | 7 +++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/userspace/falco/falco_output_queue.h b/userspace/falco/falco_output_queue.h index 09f9ec91046..f2ef98039cf 100644 --- a/userspace/falco/falco_output_queue.h +++ b/userspace/falco/falco_output_queue.h @@ -32,9 +32,14 @@ class falco_output_queue return instance; } - falco_output_response_cq queue() + bool try_pop(falco_output_response& res) { - return m_queue; + return m_queue.try_pop(res); + } + + void push(falco_output_response& res) + { + m_queue.push(res); } private: @@ -48,4 +53,4 @@ class falco_output_queue public: falco_output_queue(falco_output_queue const&) = delete; void operator=(falco_output_queue const&) = delete; -}; \ No newline at end of file +}; diff --git a/userspace/falco/grpc_server_impl.cpp b/userspace/falco/grpc_server_impl.cpp index c8138683fb7..6a06a430432 100644 --- a/userspace/falco/grpc_server_impl.cpp +++ b/userspace/falco/grpc_server_impl.cpp @@ -37,18 +37,17 @@ void falco_grpc_server_impl::subscribe(const stream_context& ctx, const falco_ou } else { - auto m_event_queue = falco_output_queue::get().queue(); // Start (or continue) streaming // ctx.m_status == stream_context::STREAMING - if(m_event_queue.try_pop(res) && !req.keepalive()) + if(falco_output_queue::get().try_pop(res) && !req.keepalive()) { ctx.m_has_more = true; return; } - while(!m_event_queue.try_pop(res) && req.keepalive()) + while(!falco_output_queue::get().try_pop(res) && req.keepalive()) { } ctx.m_has_more = req.keepalive(); } -} \ No newline at end of file +} From 29c3922af2d2e40cf3516e5407924df9bbc5c4ec Mon Sep 17 00:00:00 2001 From: Lorenzo Fontana Date: Fri, 6 Sep 2019 18:17:46 +0200 Subject: [PATCH 36/76] new(userspace/falco): send rule and message only for now Co-Authored-By: Leonardo Di Donato Signed-off-by: Lorenzo Fontana --- userspace/falco/falco_outputs.cpp | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/userspace/falco/falco_outputs.cpp b/userspace/falco/falco_outputs.cpp index 8540f5a8a9f..2163c37404c 100644 --- a/userspace/falco/falco_outputs.cpp +++ b/userspace/falco/falco_outputs.cpp @@ -303,6 +303,7 @@ int falco_outputs::handle_http(lua_State *ls) int falco_outputs::handle_grpc(lua_State *ls) { + // TODO(fntlnz, leodido): do the checks and make sure all the fields are sent // fixme > check parameters later // if(!lua_isstring(ls, -1) || // !lua_isstring(ls, -2)) @@ -311,27 +312,27 @@ int falco_outputs::handle_grpc(lua_State *ls) // lua_error(ls); // } - enum source source; - if(!source_Parse((char *)lua_tostring(ls, 3), &source)) - { - lua_pushstring(ls, "Unknown source passed to to handle_grpc()"); - lua_error(ls); - } + // enum source source; + // if(!source_Parse((char *)lua_tostring(ls, 3), &source)) + // { + // lua_pushstring(ls, "Unknown source passed to to handle_grpc()"); + // lua_error(ls); + // } - enum priority priority; - if(!priority_Parse((char *)lua_tostring(ls, 4), &priority)) - { - lua_pushstring(ls, "Unknown priority passed to to handle_grpc()"); - lua_error(ls); - } + // enum priority priority; + // if(!priority_Parse((char *)lua_tostring(ls, 4), &priority)) + // { + // lua_pushstring(ls, "Unknown priority passed to to handle_grpc()"); + // lua_error(ls); + // } falco_output_response grpc_res = falco_output_response(); grpc_res.set_rule((char *)lua_tostring(ls, 2)); - grpc_res.set_source(source); - grpc_res.set_priority(priority); + // grpc_res.set_source(source); + // grpc_res.set_priority(priority); grpc_res.set_output((char *)lua_tostring(ls, 6)); - falco_output_queue::get().queue().push(grpc_res); + falco_output_queue::get().push(grpc_res); return 1; } \ No newline at end of file From 14fe085dc3d247bab872d700d3127411cda824ef Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Wed, 11 Sep 2019 14:03:27 +0000 Subject: [PATCH 37/76] new(userspace/falco): context metadata Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/grpc_context.cpp | 19 ++++++++++++++++++- userspace/falco/grpc_context.h | 3 +++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/userspace/falco/grpc_context.cpp b/userspace/falco/grpc_context.cpp index f8559755224..4c6d5c4d92a 100644 --- a/userspace/falco/grpc_context.cpp +++ b/userspace/falco/grpc_context.cpp @@ -16,12 +16,29 @@ See the License for the specific language governing permissions and limitations under the License. */ +#include + #include "grpc_context.h" context::context(grpc::ServerContext* ctx): m_ctx(ctx) { - // todo: set thread-specific prefix "[session id][request id]" + std::string session_id; + std::string request_id; + + get_metadata(meta_session, session_id); + get_metadata(meta_session, request_id); + + std::stringstream meta; + if(!session_id.empty()) + { + meta << "[sid=" << session_id << "]"; + } + + if(!request_id.empty()) + { + meta << "[rid=" << request_id << "]"; + } } void context::get_metadata(std::string key, std::string& val) diff --git a/userspace/falco/grpc_context.h b/userspace/falco/grpc_context.h index 07021792bcf..c8c73dbee6a 100644 --- a/userspace/falco/grpc_context.h +++ b/userspace/falco/grpc_context.h @@ -24,6 +24,9 @@ limitations under the License. #include #endif +const std::string meta_session = "session_id"; +const std::string meta_request = "request_id"; + class context { public: From ced2639fdd15ddfdd23d5f3515aac127246a55a9 Mon Sep 17 00:00:00 2001 From: Lorenzo Fontana Date: Thu, 12 Sep 2019 17:16:49 +0200 Subject: [PATCH 38/76] new(userspace/engine): expose resolve_tokens to lua Co-Authored-By: Leonardo Di Donato Signed-off-by: Lorenzo Fontana --- userspace/engine/formats.cpp | 32 ++++++++++++++++++++++++++++++++ userspace/engine/formats.h | 2 ++ 2 files changed, 34 insertions(+) diff --git a/userspace/engine/formats.cpp b/userspace/engine/formats.cpp index 008c4ee8cc7..217790487ec 100644 --- a/userspace/engine/formats.cpp +++ b/userspace/engine/formats.cpp @@ -36,6 +36,7 @@ const static struct luaL_reg ll_falco [] = {"free_formatter", &falco_formats::free_formatter}, {"free_formatters", &falco_formats::free_formatters}, {"format_event", &falco_formats::format_event}, + {"resolve_tokens", &falco_formats::resolve_tokens}, {NULL,NULL} }; @@ -265,3 +266,34 @@ int falco_formats::format_event (lua_State *ls) return 1; } + + +int falco_formats::resolve_tokens(lua_State *ls) +{ + if (!lua_isstring(ls, -1) || + !lua_isstring(ls, -2)) { + lua_pushstring(ls, "Invalid arguments passed to resolve_tokens()"); + lua_error(ls); + } + gen_event* evt = (gen_event*)lua_topointer(ls, 1); + const char *source = (char *) lua_tostring(ls, 2); // TODO(fntlnz, leodido): do we need this one? + const char *format = (char *) lua_tostring(ls, 3); + string sformat = format; + + + map values; + + s_formatters->resolve_tokens((sinsp_evt *)evt, sformat, values); + + lua_newtable(ls); + int top = lua_gettop(ls); + for (std::map::iterator it = values.begin(); it != values.end(); ++it) { + const char* key = it->first.c_str(); + const char* value = it->second.c_str(); + lua_pushlstring(ls, key, it->first.size()); + lua_pushlstring(ls, value, it->second.size()); + lua_settable(ls, top); + } + + return 1; +} diff --git a/userspace/engine/formats.h b/userspace/engine/formats.h index 2e918a5f051..955737ac5ad 100644 --- a/userspace/engine/formats.h +++ b/userspace/engine/formats.h @@ -53,6 +53,8 @@ class falco_formats // formatted_string = falco.format_event(evt, formatter) static int format_event(lua_State *ls); + static int resolve_tokens(lua_State *ls); + static sinsp* s_inspector; static falco_engine *s_engine; static sinsp_evt_formatter_cache *s_formatters; From de949aa649daa94817c74cb7fd9108b066e5884f Mon Sep 17 00:00:00 2001 From: Lorenzo Fontana Date: Thu, 12 Sep 2019 17:17:57 +0200 Subject: [PATCH 39/76] new(userspace/falco): read output_fields from a lua table Co-Authored-By: Leonardo Di Donato Signed-off-by: Lorenzo Fontana --- userspace/falco/falco_outputs.cpp | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/userspace/falco/falco_outputs.cpp b/userspace/falco/falco_outputs.cpp index 2163c37404c..a14230735a2 100644 --- a/userspace/falco/falco_outputs.cpp +++ b/userspace/falco/falco_outputs.cpp @@ -301,6 +301,7 @@ int falco_outputs::handle_http(lua_State *ls) return 1; } +// TODO(fntlnz, leodido): verify if this works with k8s_audit as source int falco_outputs::handle_grpc(lua_State *ls) { // TODO(fntlnz, leodido): do the checks and make sure all the fields are sent @@ -325,13 +326,27 @@ int falco_outputs::handle_grpc(lua_State *ls) // lua_pushstring(ls, "Unknown priority passed to to handle_grpc()"); // lua_error(ls); // } - + falco_output_response grpc_res = falco_output_response(); - grpc_res.set_rule((char *)lua_tostring(ls, 2)); + // grpc_res.set_source(source); // grpc_res.set_priority(priority); + + // rule + grpc_res.set_rule((char *)lua_tostring(ls, 2)); + + // output grpc_res.set_output((char *)lua_tostring(ls, 6)); + // output fields + auto& fields = *grpc_res.mutable_output_fields(); + + lua_pushnil(ls); + while (lua_next(ls, 7) != 0) { + fields[lua_tostring(ls, -2)] = lua_tostring(ls, -1); + lua_pop(ls, 1); + } + falco_output_queue::get().push(grpc_res); return 1; From fa7c639d66fe40560753d651b8d4af7fa46a71e7 Mon Sep 17 00:00:00 2001 From: Lorenzo Fontana Date: Thu, 12 Sep 2019 17:19:19 +0200 Subject: [PATCH 40/76] update(userspace/falco/lua): passing format to resolve output fields Co-Authored-By: Leonardo Di Donato Signed-off-by: Lorenzo Fontana --- userspace/falco/lua/output.lua | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/userspace/falco/lua/output.lua b/userspace/falco/lua/output.lua index 84fecfeb2bf..1da032c81c3 100644 --- a/userspace/falco/lua/output.lua +++ b/userspace/falco/lua/output.lua @@ -19,7 +19,7 @@ local mod = {} local outputs = {} -function mod.stdout(event, rule, source, priority, priority_num, msg, options) +function mod.stdout(event, rule, source, priority, priority_num, msg, format, options) mod.stdout_message(priority, priority_num, msg, outputs) end @@ -59,7 +59,7 @@ function mod.file_open(options) end end -function mod.file(event, rule, source, priority, priority_num, msg, options) +function mod.file(event, rule, source, priority, priority_num, msg, format, options) mod.file_message(priority, priority_num, msg, options) end @@ -93,7 +93,7 @@ function mod.file_reopen(options) end end -function mod.syslog(event, rule, source, priority, priority_num, msg, options) +function mod.syslog(event, rule, source, priority, priority_num, msg, format, options) mod.syslog_message(priority, priority_num, msg, options) end @@ -116,7 +116,7 @@ function mod.program_open(options) end end -function mod.program(event, rule, source, priority, priority_num, msg, options) +function mod.program(event, rule, source, priority, priority_num, msg, format, options) mod.program_message(priority, priority_num, msg, options) end @@ -155,7 +155,7 @@ function mod.program_reopen(options) end end -function mod.http(event, rule, source, priority, priority_num, msg, options) +function mod.http(event, rule, source, priority, priority_num, msg, format, options) mod.http_message(priority, priority_num, msg, options) end @@ -169,8 +169,9 @@ end function mod.http_reopen() end -function mod.grpc(event, rule, source, priority, priority_num, msg, options) - c_outputs.handle_grpc(event, rule, source, priority, priority_num, msg, options) +function mod.grpc(event, rule, source, priority, priority_num, msg, format, options) + fields = formats.resolve_tokens(event, source, format) + c_outputs.handle_grpc(event, rule, source, priority, priority_num, msg, fields, options) end function mod.grpc_message(priority, priority_num, msg, options) @@ -215,7 +216,7 @@ function output_event(event, rule, source, priority, priority_num, format) msg = formats.format_event(event, rule, source, priority, format) for index, o in ipairs(outputs) do - o.output(event, rule, source, priority, priority_num, msg, o.options) + o.output(event, rule, source, priority, priority_num, msg, format, o.options) end end From 2342dff16dc48514c278b79ed0e81885049fe996 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Fri, 13 Sep 2019 11:45:34 +0000 Subject: [PATCH 41/76] update(usersoace/falco): allow aliases for enum types Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/falco_output.proto | 34 +++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/userspace/falco/falco_output.proto b/userspace/falco/falco_output.proto index 09b74902364..3464ec43315 100644 --- a/userspace/falco/falco_output.proto +++ b/userspace/falco/falco_output.proto @@ -2,32 +2,56 @@ syntax = "proto3"; import "google/protobuf/timestamp.proto"; -service falco_output_service { - rpc subscribe(falco_output_request) returns (stream falco_output_response); + +service service { + rpc subscribe(request) returns (stream response); } -message falco_output_request { +message request { string duration = 1; // TODO(leodido, fntlnz): not handled yet but keeping for reference. bool keepalive = 2; } enum priority { + option allow_alias = true; EMERGENCY = 0; + emergency = 0; + Emergency = 0; ALERT = 1; + alert = 1; + Alert = 1; CRITICAL = 2; + critical = 2; + Critical = 2; ERROR = 3; + error = 3; + Error = 3; WARNING = 4; + warning = 4; + Warning = 4; NOTICE = 5; + notice = 5; + Notice = 5; INFORMATIONAL = 6; + informational = 6; + Informational = 6; DEBUG = 7; + debug = 7; + Debug = 7; } enum source { + option allow_alias = true; SYSCALL = 0; + syscall = 0; + Syscall = 0; K8S_AUDIT = 1; + k8s_audit = 1; + K8s_audit = 1; + K8S_audit = 1; } -message falco_output_response { +message response { string id = 1; google.protobuf.Timestamp time = 2; priority priority = 3; @@ -35,4 +59,4 @@ message falco_output_response { string rule = 5; string output = 6; map output_fields = 7; -} \ No newline at end of file +} From c968aa4ff35a76e60d00264c8b76f9cf1a75939e Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Fri, 13 Sep 2019 11:46:23 +0000 Subject: [PATCH 42/76] update(userspace/falco): namespace the proto messages and types Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/falco_output.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/userspace/falco/falco_output.proto b/userspace/falco/falco_output.proto index 3464ec43315..7045b7cf5ce 100644 --- a/userspace/falco/falco_output.proto +++ b/userspace/falco/falco_output.proto @@ -2,6 +2,7 @@ syntax = "proto3"; import "google/protobuf/timestamp.proto"; +package falco.output; service service { rpc subscribe(request) returns (stream response); From e1cf6b5bdc77b213ab7c14f1dd5d4b67203c00e0 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Fri, 13 Sep 2019 11:47:17 +0000 Subject: [PATCH 43/76] update(userspace/falco/lua): no need to pass priority num Signed-off-by: Leonardo Di Donato --- userspace/falco/lua/output.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/userspace/falco/lua/output.lua b/userspace/falco/lua/output.lua index 1da032c81c3..77c6bdcb13d 100644 --- a/userspace/falco/lua/output.lua +++ b/userspace/falco/lua/output.lua @@ -171,7 +171,7 @@ end function mod.grpc(event, rule, source, priority, priority_num, msg, format, options) fields = formats.resolve_tokens(event, source, format) - c_outputs.handle_grpc(event, rule, source, priority, priority_num, msg, fields, options) + c_outputs.handle_grpc(event, rule, source, priority, msg, fields, options) end function mod.grpc_message(priority, priority_num, msg, options) From e5d4c3701817178e5c2c5ae1b0c5a29d20d86f09 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Fri, 13 Sep 2019 11:47:57 +0000 Subject: [PATCH 44/76] chore(userspace/falco): use the falco grpc output namespace Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/falco_output_queue.h | 8 +++++--- userspace/falco/grpc_server.cpp | 18 ++++++++---------- userspace/falco/grpc_server.h | 4 ++-- userspace/falco/grpc_server_impl.cpp | 2 +- userspace/falco/grpc_server_impl.h | 4 +++- 5 files changed, 19 insertions(+), 17 deletions(-) diff --git a/userspace/falco/falco_output_queue.h b/userspace/falco/falco_output_queue.h index f2ef98039cf..bc41d24e82d 100644 --- a/userspace/falco/falco_output_queue.h +++ b/userspace/falco/falco_output_queue.h @@ -21,7 +21,9 @@ limitations under the License. #include "falco_output.pb.h" #include "tbb/concurrent_queue.h" -typedef tbb::concurrent_queue falco_output_response_cq; +using namespace falco::output; + +typedef tbb::concurrent_queue falco_output_response_cq; class falco_output_queue { @@ -32,12 +34,12 @@ class falco_output_queue return instance; } - bool try_pop(falco_output_response& res) + bool try_pop(response& res) { return m_queue.try_pop(res); } - void push(falco_output_response& res) + void push(response& res) { m_queue.push(res); } diff --git a/userspace/falco/grpc_server.cpp b/userspace/falco/grpc_server.cpp index bf54b3769ce..1a6c21dd044 100644 --- a/userspace/falco/grpc_server.cpp +++ b/userspace/falco/grpc_server.cpp @@ -28,12 +28,12 @@ limitations under the License. #include "grpc_context.h" template<> -void request_stream_context::start(falco_grpc_server* srv) +void request_stream_context::start(falco_grpc_server* srv) { m_state = request_context_base::REQUEST; m_srv_ctx.reset(new grpc::ServerContext); auto srvctx = m_srv_ctx.get(); - m_res_writer.reset(new grpc::ServerAsyncWriter(srvctx)); + m_res_writer.reset(new grpc::ServerAsyncWriter(srvctx)); m_stream_ctx.reset(); m_req.Clear(); @@ -42,7 +42,7 @@ void request_stream_context::start( } template<> -void request_stream_context::process(falco_grpc_server* srv) +void request_stream_context::process(falco_grpc_server* srv) { // When it is the 1st process call if(m_state == request_context_base::REQUEST) @@ -52,7 +52,7 @@ void request_stream_context::proces } // Processing - falco_output_response res; + response res; (srv->*m_process_func)(*m_stream_ctx, m_req, res); // When there still are more responses to stream @@ -71,7 +71,7 @@ void request_stream_context::proces } template<> -void request_stream_context::end(falco_grpc_server* srv, bool isError) +void request_stream_context::end(falco_grpc_server* srv, bool isError) { if(m_stream_ctx) { @@ -83,7 +83,7 @@ void request_stream_context::end(fa } // Complete the processing - falco_output_response res; + response res; (srv->*m_process_func)(*m_stream_ctx, m_req, res); // subscribe() } else @@ -96,8 +96,6 @@ void request_stream_context::end(fa start(srv); } - - void falco_grpc_server::thread_process(int thread_index) { // TODO: is this right? That's what we want? @@ -164,7 +162,7 @@ void falco_grpc_server::thread_process(int thread_index) for(request_stream_context & ctx : RPC##_contexts) \ { \ ctx.m_process_func = &falco_grpc_server::IMPL; \ - ctx.m_request_func = &falco_output_service::AsyncService::Request##RPC; \ + ctx.m_request_func = &service::AsyncService::Request##RPC; \ ctx.start(this); \ } @@ -183,7 +181,7 @@ void falco_grpc_server::run() falco_logger::log(LOG_INFO, "Starting gRPC webserver at " + m_server_addr + "\n"); int context_count = m_threadiness * 1; // todo > 10 or 100? - PROCESS_STREAM(falco_output_request, falco_output_response, subscribe, subscribe, context_count) + PROCESS_STREAM(request, response, subscribe, subscribe, context_count) m_threads.resize(m_threadiness); int thread_idx = 0; diff --git a/userspace/falco/grpc_server.h b/userspace/falco/grpc_server.h index 7aec3e73247..a144d5e7e78 100644 --- a/userspace/falco/grpc_server.h +++ b/userspace/falco/grpc_server.h @@ -39,7 +39,7 @@ class falco_grpc_server : public falco_grpc_server_impl void run(); void stop(); - falco_output_service::AsyncService m_svc; + service::AsyncService m_svc; std::unique_ptr m_completion_queue; private: @@ -86,7 +86,7 @@ class request_stream_context : public request_context_base void (falco_grpc_server::*m_process_func)(const stream_context&, const Request&, Response&); // Pointer to function that requests the system to start processing given requests - void (falco_output_service::AsyncService::*m_request_func)(grpc::ServerContext*, Request*, grpc::ServerAsyncWriter*, grpc::CompletionQueue*, grpc::ServerCompletionQueue*, void*); + void (service::AsyncService::*m_request_func)(grpc::ServerContext*, Request*, grpc::ServerAsyncWriter*, grpc::CompletionQueue*, grpc::ServerCompletionQueue*, void*); void start(falco_grpc_server* srv); void process(falco_grpc_server* srv); diff --git a/userspace/falco/grpc_server_impl.cpp b/userspace/falco/grpc_server_impl.cpp index 6a06a430432..44f1682ccab 100644 --- a/userspace/falco/grpc_server_impl.cpp +++ b/userspace/falco/grpc_server_impl.cpp @@ -27,7 +27,7 @@ bool falco_grpc_server_impl::is_running() return true; } -void falco_grpc_server_impl::subscribe(const stream_context& ctx, const falco_output_request& req, falco_output_response& res) +void falco_grpc_server_impl::subscribe(const stream_context& ctx, const request& req, response& res) { if(ctx.m_status == stream_context::SUCCESS || ctx.m_status == stream_context::ERROR) { diff --git a/userspace/falco/grpc_server_impl.h b/userspace/falco/grpc_server_impl.h index c64fbb23bd2..fbf22b28b84 100644 --- a/userspace/falco/grpc_server_impl.h +++ b/userspace/falco/grpc_server_impl.h @@ -24,6 +24,8 @@ limitations under the License. #include "falco_output.grpc.pb.h" #include "grpc_context.h" +using namespace falco::output; + class falco_grpc_server_impl { public: @@ -33,7 +35,7 @@ class falco_grpc_server_impl protected: bool is_running(); - void subscribe(const stream_context& ctx, const falco_output_request& req, falco_output_response& res); + void subscribe(const stream_context& ctx, const request& req, response& res); private: std::atomic m_stop{false}; From fd0c2b411b924d8391ce254bfda8665ece8bdb48 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Fri, 13 Sep 2019 11:48:51 +0000 Subject: [PATCH 45/76] new(userspace/falco): transmit output priorities Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/falco_outputs.cpp | 39 ++++++++++++++----------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/userspace/falco/falco_outputs.cpp b/userspace/falco/falco_outputs.cpp index a14230735a2..221fd03eb30 100644 --- a/userspace/falco/falco_outputs.cpp +++ b/userspace/falco/falco_outputs.cpp @@ -21,12 +21,12 @@ limitations under the License. #include "config_falco.h" - #include "formats.h" #include "logger.h" #include "falco_output_queue.h" using namespace std; +using namespace falco::output; const static struct luaL_reg ll_falco_outputs [] = { @@ -304,6 +304,7 @@ int falco_outputs::handle_http(lua_State *ls) // TODO(fntlnz, leodido): verify if this works with k8s_audit as source int falco_outputs::handle_grpc(lua_State *ls) { + response grpc_res = response(); // TODO(fntlnz, leodido): do the checks and make sure all the fields are sent // fixme > check parameters later // if(!lua_isstring(ls, -1) || @@ -313,36 +314,32 @@ int falco_outputs::handle_grpc(lua_State *ls) // lua_error(ls); // } - // enum source source; - // if(!source_Parse((char *)lua_tostring(ls, 3), &source)) - // { - // lua_pushstring(ls, "Unknown source passed to to handle_grpc()"); - // lua_error(ls); - // } - - // enum priority priority; - // if(!priority_Parse((char *)lua_tostring(ls, 4), &priority)) - // { - // lua_pushstring(ls, "Unknown priority passed to to handle_grpc()"); - // lua_error(ls); - // } - - falco_output_response grpc_res = falco_output_response(); - - // grpc_res.set_source(source); - // grpc_res.set_priority(priority); + // todo(leodido, fntlnz) > pass the ID (?) + // grpc_res.set_id("..."); // rule grpc_res.set_rule((char *)lua_tostring(ls, 2)); + + + // priority + priority p = priority::EMERGENCY; + string pstr = (char *)lua_tostring(ls, 4); + if(!priority_Parse(pstr, &p)) + { + lua_pushstring(ls, "Unknown priority passed to to handle_grpc()"); + lua_error(ls); + } + grpc_res.set_priority(p); + // output - grpc_res.set_output((char *)lua_tostring(ls, 6)); + grpc_res.set_output((char *)lua_tostring(ls, 5)); // output fields auto& fields = *grpc_res.mutable_output_fields(); lua_pushnil(ls); - while (lua_next(ls, 7) != 0) { + while (lua_next(ls, 6) != 0) { fields[lua_tostring(ls, -2)] = lua_tostring(ls, -1); lua_pop(ls, 1); } From ac4ab6b81aa1be6537c2d00661f70506b181d5c4 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Fri, 13 Sep 2019 11:49:32 +0000 Subject: [PATCH 46/76] new(usespace/falco): transmit output sources Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/falco_outputs.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/userspace/falco/falco_outputs.cpp b/userspace/falco/falco_outputs.cpp index 221fd03eb30..239f07cfc00 100644 --- a/userspace/falco/falco_outputs.cpp +++ b/userspace/falco/falco_outputs.cpp @@ -320,7 +320,15 @@ int falco_outputs::handle_grpc(lua_State *ls) // rule grpc_res.set_rule((char *)lua_tostring(ls, 2)); - + // source + source s = source::SYSCALL; + string sstr = (char *)lua_tostring(ls, 3); + if(!source_Parse(sstr, &s)) + { + lua_pushstring(ls, "Unknown source passed to to handle_grpc()"); + lua_error(ls); + } + grpc_res.set_source(s); // priority priority p = priority::EMERGENCY; From b4d9cd331f4061003b5127875d88fc02976d9420 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Fri, 13 Sep 2019 12:04:37 +0000 Subject: [PATCH 47/76] new(userspace/falco): check lua data when handling gRPC Signed-off-by: Leonardo Di Donato --- userspace/falco/falco_outputs.cpp | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/userspace/falco/falco_outputs.cpp b/userspace/falco/falco_outputs.cpp index 239f07cfc00..9404431afdd 100644 --- a/userspace/falco/falco_outputs.cpp +++ b/userspace/falco/falco_outputs.cpp @@ -304,15 +304,19 @@ int falco_outputs::handle_http(lua_State *ls) // TODO(fntlnz, leodido): verify if this works with k8s_audit as source int falco_outputs::handle_grpc(lua_State *ls) { + // check parameters + if(!lua_istable(ls, -1) || + !lua_istable(ls, -2) || + !lua_isstring(ls, -3) || + !lua_isstring(ls, -4) || + !lua_isstring(ls, -5) || + !lua_isstring(ls, -6)) + { + lua_pushstring(ls, "Invalid arguments passed to handle_grpc()"); + lua_error(ls); + } + response grpc_res = response(); - // TODO(fntlnz, leodido): do the checks and make sure all the fields are sent - // fixme > check parameters later - // if(!lua_isstring(ls, -1) || - // !lua_isstring(ls, -2)) - // { - // lua_pushstring(ls, "Invalid arguments passed to handle_grpc()"); - // lua_error(ls); - // } // todo(leodido, fntlnz) > pass the ID (?) // grpc_res.set_id("..."); From 0e022dc8c8b3d8514104920e69924211a774f931 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Fri, 13 Sep 2019 15:42:49 +0000 Subject: [PATCH 48/76] update(userspace/falco): remove id from falco output proto Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/falco_output.proto | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/userspace/falco/falco_output.proto b/userspace/falco/falco_output.proto index 7045b7cf5ce..610ad662de5 100644 --- a/userspace/falco/falco_output.proto +++ b/userspace/falco/falco_output.proto @@ -53,11 +53,12 @@ enum source { } message response { - string id = 1; - google.protobuf.Timestamp time = 2; - priority priority = 3; - source source = 4; - string rule = 5; - string output = 6; - map output_fields = 7; + google.protobuf.Timestamp time = 1; + priority priority = 2; + source source = 3; + string rule = 4; + string output = 5; + map output_fields = 6; } + +// todo: investigate outputting tags \ No newline at end of file From 87aeca5502e80650768b7874610fcdcacb8b3ed9 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Fri, 13 Sep 2019 15:43:02 +0000 Subject: [PATCH 49/76] new(userspace/falco): trasmit output event timestamp over gRPC Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/falco_outputs.cpp | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/userspace/falco/falco_outputs.cpp b/userspace/falco/falco_outputs.cpp index 9404431afdd..7f70709870c 100644 --- a/userspace/falco/falco_outputs.cpp +++ b/userspace/falco/falco_outputs.cpp @@ -17,6 +17,8 @@ limitations under the License. */ +#include + #include "falco_outputs.h" #include "config_falco.h" @@ -305,12 +307,12 @@ int falco_outputs::handle_http(lua_State *ls) int falco_outputs::handle_grpc(lua_State *ls) { // check parameters - if(!lua_istable(ls, -1) || - !lua_istable(ls, -2) || - !lua_isstring(ls, -3) || - !lua_isstring(ls, -4) || - !lua_isstring(ls, -5) || - !lua_isstring(ls, -6)) + if(!lua_isuserdata(ls, 1) || + !lua_isstring(ls, 2) || + !lua_isstring(ls, 3) || + !lua_isstring(ls, 4) || + !lua_isstring(ls, 5) || + !lua_istable(ls, 6)) { lua_pushstring(ls, "Invalid arguments passed to handle_grpc()"); lua_error(ls); @@ -318,8 +320,10 @@ int falco_outputs::handle_grpc(lua_State *ls) response grpc_res = response(); - // todo(leodido, fntlnz) > pass the ID (?) - // grpc_res.set_id("..."); + // time + gen_event* evt = (gen_event*)lua_topointer(ls, 1); + auto& timestamp = *grpc_res.mutable_time(); + timestamp = google::protobuf::util::TimeUtil::NanosecondsToTimestamp(evt->get_ts()); // rule grpc_res.set_rule((char *)lua_tostring(ls, 2)); From 28a77c721a21e1f54c4ca4e9bd235f08ca1862fb Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Fri, 13 Sep 2019 17:03:23 +0000 Subject: [PATCH 50/76] build: use secure GRPC_LIB and GRPCPP_LIB Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f2bb7b78c46..8981e2054c7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -598,8 +598,8 @@ else() set(GRPC_SRC "${PROJECT_BINARY_DIR}/grpc-prefix/src/grpc") message(STATUS "Using bundled grpc in '${GRPC_SRC}'") set(GRPC_INCLUDE "${GRPC_SRC}/include") - set(GRPC_LIB "${GRPC_SRC}/libs/opt/libgrpc_unsecure.a") - set(GRPCPP_LIB "${GRPC_SRC}/libs/opt/libgrpc++_unsecure.a") + set(GRPC_LIB "${GRPC_SRC}/libs/opt/libgrpc.a") + set(GRPCPP_LIB "${GRPC_SRC}/libs/opt/libgrpc++.a") set(GRPC_CPP_PLUGIN "${GRPC_SRC}/bins/opt/grpc_cpp_plugin") get_filename_component(PROTOC_DIR ${PROTOC} DIRECTORY) From 285a8b9e4339352447b51c6be72eeba7bb08c6b1 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Fri, 13 Sep 2019 17:04:45 +0000 Subject: [PATCH 51/76] new(userspace/falco): grpc ssl server credentials Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/grpc_server.cpp | 53 ++++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/userspace/falco/grpc_server.cpp b/userspace/falco/grpc_server.cpp index 1a6c21dd044..e5c004ba659 100644 --- a/userspace/falco/grpc_server.cpp +++ b/userspace/falco/grpc_server.cpp @@ -157,22 +157,59 @@ void falco_grpc_server::thread_process(int thread_index) // // Create array of contexts and start processing streaming RPC request. // -#define PROCESS_STREAM(REQ, RESP, RPC, IMPL, CONTEXT_COUNT) \ - std::vector> RPC##_contexts(CONTEXT_COUNT); \ - for(request_stream_context & ctx : RPC##_contexts) \ - { \ - ctx.m_process_func = &falco_grpc_server::IMPL; \ - ctx.m_request_func = &service::AsyncService::Request##RPC; \ - ctx.start(this); \ +#define PROCESS_STREAM(REQ, RESP, RPC, IMPL, CONTEXT_COUNT) \ + std::vector> RPC##_contexts(CONTEXT_COUNT); \ + for(request_stream_context & ctx : RPC##_contexts) \ + { \ + ctx.m_process_func = &falco_grpc_server::IMPL; \ + ctx.m_request_func = &service::AsyncService::Request##RPC; \ + ctx.start(this); \ } +// todo(fntlnz, leodido) > cleanup this part (paths from config, read, includes) +#include +#include +#include + +void read(const std::string& filename, std::string& data) +{ + std::ifstream file(filename.c_str(), std::ios::in); + + if(file.is_open()) + { + std::stringstream ss; + ss << file.rdbuf(); + + file.close(); + + data = ss.str(); + } + + return; +} + void falco_grpc_server::run() { + string private_key; + string cert_chain; + string root_certs; + + read("/tmp/server.crt", cert_chain); + read("/tmp/server.key", private_key); + read("/tmp/ca.crt", root_certs); + + grpc::SslServerCredentialsOptions::PemKeyCertPair cert_pair{private_key, cert_chain}; + + grpc::SslServerCredentialsOptions ssl_opts(GRPC_SSL_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_AND_VERIFY); + ssl_opts.pem_root_certs = root_certs; + ssl_opts.pem_key_cert_pairs.push_back(cert_pair); + // Setup server grpc::ServerBuilder builder; // Listen on the given address without any authentication mechanism. - builder.AddListeningPort(m_server_addr, grpc::InsecureServerCredentials()); + builder.AddListeningPort(m_server_addr, grpc::SslServerCredentials(ssl_opts)); builder.RegisterService(&m_svc); + // builder.SetMaxSendMessageSize(GRPC_MAX_MESSAGE_SIZE); // testing max message size? // builder.SetMaxReceiveMessageSize(GRPC_MAX_MESSAGE_SIZE); // testing max message size? From f2764131fdf876d2f3b997621b92e232f57be257 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Mon, 16 Sep 2019 05:22:14 +0000 Subject: [PATCH 52/76] new(userspace/falco): handle SIGHUP and SIGINT in the main process not in the spawned threads (grpc server) Signed-off-by: Leonardo Di Donato --- userspace/falco/grpc_server.cpp | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/userspace/falco/grpc_server.cpp b/userspace/falco/grpc_server.cpp index e5c004ba659..b39c18e465f 100644 --- a/userspace/falco/grpc_server.cpp +++ b/userspace/falco/grpc_server.cpp @@ -21,7 +21,8 @@ limitations under the License. #else #include #endif -#include // sleep +#include // pthread_sigmask +#include // sleep, _exit #include "logger.h" #include "grpc_server.h" @@ -188,8 +189,26 @@ void read(const std::string& filename, std::string& data) return; } +extern "C" void handle_signal(int signum) +{ + // todo > print "Got signal << sigenum << : exiting..."); + exit(1); +} + void falco_grpc_server::run() { + // Handle SIGHUP and SIGINT in the main process. + // Not in the spawned threads. + sigset_t set; + sigemptyset(&set); + sigaddset(&set, SIGHUP); + sigaddset(&set, SIGINT); + pthread_sigmask(SIG_UNBLOCK, &set, nullptr); + struct sigaction action; + action.sa_handler = handle_signal; + sigaction(SIGHUP, &action, nullptr); + sigaction(SIGINT, &action, nullptr); + string private_key; string cert_chain; string root_certs; From 47c87227f2b9966ac869ba46ba5e0a46729c4e47 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Mon, 16 Sep 2019 05:22:40 +0000 Subject: [PATCH 53/76] new(userspace/falco): store context metadata for future usage Signed-off-by: Leonardo Di Donato --- userspace/falco/grpc_context.cpp | 9 ++++++++- userspace/falco/grpc_context.h | 3 +++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/userspace/falco/grpc_context.cpp b/userspace/falco/grpc_context.cpp index 4c6d5c4d92a..c52570492ad 100644 --- a/userspace/falco/grpc_context.cpp +++ b/userspace/falco/grpc_context.cpp @@ -29,16 +29,23 @@ context::context(grpc::ServerContext* ctx): get_metadata(meta_session, session_id); get_metadata(meta_session, request_id); + bool has_meta = false; std::stringstream meta; if(!session_id.empty()) { meta << "[sid=" << session_id << "]"; + has_meta = true; } - if(!request_id.empty()) { meta << "[rid=" << request_id << "]"; + has_meta = true; + } + if(has_meta) + { + meta << " "; } + m_prefix = meta.str(); } void context::get_metadata(std::string key, std::string& val) diff --git a/userspace/falco/grpc_context.h b/userspace/falco/grpc_context.h index c8c73dbee6a..4781517a1fb 100644 --- a/userspace/falco/grpc_context.h +++ b/userspace/falco/grpc_context.h @@ -18,6 +18,8 @@ limitations under the License. #pragma once +#include + #ifdef GRPC_INCLUDE_IS_GRPCPP #include #else @@ -37,6 +39,7 @@ class context private: grpc::ServerContext* m_ctx = nullptr; + std::string m_prefix; }; class stream_context : public context From 9b36aa656776fcdc67be5786b8f56e1f5108364e Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Mon, 16 Sep 2019 05:28:55 +0000 Subject: [PATCH 54/76] new: grpc server threadiness config Signed-off-by: Leonardo Di Donato --- falco.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/falco.yaml b/falco.yaml index 428fdd8c802..10b4fc58cd4 100644 --- a/falco.yaml +++ b/falco.yaml @@ -171,6 +171,7 @@ http_output: grpc: enabled: true bind_address: "0.0.0.0:5060" + threadiness: 8 grpc_output: enabled: true \ No newline at end of file From 48419e24160a5de26fa4ca197f41b1a3eba753be Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Mon, 16 Sep 2019 05:31:19 +0000 Subject: [PATCH 55/76] new: grpc server certificates config Signed-off-by: Leonardo Di Donato --- falco.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/falco.yaml b/falco.yaml index 10b4fc58cd4..faf6d662958 100644 --- a/falco.yaml +++ b/falco.yaml @@ -172,6 +172,9 @@ grpc: enabled: true bind_address: "0.0.0.0:5060" threadiness: 8 + private_key: "" + cert_chain: "" + root_certs: "" grpc_output: enabled: true \ No newline at end of file From 90d9b3a1d4426229985c99b9da2f2c998d986701 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Mon, 16 Sep 2019 05:31:33 +0000 Subject: [PATCH 56/76] new(userspace/falco): read all the gRPC server configs Signed-off-by: Leonardo Di Donato --- userspace/falco/configuration.cpp | 84 ++++++++++++++++++------------- userspace/falco/configuration.h | 57 +++++++++++---------- 2 files changed, 80 insertions(+), 61 deletions(-) diff --git a/userspace/falco/configuration.cpp b/userspace/falco/configuration.cpp index 842ae9c90dc..f809dbd37c6 100644 --- a/userspace/falco/configuration.cpp +++ b/userspace/falco/configuration.cpp @@ -29,20 +29,20 @@ limitations under the License. using namespace std; -falco_configuration::falco_configuration() - : m_buffered_outputs(false), - m_time_format_iso_8601(false), - m_webserver_enabled(false), - m_webserver_listen_port(8765), - m_webserver_k8s_audit_endpoint("/k8s_audit"), - m_webserver_ssl_enabled(false), - m_config(NULL) +falco_configuration::falco_configuration(): + m_buffered_outputs(false), + m_time_format_iso_8601(false), + m_webserver_enabled(false), + m_webserver_listen_port(8765), + m_webserver_k8s_audit_endpoint("/k8s_audit"), + m_webserver_ssl_enabled(false), + m_config(NULL) { } falco_configuration::~falco_configuration() { - if (m_config) + if(m_config) { delete m_config; } @@ -84,11 +84,11 @@ void falco_configuration::init(string conf_filename, list &cmdline_optio falco_outputs::output_config file_output; file_output.name = "file"; - if (m_config->get_scalar("file_output", "enabled", false)) + if(m_config->get_scalar("file_output", "enabled", false)) { string filename, keep_alive; filename = m_config->get_scalar("file_output", "filename", ""); - if (filename == string("")) + if(filename == string("")) { throw invalid_argument("Error reading config file (" + m_config_file + "): file output enabled but no filename in configuration block"); } @@ -102,25 +102,25 @@ void falco_configuration::init(string conf_filename, list &cmdline_optio falco_outputs::output_config stdout_output; stdout_output.name = "stdout"; - if (m_config->get_scalar("stdout_output", "enabled", false)) + if(m_config->get_scalar("stdout_output", "enabled", false)) { m_outputs.push_back(stdout_output); } falco_outputs::output_config syslog_output; syslog_output.name = "syslog"; - if (m_config->get_scalar("syslog_output", "enabled", false)) + if(m_config->get_scalar("syslog_output", "enabled", false)) { m_outputs.push_back(syslog_output); } falco_outputs::output_config program_output; program_output.name = "program"; - if (m_config->get_scalar("program_output", "enabled", false)) + if(m_config->get_scalar("program_output", "enabled", false)) { string program, keep_alive; program = m_config->get_scalar("program_output", "program", ""); - if (program == string("")) + if(program == string("")) { throw sinsp_exception("Error reading config file (" + m_config_file + "): program output enabled but no program in configuration block"); } @@ -134,12 +134,12 @@ void falco_configuration::init(string conf_filename, list &cmdline_optio falco_outputs::output_config http_output; http_output.name = "http"; - if (m_config->get_scalar("http_output", "enabled", false)) + if(m_config->get_scalar("http_output", "enabled", false)) { string url; url = m_config->get_scalar("http_output", "url", ""); - if (url == string("")) + if(url == string("")) { throw sinsp_exception("Error reading config file (" + m_config_file + "): http output enabled but no url in configuration block"); } @@ -148,15 +148,24 @@ void falco_configuration::init(string conf_filename, list &cmdline_optio m_outputs.push_back(http_output); } + m_grpc_enabled = m_config->get_scalar("grpc", "enabled", false); + m_grpc_bind_address = m_config->get_scalar("grpc", "bind_address", "0.0.0.0:5060"); + m_grpc_threadiness = m_config->get_scalar("grpc", "threadiness", 8); + // todo(fntlnz,leodido) > chose correct paths + m_grpc_private_key = m_config->get_scalar("grpc", "private_key", ""); + m_grpc_cert_chain = m_config->get_scalar("grpc", "cert_chain", ""); + m_grpc_root_certs = m_config->get_scalar("grpc", "root_certs", ""); + falco_outputs::output_config grpc_output; grpc_output.name = "grpc"; - if(m_config->get_scalar("grpc_output", "enabled", false)) + // gRPC output is enabled only if gRPC server is enabled too + if(m_config->get_scalar("grpc_output", "enabled", false) && m_grpc_enabled) { - // todo > grpc_output is enabled but we should constraint it to the grpc server being enabled too + // todo > m_outputs.push_back(grpc_output); } - if (m_outputs.size() == 0) + if(m_outputs.size() == 0) { throw invalid_argument("Error reading config file (" + m_config_file + "): No outputs configured. Please configure at least one output file output enabled but no filename in configuration block"); } @@ -171,7 +180,7 @@ void falco_configuration::init(string conf_filename, list &cmdline_optio string priority = m_config->get_scalar("priority", "debug"); vector::iterator it; - auto comp = [priority] (string &s) { + auto comp = [priority](string &s) { return (strcasecmp(s.c_str(), priority.c_str()) == 0); }; @@ -179,7 +188,7 @@ void falco_configuration::init(string conf_filename, list &cmdline_optio { throw invalid_argument("Unknown priority \"" + priority + "\"--must be one of emergency, alert, critical, error, warning, notice, informational, debug"); } - m_min_priority = (falco_common::priority_type) (it - falco_common::priority_names.begin()); + m_min_priority = (falco_common::priority_type)(it - falco_common::priority_names.begin()); m_buffered_outputs = m_config->get_scalar("buffered_outputs", false); m_time_format_iso_8601 = m_config->get_scalar("time_format_iso_8601", false); @@ -191,7 +200,7 @@ void falco_configuration::init(string conf_filename, list &cmdline_optio m_webserver_listen_port = m_config->get_scalar("webserver", "listen_port", 8765); m_webserver_k8s_audit_endpoint = m_config->get_scalar("webserver", "k8s_audit_endpoint", "/k8s_audit"); m_webserver_ssl_enabled = m_config->get_scalar("webserver", "ssl_enabled", false); - m_webserver_ssl_certificate = m_config->get_scalar("webserver", "ssl_certificate","/etc/falco/falco.pem"); + m_webserver_ssl_certificate = m_config->get_scalar("webserver", "ssl_certificate", "/etc/falco/falco.pem"); std::list syscall_event_drop_acts; m_config->get_sequence(syscall_event_drop_acts, "syscall_event_drops", "actions"); @@ -202,15 +211,15 @@ void falco_configuration::init(string conf_filename, list &cmdline_optio { m_syscall_evt_drop_actions.insert(syscall_evt_drop_mgr::ACT_IGNORE); } - else if (act == "log") + else if(act == "log") { m_syscall_evt_drop_actions.insert(syscall_evt_drop_mgr::ACT_LOG); } - else if (act == "alert") + else if(act == "alert") { m_syscall_evt_drop_actions.insert(syscall_evt_drop_mgr::ACT_ALERT); } - else if (act == "exit") + else if(act == "exit") { m_syscall_evt_drop_actions.insert(syscall_evt_drop_mgr::ACT_EXIT); } @@ -258,7 +267,7 @@ void falco_configuration::read_rules_file_directory(const string &path, listd_name; @@ -281,7 +290,7 @@ void falco_configuration::read_rules_file_directory(const string &path, list &parts) +static bool split(const string &str, char delim, pair &parts) { size_t pos; - if ((pos = str.find_first_of(delim)) == string::npos) { + if((pos = str.find_first_of(delim)) == string::npos) + { return false; } parts.first = str.substr(0, pos); @@ -318,16 +328,20 @@ void falco_configuration::init_cmdline_options(list &cmdline_options) void falco_configuration::set_cmdline_option(const string &opt) { - pair keyval; - pair subkey; + pair keyval; + pair subkey; - if (! split(opt, '=', keyval)) { + if(!split(opt, '=', keyval)) + { throw invalid_argument("Error parsing config option \"" + opt + "\". Must be of the form key=val or key.subkey=val"); } - if (split(keyval.first, '.', subkey)) { + if(split(keyval.first, '.', subkey)) + { m_config->set_scalar(subkey.first, subkey.second, keyval.second); - } else { + } + else + { m_config->set_scalar(keyval.first, keyval.second); } } diff --git a/userspace/falco/configuration.h b/userspace/falco/configuration.h index 585aea1d02f..51cd3b65595 100644 --- a/userspace/falco/configuration.h +++ b/userspace/falco/configuration.h @@ -45,12 +45,12 @@ class yaml_configuration { m_root = YAML::LoadFile(path); } - catch (const YAML::BadFile& ex) + catch(const YAML::BadFile& ex) { std::cerr << "Error reading config file (" + path + "): " + ex.what() + "\n"; throw; } - catch (const YAML::ParserException& ex) + catch(const YAML::ParserException& ex) { std::cerr << "Cannot read config file (" + path + "): " + ex.what() + "\n"; throw; @@ -66,11 +66,12 @@ class yaml_configuration try { auto node = m_root[key]; - if (node.IsDefined()) + if(node.IsDefined()) { return node.as(); } - } catch (const YAML::BadConversion& ex) + } + catch(const YAML::BadConversion& ex) { std::cerr << "Cannot read config file (" + m_path + "): wrong type at key " + key + "\n"; throw; @@ -83,10 +84,10 @@ class yaml_configuration * Set the top-level node identified by key to value */ template - void set_scalar(const std::string &key, const T& value) + void set_scalar(const std::string& key, const T& value) { auto node = m_root; - if (node.IsDefined()) + if(node.IsDefined()) { node[key] = value; } @@ -106,12 +107,12 @@ class yaml_configuration try { auto node = m_root[key][subkey]; - if (node.IsDefined()) + if(node.IsDefined()) { return node.as(); } } - catch (const YAML::BadConversion& ex) + catch(const YAML::BadConversion& ex) { std::cerr << "Cannot read config file (" + m_path + "): wrong type at key " + key + "\n"; throw; @@ -127,15 +128,15 @@ class yaml_configuration void set_scalar(const std::string& key, const std::string& subkey, const T& value) { auto node = m_root; - if (node.IsDefined()) + if(node.IsDefined()) { node[key][subkey] = value; } } // called with the last variadic arg (where the sequence is expected to be found) - template - void get_sequence_from_node(T& ret, const YAML::Node &node) + template + void get_sequence_from_node(T& ret, const YAML::Node& node) { if(node.IsDefined()) { @@ -154,25 +155,25 @@ class yaml_configuration } // called with the last variadic arg (where the sequence is expected to be found) - template + template void get_sequence(T& ret, const std::string& name) { return get_sequence_from_node(ret, m_root[name]); } // called with the last variadic arg (where the sequence is expected to be found) - template - void get_sequence(T& ret, const std::string& key, const std::string &subkey) + template + void get_sequence(T& ret, const std::string& key, const std::string& subkey) { try { auto node = m_root[key]; - if (node.IsDefined()) + if(node.IsDefined()) { return get_sequence_from_node(ret, node[subkey]); } } - catch (const YAML::BadConversion& ex) + catch(const YAML::BadConversion& ex) { std::cerr << "Cannot read config file (" + m_path + "): wrong type at key " + key + "\n"; throw; @@ -183,17 +184,16 @@ class yaml_configuration YAML::Node m_root; }; - class falco_configuration { - public: +public: falco_configuration(); virtual ~falco_configuration(); - void init(std::string conf_filename, std::list &cmdline_options); - void init(std::list &cmdline_options); + void init(std::string conf_filename, std::list& cmdline_options); + void init(std::list& cmdline_options); - static void read_rules_file_directory(const string &path, list &rules_filenames); + static void read_rules_file_directory(const string& path, list& rules_filenames); std::list m_rules_filenames; bool m_json_output; @@ -207,6 +207,13 @@ class falco_configuration bool m_buffered_outputs; bool m_time_format_iso_8601; + bool m_grpc_enabled; + int m_grpc_threadiness; + std::string m_grpc_bind_address; + std::string m_grpc_private_key; + std::string m_grpc_cert_chain; + std::string m_grpc_root_certs; + bool m_webserver_enabled; uint32_t m_webserver_listen_port; std::string m_webserver_k8s_audit_endpoint; @@ -219,9 +226,8 @@ class falco_configuration // Only used for testing bool m_syscall_evt_simulate_drops; - - private: - void init_cmdline_options(std::list &cmdline_options); +private: + void init_cmdline_options(std::list& cmdline_options); /** * Given a = specifier, set the appropriate option @@ -229,8 +235,7 @@ class falco_configuration * characters for nesting. Currently only 1- or 2- level keys * are supported and only scalar values are supported. */ - void set_cmdline_option(const std::string &spec); + void set_cmdline_option(const std::string& spec); yaml_configuration* m_config; }; - From 8c06e8764fdc3690f0d88da93e66b5e0a17c7cee Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Mon, 16 Sep 2019 06:25:06 +0000 Subject: [PATCH 57/76] new(userspace/falco): shutdown method for grpc server Signed-off-by: Leonardo Di Donato --- userspace/falco/grpc_server_impl.cpp | 5 +++++ userspace/falco/grpc_server_impl.h | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/userspace/falco/grpc_server_impl.cpp b/userspace/falco/grpc_server_impl.cpp index 44f1682ccab..22c6cb46d07 100644 --- a/userspace/falco/grpc_server_impl.cpp +++ b/userspace/falco/grpc_server_impl.cpp @@ -51,3 +51,8 @@ void falco_grpc_server_impl::subscribe(const stream_context& ctx, const request& ctx.m_has_more = req.keepalive(); } } + +void falco_grpc_server_impl::shutdown() +{ + m_stop = true; +} \ No newline at end of file diff --git a/userspace/falco/grpc_server_impl.h b/userspace/falco/grpc_server_impl.h index fbf22b28b84..32e1b65524d 100644 --- a/userspace/falco/grpc_server_impl.h +++ b/userspace/falco/grpc_server_impl.h @@ -32,11 +32,15 @@ class falco_grpc_server_impl falco_grpc_server_impl() = default; ~falco_grpc_server_impl() = default; + void shutdown(); + protected: bool is_running(); void subscribe(const stream_context& ctx, const request& req, response& res); + + private: std::atomic m_stop{false}; }; \ No newline at end of file From bb012af192c447e772d8cb3ce3b5178a49c36ea4 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Mon, 16 Sep 2019 07:25:28 +0000 Subject: [PATCH 58/76] fix(userspace/falco): correcly log SIGINT handling (fixes #791) Signed-off-by: Leonardo Di Donato --- userspace/falco/configuration.cpp | 2 +- userspace/falco/falco.cpp | 29 ++++++++++++++++------- userspace/falco/grpc_server.cpp | 38 ++++++++++++++++++------------- 3 files changed, 44 insertions(+), 25 deletions(-) diff --git a/userspace/falco/configuration.cpp b/userspace/falco/configuration.cpp index f809dbd37c6..ce4f31e310e 100644 --- a/userspace/falco/configuration.cpp +++ b/userspace/falco/configuration.cpp @@ -150,7 +150,7 @@ void falco_configuration::init(string conf_filename, list &cmdline_optio m_grpc_enabled = m_config->get_scalar("grpc", "enabled", false); m_grpc_bind_address = m_config->get_scalar("grpc", "bind_address", "0.0.0.0:5060"); - m_grpc_threadiness = m_config->get_scalar("grpc", "threadiness", 8); + m_grpc_threadiness = m_config->get_scalar("grpc", "threadiness", 8); // todo > limit it to avoid overshubscription? std::thread::hardware_concurrency() // todo(fntlnz,leodido) > chose correct paths m_grpc_private_key = m_config->get_scalar("grpc", "private_key", ""); m_grpc_cert_chain = m_config->get_scalar("grpc", "cert_chain", ""); diff --git a/userspace/falco/falco.cpp b/userspace/falco/falco.cpp index 1402ccfe5aa..89bb9d1d15b 100644 --- a/userspace/falco/falco.cpp +++ b/userspace/falco/falco.cpp @@ -51,6 +51,8 @@ limitations under the License. #include "grpc_server.h" #include "falco_output_queue.h" +#include + typedef function open_t; bool g_terminate = false; @@ -272,7 +274,12 @@ uint64_t do_inspect(falco_engine *engine, g_reopen_outputs = false; } - if (g_terminate || g_restart) + if(g_terminate) + { + falco_logger::log(LOG_INFO, "SIGINT received, exiting...\n"); + break; + } + else if (g_restart) { falco_logger::log(LOG_INFO, "SIGHUP Received, restarting...\n"); break; @@ -1166,13 +1173,18 @@ int falco_init(int argc, char **argv) } // grpc server - - // TODO(fntlnz,leodido): when we want to spawn multiple threads we need to have a queue per thread, or implement - // different queuing mechanisms, round robin, fanout? What we want to achieve? - int threadiness = 1; // TODO(fntlnz, leodido): make this configurable - - // TODO(fntlnz): do any handling, make sure we handle signals in the GRPC server and we clean it gracefully - std::thread grpc_server_thread (start_grpc_server, "0.0.0.0:5060", threadiness); + std::thread grpc_server_thread; + falco_grpc_server grpc_server(config.m_grpc_bind_address, config.m_grpc_threadiness); + if(config.m_grpc_enabled) + { + // TODO(fntlnz,leodido): when we want to spawn multiple threads we need to have a queue per thread, or implement + // different queuing mechanisms, round robin, fanout? What we want to achieve? + // TODO(fntlnz, leodido): make sure we clean it gracefully + // grpc_server_thread = std::thread(start_grpc_server, config.m_grpc_bind_address, config.m_grpc_threadiness); + grpc_server_thread = std::thread([&grpc_server] { + grpc_server.run(); + }); + } if(!trace_filename.empty() && !trace_is_scap) { @@ -1217,6 +1229,7 @@ int falco_init(int argc, char **argv) engine->print_stats(); sdropmgr.print_stats(); webserver.stop(); + grpc_server.shutdown(); } catch(exception &e) { diff --git a/userspace/falco/grpc_server.cpp b/userspace/falco/grpc_server.cpp index b39c18e465f..8cc8646e580 100644 --- a/userspace/falco/grpc_server.cpp +++ b/userspace/falco/grpc_server.cpp @@ -21,8 +21,8 @@ limitations under the License. #else #include #endif -#include // pthread_sigmask -#include // sleep, _exit +#include // pthread_sigmask +#include // sleep, _exit #include "logger.h" #include "grpc_server.h" @@ -99,11 +99,11 @@ void request_stream_context::end(falco_grpc_server* srv, bool void falco_grpc_server::thread_process(int thread_index) { - // TODO: is this right? That's what we want? // Tell pthread to not handle termination signals in the current thread sigset_t set; sigemptyset(&set); - sigaddset(&set, SIGHUP); + sigaddset(&set, SIGTERM); + // sigaddset(&set, SIGHUP); // todo > SIGHUP should restart Falco, what to do? sigaddset(&set, SIGINT); pthread_sigmask(SIG_BLOCK, &set, nullptr); @@ -191,23 +191,25 @@ void read(const std::string& filename, std::string& data) extern "C" void handle_signal(int signum) { - // todo > print "Got signal << sigenum << : exiting..."); - exit(1); + // todo > print "Got signal << sigenum << : exiting..."); + // exit(1); } void falco_grpc_server::run() { // Handle SIGHUP and SIGINT in the main process. // Not in the spawned threads. - sigset_t set; - sigemptyset(&set); - sigaddset(&set, SIGHUP); - sigaddset(&set, SIGINT); - pthread_sigmask(SIG_UNBLOCK, &set, nullptr); - struct sigaction action; - action.sa_handler = handle_signal; - sigaction(SIGHUP, &action, nullptr); - sigaction(SIGINT, &action, nullptr); + // sigset_t set; + // sigemptyset(&set); + // sigaddset(&set, SIGTERM); + // // sigaddset(&set, SIGHUP); // todo > SIGHUP should restart Falco, what to do? + // sigaddset(&set, SIGINT); + // pthread_sigmask(SIG_UNBLOCK, &set, nullptr); + + // struct sigaction action; + // action.sa_handler = handle_signal; + // sigaction(SIGHUP, &action, nullptr); + // sigaction(SIGINT, &action, nullptr); string private_key; string cert_chain; @@ -236,7 +238,11 @@ void falco_grpc_server::run() m_server = builder.BuildAndStart(); falco_logger::log(LOG_INFO, "Starting gRPC webserver at " + m_server_addr + "\n"); - int context_count = m_threadiness * 1; // todo > 10 or 100? + // Create context for server threads + // The number of contexts is multiple of the number of threads + // This defines the number of simultaneous completion queue requests of the same type (service::AsyncService::Request##RPC) + // For this approach to be sufficient falco_grpc_server::IMPL have to be fast + int context_count = m_threadiness * 10; PROCESS_STREAM(request, response, subscribe, subscribe, context_count) m_threads.resize(m_threadiness); From ef460b8a42cef88bc53f5f778cf620b1bfeee746 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Mon, 16 Sep 2019 10:12:37 +0000 Subject: [PATCH 59/76] fix(userspace/falco): handle grpc server thread stop gracefully Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/falco.cpp | 19 ++++++++++++------- userspace/falco/grpc_server.cpp | 22 +++------------------- userspace/falco/grpc_server.h | 4 ++++ 3 files changed, 19 insertions(+), 26 deletions(-) diff --git a/userspace/falco/falco.cpp b/userspace/falco/falco.cpp index 89bb9d1d15b..f5551f9a04c 100644 --- a/userspace/falco/falco.cpp +++ b/userspace/falco/falco.cpp @@ -51,8 +51,6 @@ limitations under the License. #include "grpc_server.h" #include "falco_output_queue.h" -#include - typedef function open_t; bool g_terminate = false; @@ -458,6 +456,8 @@ int falco_init(int argc, char **argv) scap_stats cstats; falco_webserver webserver; + falco_grpc_server grpc_server; + std::thread grpc_server_thread; static struct option long_options[] = { @@ -1173,14 +1173,11 @@ int falco_init(int argc, char **argv) } // grpc server - std::thread grpc_server_thread; - falco_grpc_server grpc_server(config.m_grpc_bind_address, config.m_grpc_threadiness); if(config.m_grpc_enabled) { // TODO(fntlnz,leodido): when we want to spawn multiple threads we need to have a queue per thread, or implement // different queuing mechanisms, round robin, fanout? What we want to achieve? - // TODO(fntlnz, leodido): make sure we clean it gracefully - // grpc_server_thread = std::thread(start_grpc_server, config.m_grpc_bind_address, config.m_grpc_threadiness); + grpc_server.init(config.m_grpc_bind_address, config.m_grpc_threadiness); grpc_server_thread = std::thread([&grpc_server] { grpc_server.run(); }); @@ -1229,7 +1226,11 @@ int falco_init(int argc, char **argv) engine->print_stats(); sdropmgr.print_stats(); webserver.stop(); - grpc_server.shutdown(); + if(grpc_server_thread.joinable()) + { + grpc_server.shutdown(); + grpc_server_thread.join(); + } } catch(exception &e) { @@ -1238,6 +1239,10 @@ int falco_init(int argc, char **argv) result = EXIT_FAILURE; webserver.stop(); + if (grpc_server_thread.joinable()) { + grpc_server.shutdown(); + grpc_server_thread.join(); + } } exit: diff --git a/userspace/falco/grpc_server.cpp b/userspace/falco/grpc_server.cpp index 8cc8646e580..42f555737a6 100644 --- a/userspace/falco/grpc_server.cpp +++ b/userspace/falco/grpc_server.cpp @@ -22,7 +22,6 @@ limitations under the License. #include #endif #include // pthread_sigmask -#include // sleep, _exit #include "logger.h" #include "grpc_server.h" @@ -189,28 +188,14 @@ void read(const std::string& filename, std::string& data) return; } -extern "C" void handle_signal(int signum) +void falco_grpc_server::init(std::string server_addr, int threadiness) { - // todo > print "Got signal << sigenum << : exiting..."); - // exit(1); + m_server_addr = server_addr; + m_threadiness = threadiness; } void falco_grpc_server::run() { - // Handle SIGHUP and SIGINT in the main process. - // Not in the spawned threads. - // sigset_t set; - // sigemptyset(&set); - // sigaddset(&set, SIGTERM); - // // sigaddset(&set, SIGHUP); // todo > SIGHUP should restart Falco, what to do? - // sigaddset(&set, SIGINT); - // pthread_sigmask(SIG_UNBLOCK, &set, nullptr); - - // struct sigaction action; - // action.sa_handler = handle_signal; - // sigaction(SIGHUP, &action, nullptr); - // sigaction(SIGINT, &action, nullptr); - string private_key; string cert_chain; string root_certs; @@ -254,7 +239,6 @@ void falco_grpc_server::run() while(is_running()) { - sleep(1); // todo > do we want to sleep here? } stop(); diff --git a/userspace/falco/grpc_server.h b/userspace/falco/grpc_server.h index a144d5e7e78..53c61100023 100644 --- a/userspace/falco/grpc_server.h +++ b/userspace/falco/grpc_server.h @@ -28,6 +28,9 @@ limitations under the License. class falco_grpc_server : public falco_grpc_server_impl { public: + falco_grpc_server() + { + } falco_grpc_server(std::string server_addr, int threadiness): m_server_addr(server_addr), m_threadiness(threadiness) @@ -35,6 +38,7 @@ class falco_grpc_server : public falco_grpc_server_impl } virtual ~falco_grpc_server() = default; + void init(std::string server_addr, int threadiness); void thread_process(int thread_index); void run(); void stop(); From c2cd57df130ece6b054c39309955b1d70a9d002f Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Mon, 16 Sep 2019 10:16:46 +0000 Subject: [PATCH 60/76] update: grpc server disabled by default Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- falco.yaml | 2 +- userspace/falco/configuration.cpp | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/falco.yaml b/falco.yaml index faf6d662958..be48a5e293d 100644 --- a/falco.yaml +++ b/falco.yaml @@ -169,7 +169,7 @@ http_output: url: http://some.url grpc: - enabled: true + enabled: false bind_address: "0.0.0.0:5060" threadiness: 8 private_key: "" diff --git a/userspace/falco/configuration.cpp b/userspace/falco/configuration.cpp index ce4f31e310e..b1758c879c9 100644 --- a/userspace/falco/configuration.cpp +++ b/userspace/falco/configuration.cpp @@ -159,9 +159,8 @@ void falco_configuration::init(string conf_filename, list &cmdline_optio falco_outputs::output_config grpc_output; grpc_output.name = "grpc"; // gRPC output is enabled only if gRPC server is enabled too - if(m_config->get_scalar("grpc_output", "enabled", false) && m_grpc_enabled) + if(m_config->get_scalar("grpc_output", "enabled", true) && m_grpc_enabled) { - // todo > m_outputs.push_back(grpc_output); } From f412a6b0605a0bd9b34326c08194c55101988170 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Mon, 16 Sep 2019 11:01:28 +0000 Subject: [PATCH 61/76] new(userspace/falco): config certificates for the gRPC server Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- falco.yaml | 6 +++--- userspace/falco/configuration.cpp | 6 +++--- userspace/falco/falco.cpp | 2 +- userspace/falco/grpc_server.cpp | 18 +++++++----------- userspace/falco/grpc_server.h | 19 ++++++++++++------- 5 files changed, 26 insertions(+), 25 deletions(-) diff --git a/falco.yaml b/falco.yaml index be48a5e293d..5ef62edc086 100644 --- a/falco.yaml +++ b/falco.yaml @@ -172,9 +172,9 @@ grpc: enabled: false bind_address: "0.0.0.0:5060" threadiness: 8 - private_key: "" - cert_chain: "" - root_certs: "" + private_key: "/etc/falco/certs/server.key" + cert_chain: "/etc/falco/certs/server.crt" + root_certs: "/etc/falco/certs/ca.crt" grpc_output: enabled: true \ No newline at end of file diff --git a/userspace/falco/configuration.cpp b/userspace/falco/configuration.cpp index b1758c879c9..bfe6512ee7c 100644 --- a/userspace/falco/configuration.cpp +++ b/userspace/falco/configuration.cpp @@ -152,9 +152,9 @@ void falco_configuration::init(string conf_filename, list &cmdline_optio m_grpc_bind_address = m_config->get_scalar("grpc", "bind_address", "0.0.0.0:5060"); m_grpc_threadiness = m_config->get_scalar("grpc", "threadiness", 8); // todo > limit it to avoid overshubscription? std::thread::hardware_concurrency() // todo(fntlnz,leodido) > chose correct paths - m_grpc_private_key = m_config->get_scalar("grpc", "private_key", ""); - m_grpc_cert_chain = m_config->get_scalar("grpc", "cert_chain", ""); - m_grpc_root_certs = m_config->get_scalar("grpc", "root_certs", ""); + m_grpc_private_key = m_config->get_scalar("grpc", "private_key", "/etc/falco/certs/server.key"); + m_grpc_cert_chain = m_config->get_scalar("grpc", "cert_chain", "/etc/falco/certs/server.crt"); + m_grpc_root_certs = m_config->get_scalar("grpc", "root_certs", "/etc/falco/certs/ca.crt"); falco_outputs::output_config grpc_output; grpc_output.name = "grpc"; diff --git a/userspace/falco/falco.cpp b/userspace/falco/falco.cpp index f5551f9a04c..8d6a423e6e9 100644 --- a/userspace/falco/falco.cpp +++ b/userspace/falco/falco.cpp @@ -1177,7 +1177,7 @@ int falco_init(int argc, char **argv) { // TODO(fntlnz,leodido): when we want to spawn multiple threads we need to have a queue per thread, or implement // different queuing mechanisms, round robin, fanout? What we want to achieve? - grpc_server.init(config.m_grpc_bind_address, config.m_grpc_threadiness); + grpc_server.init(config.m_grpc_bind_address, config.m_grpc_threadiness, config.m_grpc_private_key, config.m_grpc_cert_chain, config.m_grpc_root_certs); grpc_server_thread = std::thread([&grpc_server] { grpc_server.run(); }); diff --git a/userspace/falco/grpc_server.cpp b/userspace/falco/grpc_server.cpp index 42f555737a6..c1d46ef7ab5 100644 --- a/userspace/falco/grpc_server.cpp +++ b/userspace/falco/grpc_server.cpp @@ -188,10 +188,13 @@ void read(const std::string& filename, std::string& data) return; } -void falco_grpc_server::init(std::string server_addr, int threadiness) +void falco_grpc_server::init(std::string server_addr, int threadiness, std::string private_key, std::string cert_chain, std::string root_certs) { m_server_addr = server_addr; m_threadiness = threadiness; + m_private_key = private_key; + m_cert_chain = cert_chain; + m_root_certs = root_certs; } void falco_grpc_server::run() @@ -200,9 +203,9 @@ void falco_grpc_server::run() string cert_chain; string root_certs; - read("/tmp/server.crt", cert_chain); - read("/tmp/server.key", private_key); - read("/tmp/ca.crt", root_certs); + read(m_cert_chain, cert_chain); + read(m_private_key, private_key); + read(m_root_certs, root_certs); grpc::SslServerCredentialsOptions::PemKeyCertPair cert_pair{private_key, cert_chain}; @@ -266,10 +269,3 @@ void falco_grpc_server::stop() { } } - -bool start_grpc_server(std::string server_address, int threadiness) -{ - falco_grpc_server srv(server_address, threadiness); - srv.run(); - return true; -} diff --git a/userspace/falco/grpc_server.h b/userspace/falco/grpc_server.h index 53c61100023..ed8822a9414 100644 --- a/userspace/falco/grpc_server.h +++ b/userspace/falco/grpc_server.h @@ -31,14 +31,17 @@ class falco_grpc_server : public falco_grpc_server_impl falco_grpc_server() { } - falco_grpc_server(std::string server_addr, int threadiness): + falco_grpc_server(std::string server_addr, int threadiness, std::string private_key, std::string cert_chain, std::string root_certs): m_server_addr(server_addr), - m_threadiness(threadiness) + m_threadiness(threadiness), + m_private_key(private_key), + m_cert_chain(cert_chain), + m_root_certs(root_certs) { } virtual ~falco_grpc_server() = default; - void init(std::string server_addr, int threadiness); + void init(std::string server_addr, int threadiness, std::string private_key, std::string cert_chain, std::string root_certs); void thread_process(int thread_index); void run(); void stop(); @@ -47,14 +50,16 @@ class falco_grpc_server : public falco_grpc_server_impl std::unique_ptr m_completion_queue; private: - std::unique_ptr m_server; std::string m_server_addr; - int m_threadiness = 0; + int m_threadiness; + std::string m_private_key; + std::string m_cert_chain; + std::string m_root_certs; + + std::unique_ptr m_server; std::vector m_threads; }; -bool start_grpc_server(std::string server_address, int threadiness); - class request_context_base { public: From 5542b60a79818259124ac681ef17b05a54dcef59 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Mon, 16 Sep 2019 21:26:59 +0000 Subject: [PATCH 62/76] chore: typos and miscellanea Signed-off-by: Leonardo Di Donato --- userspace/engine/formats.cpp | 7 ++----- userspace/falco/falco.cpp | 6 +++--- userspace/falco/falco_output.proto | 4 +--- userspace/falco/falco_outputs.cpp | 1 - userspace/falco/grpc_server_impl.h | 2 -- userspace/falco/lua/output.lua | 2 +- 6 files changed, 7 insertions(+), 15 deletions(-) diff --git a/userspace/engine/formats.cpp b/userspace/engine/formats.cpp index 217790487ec..d03801b4601 100644 --- a/userspace/engine/formats.cpp +++ b/userspace/engine/formats.cpp @@ -267,7 +267,6 @@ int falco_formats::format_event (lua_State *ls) } - int falco_formats::resolve_tokens(lua_State *ls) { if (!lua_isstring(ls, -1) || @@ -275,12 +274,10 @@ int falco_formats::resolve_tokens(lua_State *ls) lua_pushstring(ls, "Invalid arguments passed to resolve_tokens()"); lua_error(ls); } - gen_event* evt = (gen_event*)lua_topointer(ls, 1); - const char *source = (char *) lua_tostring(ls, 2); // TODO(fntlnz, leodido): do we need this one? - const char *format = (char *) lua_tostring(ls, 3); + gen_event* evt = (gen_event*) lua_topointer(ls, 1); + const char *format = (char *) lua_tostring(ls, 2); string sformat = format; - map values; s_formatters->resolve_tokens((sinsp_evt *)evt, sformat, values); diff --git a/userspace/falco/falco.cpp b/userspace/falco/falco.cpp index 8d6a423e6e9..14526086c2c 100644 --- a/userspace/falco/falco.cpp +++ b/userspace/falco/falco.cpp @@ -34,7 +34,6 @@ limitations under the License. #include #include - #include #include "logger.h" @@ -279,7 +278,7 @@ uint64_t do_inspect(falco_engine *engine, } else if (g_restart) { - falco_logger::log(LOG_INFO, "SIGHUP Received, restarting...\n"); + falco_logger::log(LOG_INFO, "SIGHUP received, restarting...\n"); break; } else if(rc == SCAP_TIMEOUT) @@ -1239,7 +1238,8 @@ int falco_init(int argc, char **argv) result = EXIT_FAILURE; webserver.stop(); - if (grpc_server_thread.joinable()) { + if(grpc_server_thread.joinable()) + { grpc_server.shutdown(); grpc_server_thread.join(); } diff --git a/userspace/falco/falco_output.proto b/userspace/falco/falco_output.proto index 610ad662de5..9bf6732a21f 100644 --- a/userspace/falco/falco_output.proto +++ b/userspace/falco/falco_output.proto @@ -59,6 +59,4 @@ message response { string rule = 4; string output = 5; map output_fields = 6; -} - -// todo: investigate outputting tags \ No newline at end of file +} \ No newline at end of file diff --git a/userspace/falco/falco_outputs.cpp b/userspace/falco/falco_outputs.cpp index 7f70709870c..610893b3dfd 100644 --- a/userspace/falco/falco_outputs.cpp +++ b/userspace/falco/falco_outputs.cpp @@ -303,7 +303,6 @@ int falco_outputs::handle_http(lua_State *ls) return 1; } -// TODO(fntlnz, leodido): verify if this works with k8s_audit as source int falco_outputs::handle_grpc(lua_State *ls) { // check parameters diff --git a/userspace/falco/grpc_server_impl.h b/userspace/falco/grpc_server_impl.h index 32e1b65524d..451e5ba2663 100644 --- a/userspace/falco/grpc_server_impl.h +++ b/userspace/falco/grpc_server_impl.h @@ -39,8 +39,6 @@ class falco_grpc_server_impl void subscribe(const stream_context& ctx, const request& req, response& res); - - private: std::atomic m_stop{false}; }; \ No newline at end of file diff --git a/userspace/falco/lua/output.lua b/userspace/falco/lua/output.lua index 77c6bdcb13d..751eaee34b5 100644 --- a/userspace/falco/lua/output.lua +++ b/userspace/falco/lua/output.lua @@ -170,7 +170,7 @@ function mod.http_reopen() end function mod.grpc(event, rule, source, priority, priority_num, msg, format, options) - fields = formats.resolve_tokens(event, source, format) + fields = formats.resolve_tokens(event, format) c_outputs.handle_grpc(event, rule, source, priority, msg, fields, options) end From 57ae5f03dbced363ad58f3a0e4245951f318fd38 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Tue, 17 Sep 2019 21:49:12 +0000 Subject: [PATCH 63/76] update(userspace/engine): resolve token Signed-off-by: Leonardo Di Donato --- userspace/engine/formats.cpp | 4 ++-- userspace/engine/formats.h | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/userspace/engine/formats.cpp b/userspace/engine/formats.cpp index d03801b4601..ae7571e774c 100644 --- a/userspace/engine/formats.cpp +++ b/userspace/engine/formats.cpp @@ -269,8 +269,8 @@ int falco_formats::format_event (lua_State *ls) int falco_formats::resolve_tokens(lua_State *ls) { - if (!lua_isstring(ls, -1) || - !lua_isstring(ls, -2)) { + if (!lua_isuserdata(ls, 1) || + !lua_isstring(ls, 2)) { lua_pushstring(ls, "Invalid arguments passed to resolve_tokens()"); lua_error(ls); } diff --git a/userspace/engine/formats.h b/userspace/engine/formats.h index 955737ac5ad..01677a63146 100644 --- a/userspace/engine/formats.h +++ b/userspace/engine/formats.h @@ -53,6 +53,7 @@ class falco_formats // formatted_string = falco.format_event(evt, formatter) static int format_event(lua_State *ls); + // resolve_tokens = falco.resolve_tokens(evt, formatter) static int resolve_tokens(lua_State *ls); static sinsp* s_inspector; From a937484ac250f4c5e37d9d9969d0aa92fe98a07e Mon Sep 17 00:00:00 2001 From: Lorenzo Fontana Date: Thu, 19 Sep 2019 16:05:39 +0200 Subject: [PATCH 64/76] fix(userspace/falco): grpc server implementation subscribe handle output queue stop Signed-off-by: Lorenzo Fontana --- userspace/falco/grpc_server.cpp | 46 +++++++++------------------- userspace/falco/grpc_server.h | 2 -- userspace/falco/grpc_server_impl.cpp | 6 ++-- userspace/falco/grpc_server_impl.h | 2 +- 4 files changed, 18 insertions(+), 38 deletions(-) diff --git a/userspace/falco/grpc_server.cpp b/userspace/falco/grpc_server.cpp index c1d46ef7ab5..d8078c9f6e1 100644 --- a/userspace/falco/grpc_server.cpp +++ b/userspace/falco/grpc_server.cpp @@ -21,7 +21,6 @@ limitations under the License. #else #include #endif -#include // pthread_sigmask #include "logger.h" #include "grpc_server.h" @@ -36,7 +35,6 @@ void request_stream_context::start(falco_grpc_server* srv) m_res_writer.reset(new grpc::ServerAsyncWriter(srvctx)); m_stream_ctx.reset(); m_req.Clear(); - auto cq = srv->m_completion_queue.get(); (srv->m_svc.*m_request_func)(srvctx, &m_req, m_res_writer.get(), cq, cq, this); } @@ -71,40 +69,22 @@ void request_stream_context::process(falco_grpc_server* srv) } template<> -void request_stream_context::end(falco_grpc_server* srv, bool isError) +void request_stream_context::end(falco_grpc_server* srv, bool errored) { if(m_stream_ctx) { - m_stream_ctx->m_status = stream_context::SUCCESS; - if(isError) - { - m_stream_ctx->m_status = stream_context::ERROR; - // todo > log error - } + m_stream_ctx->m_status = errored ? stream_context::ERROR : stream_context::SUCCESS; // Complete the processing response res; (srv->*m_process_func)(*m_stream_ctx, m_req, res); // subscribe() } - else - { - // Handle the edge case when `m_request_func` event failed - // which means `m_stream_ctx` was not set - // todo > log error - } start(srv); } void falco_grpc_server::thread_process(int thread_index) { - // Tell pthread to not handle termination signals in the current thread - sigset_t set; - sigemptyset(&set); - sigaddset(&set, SIGTERM); - // sigaddset(&set, SIGHUP); // todo > SIGHUP should restart Falco, what to do? - sigaddset(&set, SIGINT); - pthread_sigmask(SIG_BLOCK, &set, nullptr); void* tag = nullptr; bool event_read_success = false; @@ -143,15 +123,13 @@ void falco_grpc_server::thread_process(int thread_index) case request_context_base::FINISH: // Completion of ServerAsyncWriter::Finish() ctx->end(this, false); - + break; default: // todo > log "unkown completion queue event" // todo > abort? break; } } - - // todo > log "thread `thread_index` complete" } // @@ -240,32 +218,36 @@ void falco_grpc_server::run() thread = std::thread(&falco_grpc_server::thread_process, this, thread_idx++); } - while(is_running()) + while(falco_grpc_server_impl::is_running()) { + sleep(1); } - stop(); } void falco_grpc_server::stop() { + falco_logger::log(LOG_INFO, "Shutting down gRPC server. Waiting until external connections are closed by clients\n"); m_server->Shutdown(); m_completion_queue->Shutdown(); - // todo > log "waiting for the server threads to complete" - + falco_logger::log(LOG_INFO, "Waiting for the gRPC threads to complete\n"); for(std::thread& t : m_threads) { - t.join(); + if(t.joinable()) + { + t.join(); + } } m_threads.clear(); - // todo > log "all server threads complete" - + falco_logger::log(LOG_INFO, "Ignoring all the remaining gRPC events\n"); // Ignore remaining events void* ignore_tag = nullptr; bool ignore_ok = false; while(m_completion_queue->Next(&ignore_tag, &ignore_ok)) { } + + falco_logger::log(LOG_INFO, "gRPC shutdown is now complete\n"); } diff --git a/userspace/falco/grpc_server.h b/userspace/falco/grpc_server.h index ed8822a9414..672aec201d8 100644 --- a/userspace/falco/grpc_server.h +++ b/userspace/falco/grpc_server.h @@ -21,8 +21,6 @@ limitations under the License. #include #include -#include - #include "grpc_server_impl.h" class falco_grpc_server : public falco_grpc_server_impl diff --git a/userspace/falco/grpc_server_impl.cpp b/userspace/falco/grpc_server_impl.cpp index 22c6cb46d07..32cb99f72e9 100644 --- a/userspace/falco/grpc_server_impl.cpp +++ b/userspace/falco/grpc_server_impl.cpp @@ -44,15 +44,15 @@ void falco_grpc_server_impl::subscribe(const stream_context& ctx, const request& ctx.m_has_more = true; return; } - while(!falco_output_queue::get().try_pop(res) && req.keepalive()) + while(is_running() && !falco_output_queue::get().try_pop(res) && req.keepalive()) { } - ctx.m_has_more = req.keepalive(); + ctx.m_has_more = !is_running() ? false : req.keepalive(); } } void falco_grpc_server_impl::shutdown() { m_stop = true; -} \ No newline at end of file +} diff --git a/userspace/falco/grpc_server_impl.h b/userspace/falco/grpc_server_impl.h index 451e5ba2663..54277a2258b 100644 --- a/userspace/falco/grpc_server_impl.h +++ b/userspace/falco/grpc_server_impl.h @@ -41,4 +41,4 @@ class falco_grpc_server_impl private: std::atomic m_stop{false}; -}; \ No newline at end of file +}; From 6f3698c130f13d6347e747f28c63af8e94680b08 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Mon, 23 Sep 2019 13:21:59 +0000 Subject: [PATCH 65/76] new(userspace/engine): json event to map type Signed-off-by: Leonardo Di Donato --- userspace/engine/json_evt.cpp | 25 +++++++++++++++++++++++++ userspace/engine/json_evt.h | 1 + 2 files changed, 26 insertions(+) diff --git a/userspace/engine/json_evt.cpp b/userspace/engine/json_evt.cpp index 1aabfaa1cd4..81393f7832d 100644 --- a/userspace/engine/json_evt.cpp +++ b/userspace/engine/json_evt.cpp @@ -796,6 +796,7 @@ std::string json_event_formatter::tostring(json_event *ev) std::string json_event_formatter::tojson(json_event *ev) { nlohmann::json ret; + // todo(leodido, fntlnz) > assign tomap() result to ret (implicit conversion using = operator) std::list> resolved; @@ -806,6 +807,7 @@ std::string json_event_formatter::tojson(json_event *ev) // Only include the fields and not the raw text blocks. if(!res.first.empty()) { + // todo(leodido, fntlnz) > do we want "" rather than empty res.second values? ret[res.first] = res.second; } } @@ -813,6 +815,29 @@ std::string json_event_formatter::tojson(json_event *ev) return ret.dump(); } +std::map json_event_formatter::tomap(json_event *ev) +{ + std::map ret; + std::list> res; + + resolve_tokens(ev, res); + + for(auto &r : res) + { + // Only include the fields and not the raw text blocks. + if(!r.first.empty()) + { + if(r.second.empty()) + { + r.second = ""; + } + ret.insert(r); + } + } + + return ret; +} + void json_event_formatter::parse_format() { string tformat = m_format; diff --git a/userspace/engine/json_evt.h b/userspace/engine/json_evt.h index b4ad05733a5..8bab207109c 100644 --- a/userspace/engine/json_evt.h +++ b/userspace/engine/json_evt.h @@ -287,6 +287,7 @@ class json_event_formatter std::string tostring(json_event *ev); std::string tojson(json_event *ev); + std::map tomap(json_event *ev); void resolve_tokens(json_event *ev, std::list> &resolved); From 99bc5b4d44844fa8bb42289ebf0391eb14b4afd9 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Mon, 23 Sep 2019 14:13:25 +0000 Subject: [PATCH 66/76] fix(userspace/falco): distinguish between sinsp and json events when resolving tokens Signed-off-by: Leonardo Di Donato --- userspace/engine/formats.cpp | 42 ++++++++++++++++++++----------- userspace/falco/falco_outputs.cpp | 20 +++++++-------- userspace/falco/lua/output.lua | 13 +++++++++- 3 files changed, 50 insertions(+), 25 deletions(-) diff --git a/userspace/engine/formats.cpp b/userspace/engine/formats.cpp index ae7571e774c..9527c990ded 100644 --- a/userspace/engine/formats.cpp +++ b/userspace/engine/formats.cpp @@ -20,7 +20,6 @@ limitations under the License. #include #include "formats.h" -#include "logger.h" #include "falco_engine.h" @@ -266,31 +265,46 @@ int falco_formats::format_event (lua_State *ls) return 1; } +#include int falco_formats::resolve_tokens(lua_State *ls) { - if (!lua_isuserdata(ls, 1) || - !lua_isstring(ls, 2)) { - lua_pushstring(ls, "Invalid arguments passed to resolve_tokens()"); - lua_error(ls); - } - gen_event* evt = (gen_event*) lua_topointer(ls, 1); - const char *format = (char *) lua_tostring(ls, 2); + // if(!lua_isstring(ls, -1) || + // !lua_isstring(ls, -2) || + // !lua_islightuserdata(ls, -3)) + // { + // lua_pushstring(ls, "Invalid arguments passed to resolve_tokens()"); + // lua_error(ls); + // } + gen_event *evt = (gen_event *)lua_topointer(ls, 1); + string source = luaL_checkstring(ls, 2); + const char *format = (char *)lua_tostring(ls, 3); string sformat = format; - map values; - - s_formatters->resolve_tokens((sinsp_evt *)evt, sformat, values); + map values; + if(source == "syscall") + { + s_formatters->resolve_tokens((sinsp_evt *)evt, sformat, values); + } + // k8s_audit + else + { + json_event_formatter json_formatter(s_engine->json_factory(), sformat); + values = json_formatter.tomap((json_event*) evt); + } lua_newtable(ls); int top = lua_gettop(ls); - for (std::map::iterator it = values.begin(); it != values.end(); ++it) { - const char* key = it->first.c_str(); - const char* value = it->second.c_str(); + for(map::iterator it = values.begin(); it != values.end(); ++it) + { + std::cout << it->first << ":"<< it->second << ", "; + const char *key = it->first.c_str(); + const char *value = it->second.c_str(); lua_pushlstring(ls, key, it->first.size()); lua_pushlstring(ls, value, it->second.size()); lua_settable(ls, top); } + std::cout << std::endl; return 1; } diff --git a/userspace/falco/falco_outputs.cpp b/userspace/falco/falco_outputs.cpp index 610893b3dfd..2728d7b73de 100644 --- a/userspace/falco/falco_outputs.cpp +++ b/userspace/falco/falco_outputs.cpp @@ -306,16 +306,16 @@ int falco_outputs::handle_http(lua_State *ls) int falco_outputs::handle_grpc(lua_State *ls) { // check parameters - if(!lua_isuserdata(ls, 1) || - !lua_isstring(ls, 2) || - !lua_isstring(ls, 3) || - !lua_isstring(ls, 4) || - !lua_isstring(ls, 5) || - !lua_istable(ls, 6)) - { - lua_pushstring(ls, "Invalid arguments passed to handle_grpc()"); - lua_error(ls); - } + // if(!lua_isuserdata(ls, 1) || + // !lua_isstring(ls, 2) || + // !lua_isstring(ls, 3) || + // !lua_isstring(ls, 4) || + // !lua_isstring(ls, 5) || + // !lua_istable(ls, 6)) + // { + // lua_pushstring(ls, "Invalid arguments passed to handle_grpc()"); + // lua_error(ls); + // } response grpc_res = response(); diff --git a/userspace/falco/lua/output.lua b/userspace/falco/lua/output.lua index 751eaee34b5..cfa284c834a 100644 --- a/userspace/falco/lua/output.lua +++ b/userspace/falco/lua/output.lua @@ -170,7 +170,7 @@ function mod.http_reopen() end function mod.grpc(event, rule, source, priority, priority_num, msg, format, options) - fields = formats.resolve_tokens(event, format) + fields = formats.resolve_tokens(event, source, format) c_outputs.handle_grpc(event, rule, source, priority, msg, fields, options) end @@ -178,6 +178,7 @@ function mod.grpc_message(priority, priority_num, msg, options) -- todo end + function mod.grpc_cleanup() end @@ -215,6 +216,16 @@ function output_event(event, rule, source, priority, priority_num, format) msg = formats.format_event(event, rule, source, priority, format) + print("---") + print(event) + print(rule) + print(source) + print(priority) + print(priority_num) + print(msg) + print(format) + print("---") + for index, o in ipairs(outputs) do o.output(event, rule, source, priority, priority_num, msg, format, o.options) end From 356e14330fab238c0a1b2a83aaf7a0a029897f15 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Mon, 23 Sep 2019 18:48:13 +0000 Subject: [PATCH 67/76] fix(userspace/falco): pop output fields lua table and correctly check parameters on the stack Signed-off-by: Leonardo Di Donato --- userspace/engine/formats.cpp | 30 ++++++++++++------------------ userspace/falco/falco_outputs.cpp | 26 ++++++++++++++------------ userspace/falco/lua/output.lua | 10 ---------- 3 files changed, 26 insertions(+), 40 deletions(-) diff --git a/userspace/engine/formats.cpp b/userspace/engine/formats.cpp index 9527c990ded..bea9e5f02bc 100644 --- a/userspace/engine/formats.cpp +++ b/userspace/engine/formats.cpp @@ -265,17 +265,15 @@ int falco_formats::format_event (lua_State *ls) return 1; } -#include - int falco_formats::resolve_tokens(lua_State *ls) { - // if(!lua_isstring(ls, -1) || - // !lua_isstring(ls, -2) || - // !lua_islightuserdata(ls, -3)) - // { - // lua_pushstring(ls, "Invalid arguments passed to resolve_tokens()"); - // lua_error(ls); - // } + if(!lua_isstring(ls, -1) || + !lua_isstring(ls, -2) || + !lua_islightuserdata(ls, -3)) + { + lua_pushstring(ls, "Invalid arguments passed to resolve_tokens()"); + lua_error(ls); + } gen_event *evt = (gen_event *)lua_topointer(ls, 1); string source = luaL_checkstring(ls, 2); const char *format = (char *)lua_tostring(ls, 3); @@ -292,19 +290,15 @@ int falco_formats::resolve_tokens(lua_State *ls) json_event_formatter json_formatter(s_engine->json_factory(), sformat); values = json_formatter.tomap((json_event*) evt); } + // todo(leodido, fntlnz) > check explicitly for k8s_audit, otherwise throw exception lua_newtable(ls); - int top = lua_gettop(ls); - for(map::iterator it = values.begin(); it != values.end(); ++it) + for(auto const& v : values) { - std::cout << it->first << ":"<< it->second << ", "; - const char *key = it->first.c_str(); - const char *value = it->second.c_str(); - lua_pushlstring(ls, key, it->first.size()); - lua_pushlstring(ls, value, it->second.size()); - lua_settable(ls, top); + lua_pushstring(ls, v.first.c_str()); + lua_pushstring(ls, v.second.c_str()); + lua_settable(ls, -3); } - std::cout << std::endl; return 1; } diff --git a/userspace/falco/falco_outputs.cpp b/userspace/falco/falco_outputs.cpp index 2728d7b73de..d248840b674 100644 --- a/userspace/falco/falco_outputs.cpp +++ b/userspace/falco/falco_outputs.cpp @@ -306,16 +306,17 @@ int falco_outputs::handle_http(lua_State *ls) int falco_outputs::handle_grpc(lua_State *ls) { // check parameters - // if(!lua_isuserdata(ls, 1) || - // !lua_isstring(ls, 2) || - // !lua_isstring(ls, 3) || - // !lua_isstring(ls, 4) || - // !lua_isstring(ls, 5) || - // !lua_istable(ls, 6)) - // { - // lua_pushstring(ls, "Invalid arguments passed to handle_grpc()"); - // lua_error(ls); - // } + if(!lua_islightuserdata(ls, -7) || + !lua_isstring(ls, -6) || + !lua_isstring(ls, -5) || + !lua_isstring(ls, -4) || + !lua_isstring(ls, -3) || + !lua_istable(ls, -2) || + !lua_istable(ls, -1)) + { + lua_pushstring(ls, "Invalid arguments passed to handle_grpc()"); + lua_error(ls); + } response grpc_res = response(); @@ -353,11 +354,12 @@ int falco_outputs::handle_grpc(lua_State *ls) // output fields auto& fields = *grpc_res.mutable_output_fields(); - lua_pushnil(ls); + lua_pushnil(ls); // so that lua_next removes it from stack and puts (k, v) on it while (lua_next(ls, 6) != 0) { fields[lua_tostring(ls, -2)] = lua_tostring(ls, -1); - lua_pop(ls, 1); + lua_pop(ls, 1); // remove value, keep key for lua_next } + lua_pop(ls, 1); // pop table falco_output_queue::get().push(grpc_res); diff --git a/userspace/falco/lua/output.lua b/userspace/falco/lua/output.lua index cfa284c834a..6a635d350ef 100644 --- a/userspace/falco/lua/output.lua +++ b/userspace/falco/lua/output.lua @@ -216,16 +216,6 @@ function output_event(event, rule, source, priority, priority_num, format) msg = formats.format_event(event, rule, source, priority, format) - print("---") - print(event) - print(rule) - print(source) - print(priority) - print(priority_num) - print(msg) - print(format) - print("---") - for index, o in ipairs(outputs) do o.output(event, rule, source, priority, priority_num, msg, format, o.options) end From 8d50c805c0feac4f1358c7c62397a179cd3ed189 Mon Sep 17 00:00:00 2001 From: Lorenzo Fontana Date: Tue, 24 Sep 2019 15:32:40 +0200 Subject: [PATCH 68/76] new(userspace/falco): utils file definition with read function Co-Authored-By: Leonardo Di Donato Signed-off-by: Lorenzo Fontana --- userspace/falco/CMakeLists.txt | 1 + userspace/falco/utils.cpp | 36 ++++++++++++++++++++++++++++++++++ userspace/falco/utils.h | 32 ++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 userspace/falco/utils.cpp create mode 100644 userspace/falco/utils.h diff --git a/userspace/falco/CMakeLists.txt b/userspace/falco/CMakeLists.txt index 7b018446cba..adcd6168d2c 100644 --- a/userspace/falco/CMakeLists.txt +++ b/userspace/falco/CMakeLists.txt @@ -42,6 +42,7 @@ add_executable(falco grpc_context.cpp grpc_server_impl.cpp grpc_server.cpp + utils.cpp ${CMAKE_CURRENT_BINARY_DIR}/falco_output.grpc.pb.cc ${CMAKE_CURRENT_BINARY_DIR}/falco_output.pb.cc) diff --git a/userspace/falco/utils.cpp b/userspace/falco/utils.cpp new file mode 100644 index 00000000000..344539cb206 --- /dev/null +++ b/userspace/falco/utils.cpp @@ -0,0 +1,36 @@ +/* +Copyright (C) 2016-2019 The Falco Authors + +This file is part of falco. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#include "utils.h" + +void falco::utils::read(const std::string& filename, std::string& data) +{ + std::ifstream file(filename.c_str(), std::ios::in); + + if(file.is_open()) + { + std::stringstream ss; + ss << file.rdbuf(); + + file.close(); + + data = ss.str(); + } + + return; +} diff --git a/userspace/falco/utils.h b/userspace/falco/utils.h new file mode 100644 index 00000000000..cf466cd145d --- /dev/null +++ b/userspace/falco/utils.h @@ -0,0 +1,32 @@ +/* +Copyright (C) 2016-2019 The Falco Authors + +This file is part of falco. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#pragma once + +#include +#include +#include +#include + +namespace falco +{ +namespace utils +{ +void read(const std::string& filename, std::string& data); +} // namespace utils +} // namespace falco From c446659af01c7529d1d77b1d4345a46574cb8a6a Mon Sep 17 00:00:00 2001 From: Lorenzo Fontana Date: Tue, 24 Sep 2019 15:33:44 +0200 Subject: [PATCH 69/76] new(userspace/falco): namespace for falco grpc Co-Authored-By: Leonardo Di Donato Signed-off-by: Lorenzo Fontana --- userspace/falco/falco.cpp | 2 +- userspace/falco/grpc_context.cpp | 8 ++-- userspace/falco/grpc_context.h | 15 ++++++-- userspace/falco/grpc_server.cpp | 57 +++++++++------------------- userspace/falco/grpc_server.h | 16 +++++--- userspace/falco/grpc_server_impl.cpp | 6 +-- userspace/falco/grpc_server_impl.h | 10 +++-- 7 files changed, 55 insertions(+), 59 deletions(-) diff --git a/userspace/falco/falco.cpp b/userspace/falco/falco.cpp index 14526086c2c..71650a027df 100644 --- a/userspace/falco/falco.cpp +++ b/userspace/falco/falco.cpp @@ -455,7 +455,7 @@ int falco_init(int argc, char **argv) scap_stats cstats; falco_webserver webserver; - falco_grpc_server grpc_server; + falco::grpc::falco_grpc_server grpc_server; std::thread grpc_server_thread; static struct option long_options[] = diff --git a/userspace/falco/grpc_context.cpp b/userspace/falco/grpc_context.cpp index c52570492ad..f689c59812a 100644 --- a/userspace/falco/grpc_context.cpp +++ b/userspace/falco/grpc_context.cpp @@ -20,7 +20,7 @@ limitations under the License. #include "grpc_context.h" -context::context(grpc::ServerContext* ctx): +falco::grpc::context::context(::grpc::ServerContext* ctx): m_ctx(ctx) { std::string session_id; @@ -48,12 +48,12 @@ context::context(grpc::ServerContext* ctx): m_prefix = meta.str(); } -void context::get_metadata(std::string key, std::string& val) +void falco::grpc::context::context::get_metadata(std::string key, std::string& val) { - const std::multimap& client_metadata = m_ctx->client_metadata(); + const std::multimap<::grpc::string_ref, ::grpc::string_ref>& client_metadata = m_ctx->client_metadata(); auto it = client_metadata.find(key); if(it != client_metadata.end()) { val.assign(it->second.data(), it->second.size()); } -} \ No newline at end of file +} diff --git a/userspace/falco/grpc_context.h b/userspace/falco/grpc_context.h index 4781517a1fb..bb8eec62dc0 100644 --- a/userspace/falco/grpc_context.h +++ b/userspace/falco/grpc_context.h @@ -26,26 +26,31 @@ limitations under the License. #include #endif +namespace falco +{ +namespace grpc +{ + const std::string meta_session = "session_id"; const std::string meta_request = "request_id"; class context { public: - context(grpc::ServerContext* ctx); + context(::grpc::ServerContext* ctx); ~context() = default; void get_metadata(std::string key, std::string& val); private: - grpc::ServerContext* m_ctx = nullptr; + ::grpc::ServerContext* m_ctx = nullptr; std::string m_prefix; }; class stream_context : public context { public: - stream_context(grpc::ServerContext* ctx): + stream_context(::grpc::ServerContext* ctx): context(ctx){}; ~stream_context() = default; @@ -59,4 +64,6 @@ class stream_context : public context mutable void* m_stream = nullptr; // Are there more responses to stream? mutable bool m_has_more = false; -}; \ No newline at end of file +}; +} // namespace grpc +} // namespace falco diff --git a/userspace/falco/grpc_server.cpp b/userspace/falco/grpc_server.cpp index d8078c9f6e1..0c4c2cf33f8 100644 --- a/userspace/falco/grpc_server.cpp +++ b/userspace/falco/grpc_server.cpp @@ -25,14 +25,15 @@ limitations under the License. #include "logger.h" #include "grpc_server.h" #include "grpc_context.h" +#include "utils.h" template<> -void request_stream_context::start(falco_grpc_server* srv) +void falco::grpc::request_stream_context::start(falco_grpc_server* srv) { m_state = request_context_base::REQUEST; - m_srv_ctx.reset(new grpc::ServerContext); + m_srv_ctx.reset(new ::grpc::ServerContext); auto srvctx = m_srv_ctx.get(); - m_res_writer.reset(new grpc::ServerAsyncWriter(srvctx)); + m_res_writer.reset(new ::grpc::ServerAsyncWriter(srvctx)); m_stream_ctx.reset(); m_req.Clear(); auto cq = srv->m_completion_queue.get(); @@ -40,7 +41,7 @@ void request_stream_context::start(falco_grpc_server* srv) } template<> -void request_stream_context::process(falco_grpc_server* srv) +void falco::grpc::request_stream_context::process(falco_grpc_server* srv) { // When it is the 1st process call if(m_state == request_context_base::REQUEST) @@ -64,12 +65,12 @@ void request_stream_context::process(falco_grpc_server* srv) // Communicate to the gRPC runtime that we have finished. // The memory address of `this` instance uniquely identifies the event. m_state = request_context_base::FINISH; - m_res_writer->Finish(grpc::Status::OK, this); + m_res_writer->Finish(::grpc::Status::OK, this); } } template<> -void request_stream_context::end(falco_grpc_server* srv, bool errored) +void falco::grpc::request_stream_context::end(falco_grpc_server* srv, bool errored) { if(m_stream_ctx) { @@ -83,7 +84,7 @@ void request_stream_context::end(falco_grpc_server* srv, bool start(srv); } -void falco_grpc_server::thread_process(int thread_index) +void falco::grpc::falco_grpc_server::thread_process(int thread_index) { void* tag = nullptr; @@ -144,29 +145,7 @@ void falco_grpc_server::thread_process(int thread_index) ctx.start(this); \ } -// todo(fntlnz, leodido) > cleanup this part (paths from config, read, includes) -#include -#include -#include - -void read(const std::string& filename, std::string& data) -{ - std::ifstream file(filename.c_str(), std::ios::in); - - if(file.is_open()) - { - std::stringstream ss; - ss << file.rdbuf(); - - file.close(); - - data = ss.str(); - } - - return; -} - -void falco_grpc_server::init(std::string server_addr, int threadiness, std::string private_key, std::string cert_chain, std::string root_certs) +void falco::grpc::falco_grpc_server::init(std::string server_addr, int threadiness, std::string private_key, std::string cert_chain, std::string root_certs) { m_server_addr = server_addr; m_threadiness = threadiness; @@ -175,26 +154,26 @@ void falco_grpc_server::init(std::string server_addr, int threadiness, std::stri m_root_certs = root_certs; } -void falco_grpc_server::run() +void falco::grpc::falco_grpc_server::run() { string private_key; string cert_chain; string root_certs; - read(m_cert_chain, cert_chain); - read(m_private_key, private_key); - read(m_root_certs, root_certs); + falco::utils::read(m_cert_chain, cert_chain); + falco::utils::read(m_private_key, private_key); + falco::utils::read(m_root_certs, root_certs); - grpc::SslServerCredentialsOptions::PemKeyCertPair cert_pair{private_key, cert_chain}; + ::grpc::SslServerCredentialsOptions::PemKeyCertPair cert_pair{private_key, cert_chain}; - grpc::SslServerCredentialsOptions ssl_opts(GRPC_SSL_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_AND_VERIFY); + ::grpc::SslServerCredentialsOptions ssl_opts(GRPC_SSL_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_AND_VERIFY); ssl_opts.pem_root_certs = root_certs; ssl_opts.pem_key_cert_pairs.push_back(cert_pair); // Setup server - grpc::ServerBuilder builder; + ::grpc::ServerBuilder builder; // Listen on the given address without any authentication mechanism. - builder.AddListeningPort(m_server_addr, grpc::SslServerCredentials(ssl_opts)); + builder.AddListeningPort(m_server_addr, ::grpc::SslServerCredentials(ssl_opts)); builder.RegisterService(&m_svc); // builder.SetMaxSendMessageSize(GRPC_MAX_MESSAGE_SIZE); // testing max message size? @@ -225,7 +204,7 @@ void falco_grpc_server::run() stop(); } -void falco_grpc_server::stop() +void falco::grpc::falco_grpc_server::stop() { falco_logger::log(LOG_INFO, "Shutting down gRPC server. Waiting until external connections are closed by clients\n"); m_server->Shutdown(); diff --git a/userspace/falco/grpc_server.h b/userspace/falco/grpc_server.h index 672aec201d8..f40935babc7 100644 --- a/userspace/falco/grpc_server.h +++ b/userspace/falco/grpc_server.h @@ -23,6 +23,10 @@ limitations under the License. #include "grpc_server_impl.h" +namespace falco +{ +namespace grpc +{ class falco_grpc_server : public falco_grpc_server_impl { public: @@ -45,7 +49,7 @@ class falco_grpc_server : public falco_grpc_server_impl void stop(); service::AsyncService m_svc; - std::unique_ptr m_completion_queue; + std::unique_ptr<::grpc::ServerCompletionQueue> m_completion_queue; private: std::string m_server_addr; @@ -54,7 +58,7 @@ class falco_grpc_server : public falco_grpc_server_impl std::string m_cert_chain; std::string m_root_certs; - std::unique_ptr m_server; + std::unique_ptr<::grpc::Server> m_server; std::vector m_threads; }; @@ -64,7 +68,7 @@ class request_context_base request_context_base() = default; ~request_context_base() = default; - std::unique_ptr m_srv_ctx; + std::unique_ptr<::grpc::ServerContext> m_srv_ctx; enum : char { UNKNOWN = 0, @@ -93,14 +97,16 @@ class request_stream_context : public request_context_base void (falco_grpc_server::*m_process_func)(const stream_context&, const Request&, Response&); // Pointer to function that requests the system to start processing given requests - void (service::AsyncService::*m_request_func)(grpc::ServerContext*, Request*, grpc::ServerAsyncWriter*, grpc::CompletionQueue*, grpc::ServerCompletionQueue*, void*); + void (service::AsyncService::*m_request_func)(::grpc::ServerContext*, Request*, ::grpc::ServerAsyncWriter*, ::grpc::CompletionQueue*, ::grpc::ServerCompletionQueue*, void*); void start(falco_grpc_server* srv); void process(falco_grpc_server* srv); void end(falco_grpc_server* srv, bool isError); private: - std::unique_ptr> m_res_writer; + std::unique_ptr<::grpc::ServerAsyncWriter> m_res_writer; std::unique_ptr m_stream_ctx; Request m_req; }; +} // namespace grpc +} // namespace falco diff --git a/userspace/falco/grpc_server_impl.cpp b/userspace/falco/grpc_server_impl.cpp index 32cb99f72e9..67497be6d18 100644 --- a/userspace/falco/grpc_server_impl.cpp +++ b/userspace/falco/grpc_server_impl.cpp @@ -18,7 +18,7 @@ limitations under the License. #include "grpc_server_impl.h" -bool falco_grpc_server_impl::is_running() +bool falco::grpc::falco_grpc_server_impl::is_running() { if(m_stop) { @@ -27,7 +27,7 @@ bool falco_grpc_server_impl::is_running() return true; } -void falco_grpc_server_impl::subscribe(const stream_context& ctx, const request& req, response& res) +void falco::grpc::falco_grpc_server_impl::subscribe(const stream_context& ctx, const output::request& req, output::response& res) { if(ctx.m_status == stream_context::SUCCESS || ctx.m_status == stream_context::ERROR) { @@ -52,7 +52,7 @@ void falco_grpc_server_impl::subscribe(const stream_context& ctx, const request& } } -void falco_grpc_server_impl::shutdown() +void falco::grpc::falco_grpc_server_impl::shutdown() { m_stop = true; } diff --git a/userspace/falco/grpc_server_impl.h b/userspace/falco/grpc_server_impl.h index 54277a2258b..ef3185566d1 100644 --- a/userspace/falco/grpc_server_impl.h +++ b/userspace/falco/grpc_server_impl.h @@ -24,8 +24,10 @@ limitations under the License. #include "falco_output.grpc.pb.h" #include "grpc_context.h" -using namespace falco::output; - +namespace falco +{ +namespace grpc +{ class falco_grpc_server_impl { public: @@ -37,8 +39,10 @@ class falco_grpc_server_impl protected: bool is_running(); - void subscribe(const stream_context& ctx, const request& req, response& res); + void subscribe(const stream_context& ctx, const output::request& req, output::response& res); private: std::atomic m_stop{false}; }; +} // namespace grpc +} // namespace falco From 65fb36d975631eb61822566871f17903cb4a4928 Mon Sep 17 00:00:00 2001 From: Lorenzo Fontana Date: Tue, 24 Sep 2019 15:41:39 +0200 Subject: [PATCH 70/76] update(userspace/falco): falco_grpc_server is now just server Co-Authored-By: Leonardo Di Donato Signed-off-by: Lorenzo Fontana --- userspace/falco/falco.cpp | 2 +- userspace/falco/grpc_server.cpp | 26 +++++++++++++------------- userspace/falco/grpc_server.h | 22 +++++++++++----------- userspace/falco/grpc_server_impl.cpp | 6 +++--- userspace/falco/grpc_server_impl.h | 6 +++--- 5 files changed, 31 insertions(+), 31 deletions(-) diff --git a/userspace/falco/falco.cpp b/userspace/falco/falco.cpp index 71650a027df..7b2555620f9 100644 --- a/userspace/falco/falco.cpp +++ b/userspace/falco/falco.cpp @@ -455,7 +455,7 @@ int falco_init(int argc, char **argv) scap_stats cstats; falco_webserver webserver; - falco::grpc::falco_grpc_server grpc_server; + falco::grpc::server grpc_server; std::thread grpc_server_thread; static struct option long_options[] = diff --git a/userspace/falco/grpc_server.cpp b/userspace/falco/grpc_server.cpp index 0c4c2cf33f8..a8e6b8be2e6 100644 --- a/userspace/falco/grpc_server.cpp +++ b/userspace/falco/grpc_server.cpp @@ -28,7 +28,7 @@ limitations under the License. #include "utils.h" template<> -void falco::grpc::request_stream_context::start(falco_grpc_server* srv) +void falco::grpc::request_stream_context::start(server* srv) { m_state = request_context_base::REQUEST; m_srv_ctx.reset(new ::grpc::ServerContext); @@ -41,7 +41,7 @@ void falco::grpc::request_stream_context -void falco::grpc::request_stream_context::process(falco_grpc_server* srv) +void falco::grpc::request_stream_context::process(server* srv) { // When it is the 1st process call if(m_state == request_context_base::REQUEST) @@ -70,7 +70,7 @@ void falco::grpc::request_stream_context -void falco::grpc::request_stream_context::end(falco_grpc_server* srv, bool errored) +void falco::grpc::request_stream_context::end(server* srv, bool errored) { if(m_stream_ctx) { @@ -84,7 +84,7 @@ void falco::grpc::request_stream_context> RPC##_contexts(CONTEXT_COUNT); \ for(request_stream_context & ctx : RPC##_contexts) \ { \ - ctx.m_process_func = &falco_grpc_server::IMPL; \ + ctx.m_process_func = &server::IMPL; \ ctx.m_request_func = &service::AsyncService::Request##RPC; \ ctx.start(this); \ } -void falco::grpc::falco_grpc_server::init(std::string server_addr, int threadiness, std::string private_key, std::string cert_chain, std::string root_certs) +void falco::grpc::server::init(std::string server_addr, int threadiness, std::string private_key, std::string cert_chain, std::string root_certs) { m_server_addr = server_addr; m_threadiness = threadiness; @@ -154,7 +154,7 @@ void falco::grpc::falco_grpc_server::init(std::string server_addr, int threadine m_root_certs = root_certs; } -void falco::grpc::falco_grpc_server::run() +void falco::grpc::server::run() { string private_key; string cert_chain; @@ -181,12 +181,12 @@ void falco::grpc::falco_grpc_server::run() m_completion_queue = builder.AddCompletionQueue(); m_server = builder.BuildAndStart(); - falco_logger::log(LOG_INFO, "Starting gRPC webserver at " + m_server_addr + "\n"); + falco_logger::log(LOG_INFO, "Starting gRPC server at " + m_server_addr + "\n"); // Create context for server threads // The number of contexts is multiple of the number of threads // This defines the number of simultaneous completion queue requests of the same type (service::AsyncService::Request##RPC) - // For this approach to be sufficient falco_grpc_server::IMPL have to be fast + // For this approach to be sufficient server::IMPL have to be fast int context_count = m_threadiness * 10; PROCESS_STREAM(request, response, subscribe, subscribe, context_count) @@ -194,17 +194,17 @@ void falco::grpc::falco_grpc_server::run() int thread_idx = 0; for(std::thread& thread : m_threads) { - thread = std::thread(&falco_grpc_server::thread_process, this, thread_idx++); + thread = std::thread(&server::thread_process, this, thread_idx++); } - while(falco_grpc_server_impl::is_running()) + while(server_impl::is_running()) { sleep(1); } stop(); } -void falco::grpc::falco_grpc_server::stop() +void falco::grpc::server::stop() { falco_logger::log(LOG_INFO, "Shutting down gRPC server. Waiting until external connections are closed by clients\n"); m_server->Shutdown(); @@ -228,5 +228,5 @@ void falco::grpc::falco_grpc_server::stop() { } - falco_logger::log(LOG_INFO, "gRPC shutdown is now complete\n"); + falco_logger::log(LOG_INFO, "Shutting down gRPC server complete\n"); } diff --git a/userspace/falco/grpc_server.h b/userspace/falco/grpc_server.h index f40935babc7..461a77a0b82 100644 --- a/userspace/falco/grpc_server.h +++ b/userspace/falco/grpc_server.h @@ -27,13 +27,13 @@ namespace falco { namespace grpc { -class falco_grpc_server : public falco_grpc_server_impl +class server : public server_impl { public: - falco_grpc_server() + server() { } - falco_grpc_server(std::string server_addr, int threadiness, std::string private_key, std::string cert_chain, std::string root_certs): + server(std::string server_addr, int threadiness, std::string private_key, std::string cert_chain, std::string root_certs): m_server_addr(server_addr), m_threadiness(threadiness), m_private_key(private_key), @@ -41,7 +41,7 @@ class falco_grpc_server : public falco_grpc_server_impl m_root_certs(root_certs) { } - virtual ~falco_grpc_server() = default; + virtual ~server() = default; void init(std::string server_addr, int threadiness, std::string private_key, std::string cert_chain, std::string root_certs); void thread_process(int thread_index); @@ -76,9 +76,9 @@ class request_context_base WRITE, FINISH } m_state = UNKNOWN; - virtual void start(falco_grpc_server* srv) = 0; - virtual void process(falco_grpc_server* srv) = 0; - virtual void end(falco_grpc_server* srv, bool isError) = 0; + virtual void start(server* srv) = 0; + virtual void process(server* srv) = 0; + virtual void end(server* srv, bool isError) = 0; }; // @@ -94,14 +94,14 @@ class request_stream_context : public request_context_base ~request_stream_context() = default; // Pointer to function that does actual processing - void (falco_grpc_server::*m_process_func)(const stream_context&, const Request&, Response&); + void (server::*m_process_func)(const stream_context&, const Request&, Response&); // Pointer to function that requests the system to start processing given requests void (service::AsyncService::*m_request_func)(::grpc::ServerContext*, Request*, ::grpc::ServerAsyncWriter*, ::grpc::CompletionQueue*, ::grpc::ServerCompletionQueue*, void*); - void start(falco_grpc_server* srv); - void process(falco_grpc_server* srv); - void end(falco_grpc_server* srv, bool isError); + void start(server* srv); + void process(server* srv); + void end(server* srv, bool isError); private: std::unique_ptr<::grpc::ServerAsyncWriter> m_res_writer; diff --git a/userspace/falco/grpc_server_impl.cpp b/userspace/falco/grpc_server_impl.cpp index 67497be6d18..5b74dbdd1bd 100644 --- a/userspace/falco/grpc_server_impl.cpp +++ b/userspace/falco/grpc_server_impl.cpp @@ -18,7 +18,7 @@ limitations under the License. #include "grpc_server_impl.h" -bool falco::grpc::falco_grpc_server_impl::is_running() +bool falco::grpc::server_impl::is_running() { if(m_stop) { @@ -27,7 +27,7 @@ bool falco::grpc::falco_grpc_server_impl::is_running() return true; } -void falco::grpc::falco_grpc_server_impl::subscribe(const stream_context& ctx, const output::request& req, output::response& res) +void falco::grpc::server_impl::subscribe(const stream_context& ctx, const output::request& req, output::response& res) { if(ctx.m_status == stream_context::SUCCESS || ctx.m_status == stream_context::ERROR) { @@ -52,7 +52,7 @@ void falco::grpc::falco_grpc_server_impl::subscribe(const stream_context& ctx, c } } -void falco::grpc::falco_grpc_server_impl::shutdown() +void falco::grpc::server_impl::shutdown() { m_stop = true; } diff --git a/userspace/falco/grpc_server_impl.h b/userspace/falco/grpc_server_impl.h index ef3185566d1..d1ff4fe0e3a 100644 --- a/userspace/falco/grpc_server_impl.h +++ b/userspace/falco/grpc_server_impl.h @@ -28,11 +28,11 @@ namespace falco { namespace grpc { -class falco_grpc_server_impl +class server_impl { public: - falco_grpc_server_impl() = default; - ~falco_grpc_server_impl() = default; + server_impl() = default; + ~server_impl() = default; void shutdown(); From 92391f182ac622cdc54a9707299dd1f325c9c613 Mon Sep 17 00:00:00 2001 From: Lorenzo Fontana Date: Tue, 24 Sep 2019 17:31:22 +0200 Subject: [PATCH 71/76] chore(userspace/falco): better organization of schema and grpc server Co-Authored-By: Leonardo Di Donato Signed-off-by: Lorenzo Fontana --- userspace/falco/CMakeLists.txt | 8 +++-- userspace/falco/falco.cpp | 1 - userspace/falco/falco_output.proto | 50 ++++------------------------ userspace/falco/falco_output_queue.h | 24 +++++++------ userspace/falco/falco_outputs.cpp | 10 +++--- userspace/falco/grpc_server.cpp | 30 +++++++---------- userspace/falco/grpc_server.h | 4 +-- userspace/falco/grpc_server_impl.cpp | 5 +-- userspace/falco/grpc_server_impl.h | 2 -- userspace/falco/schema.proto | 42 +++++++++++++++++++++++ 10 files changed, 91 insertions(+), 85 deletions(-) create mode 100644 userspace/falco/schema.proto diff --git a/userspace/falco/CMakeLists.txt b/userspace/falco/CMakeLists.txt index adcd6168d2c..6b8d0f9a8a5 100644 --- a/userspace/falco/CMakeLists.txt +++ b/userspace/falco/CMakeLists.txt @@ -20,13 +20,16 @@ if(NOT SYSDIG_DIR) endif() configure_file("${SYSDIG_DIR}/userspace/sysdig/config_sysdig.h.in" config_sysdig.h) + add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/falco_output.grpc.pb.cc ${CMAKE_CURRENT_BINARY_DIR}/falco_output.grpc.pb.h ${CMAKE_CURRENT_BINARY_DIR}/falco_output.pb.cc ${CMAKE_CURRENT_BINARY_DIR}/falco_output.pb.h + ${CMAKE_CURRENT_BINARY_DIR}/schema.pb.cc + ${CMAKE_CURRENT_BINARY_DIR}/schema.pb.h COMMENT "Generate gRPC code for falco_output" DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/falco_output.proto - COMMAND ${PROTOC} -I ${CMAKE_CURRENT_SOURCE_DIR} --cpp_out=. ${CMAKE_CURRENT_SOURCE_DIR}/falco_output.proto + COMMAND ${PROTOC} -I ${CMAKE_CURRENT_SOURCE_DIR} --cpp_out=. ${CMAKE_CURRENT_SOURCE_DIR}/falco_output.proto ${CMAKE_CURRENT_SOURCE_DIR}/schema.proto COMMAND ${PROTOC} -I ${CMAKE_CURRENT_SOURCE_DIR} --grpc_out=. --plugin=protoc-gen-grpc=${GRPC_CPP_PLUGIN} ${CMAKE_CURRENT_SOURCE_DIR}/falco_output.proto WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) @@ -44,7 +47,8 @@ add_executable(falco grpc_server.cpp utils.cpp ${CMAKE_CURRENT_BINARY_DIR}/falco_output.grpc.pb.cc - ${CMAKE_CURRENT_BINARY_DIR}/falco_output.pb.cc) + ${CMAKE_CURRENT_BINARY_DIR}/falco_output.pb.cc + ${CMAKE_CURRENT_BINARY_DIR}/schema.pb.cc) target_include_directories(falco PUBLIC "${SYSDIG_DIR}/userspace/sysdig" diff --git a/userspace/falco/falco.cpp b/userspace/falco/falco.cpp index 7b2555620f9..469f51e26fb 100644 --- a/userspace/falco/falco.cpp +++ b/userspace/falco/falco.cpp @@ -48,7 +48,6 @@ limitations under the License. #include "statsfilewriter.h" #include "webserver.h" #include "grpc_server.h" -#include "falco_output_queue.h" typedef function open_t; diff --git a/userspace/falco/falco_output.proto b/userspace/falco/falco_output.proto index 9bf6732a21f..44cbf72cb77 100644 --- a/userspace/falco/falco_output.proto +++ b/userspace/falco/falco_output.proto @@ -1,6 +1,7 @@ syntax = "proto3"; import "google/protobuf/timestamp.proto"; +import "schema.proto"; package falco.output; @@ -9,54 +10,17 @@ service service { } message request { - string duration = 1; // TODO(leodido, fntlnz): not handled yet but keeping for reference. - bool keepalive = 2; -} - -enum priority { - option allow_alias = true; - EMERGENCY = 0; - emergency = 0; - Emergency = 0; - ALERT = 1; - alert = 1; - Alert = 1; - CRITICAL = 2; - critical = 2; - Critical = 2; - ERROR = 3; - error = 3; - Error = 3; - WARNING = 4; - warning = 4; - Warning = 4; - NOTICE = 5; - notice = 5; - Notice = 5; - INFORMATIONAL = 6; - informational = 6; - Informational = 6; - DEBUG = 7; - debug = 7; - Debug = 7; -} - -enum source { - option allow_alias = true; - SYSCALL = 0; - syscall = 0; - Syscall = 0; - K8S_AUDIT = 1; - k8s_audit = 1; - K8s_audit = 1; - K8S_audit = 1; + bool keepalive = 1; + // string duration = 2; // TODO(leodido, fntlnz): not handled yet but keeping for reference. + // repeated string tags = 3; // TODO(leodido, fntlnz): not handled yet but keeping for reference. } message response { google.protobuf.Timestamp time = 1; - priority priority = 2; - source source = 3; + falco.schema.priority priority = 2; + falco.schema.source source = 3; string rule = 4; string output = 5; map output_fields = 6; + repeated string tags = 7; } \ No newline at end of file diff --git a/userspace/falco/falco_output_queue.h b/userspace/falco/falco_output_queue.h index bc41d24e82d..639ba952867 100644 --- a/userspace/falco/falco_output_queue.h +++ b/userspace/falco/falco_output_queue.h @@ -21,16 +21,18 @@ limitations under the License. #include "falco_output.pb.h" #include "tbb/concurrent_queue.h" -using namespace falco::output; - -typedef tbb::concurrent_queue falco_output_response_cq; +namespace falco +{ +namespace output +{ +typedef tbb::concurrent_queue response_cq; -class falco_output_queue +class queue { public: - static falco_output_queue& get() + static queue& get() { - static falco_output_queue instance; + static queue instance; return instance; } @@ -45,14 +47,16 @@ class falco_output_queue } private: - falco_output_queue() + queue() { } - falco_output_response_cq m_queue; + response_cq m_queue; // We can use the better technique of deleting the methods we don't want. public: - falco_output_queue(falco_output_queue const&) = delete; - void operator=(falco_output_queue const&) = delete; + queue(queue const&) = delete; + void operator=(queue const&) = delete; }; +} // namespace output +} // namespace falco diff --git a/userspace/falco/falco_outputs.cpp b/userspace/falco/falco_outputs.cpp index d248840b674..581ec587a74 100644 --- a/userspace/falco/falco_outputs.cpp +++ b/userspace/falco/falco_outputs.cpp @@ -329,9 +329,9 @@ int falco_outputs::handle_grpc(lua_State *ls) grpc_res.set_rule((char *)lua_tostring(ls, 2)); // source - source s = source::SYSCALL; + falco::schema::source s = falco::schema::source::SYSCALL; string sstr = (char *)lua_tostring(ls, 3); - if(!source_Parse(sstr, &s)) + if(!falco::schema::source_Parse(sstr, &s)) { lua_pushstring(ls, "Unknown source passed to to handle_grpc()"); lua_error(ls); @@ -339,9 +339,9 @@ int falco_outputs::handle_grpc(lua_State *ls) grpc_res.set_source(s); // priority - priority p = priority::EMERGENCY; + falco::schema::priority p = falco::schema::priority::EMERGENCY; string pstr = (char *)lua_tostring(ls, 4); - if(!priority_Parse(pstr, &p)) + if(!falco::schema::priority_Parse(pstr, &p)) { lua_pushstring(ls, "Unknown priority passed to to handle_grpc()"); lua_error(ls); @@ -361,7 +361,7 @@ int falco_outputs::handle_grpc(lua_State *ls) } lua_pop(ls, 1); // pop table - falco_output_queue::get().push(grpc_res); + falco::output::queue::get().push(grpc_res); return 1; } \ No newline at end of file diff --git a/userspace/falco/grpc_server.cpp b/userspace/falco/grpc_server.cpp index a8e6b8be2e6..a0a448ce203 100644 --- a/userspace/falco/grpc_server.cpp +++ b/userspace/falco/grpc_server.cpp @@ -33,7 +33,7 @@ void falco::grpc::request_stream_context(srvctx)); + m_res_writer.reset(new ::grpc::ServerAsyncWriter(srvctx)); m_stream_ctx.reset(); m_req.Clear(); auto cq = srv->m_completion_queue.get(); @@ -51,7 +51,7 @@ void falco::grpc::request_stream_context*m_process_func)(*m_stream_ctx, m_req, res); // When there still are more responses to stream @@ -77,7 +77,7 @@ void falco::grpc::request_stream_contextm_status = errored ? stream_context::ERROR : stream_context::SUCCESS; // Complete the processing - response res; + output::response res; (srv->*m_process_func)(*m_stream_ctx, m_req, res); // subscribe() } @@ -136,13 +136,13 @@ void falco::grpc::server::thread_process(int thread_index) // // Create array of contexts and start processing streaming RPC request. // -#define PROCESS_STREAM(REQ, RESP, RPC, IMPL, CONTEXT_COUNT) \ - std::vector> RPC##_contexts(CONTEXT_COUNT); \ - for(request_stream_context & ctx : RPC##_contexts) \ - { \ - ctx.m_process_func = &server::IMPL; \ - ctx.m_request_func = &service::AsyncService::Request##RPC; \ - ctx.start(this); \ +#define REGISTER_STREAM(REQ, RESP, RPC, IMPL, CONTEXT_NUM) \ + std::vector> RPC##_contexts(CONTEXT_NUM); \ + for(request_stream_context & ctx : RPC##_contexts) \ + { \ + ctx.m_process_func = &server::IMPL; \ + ctx.m_request_func = &output::service::AsyncService::Request##RPC; \ + ctx.start(this); \ } void falco::grpc::server::init(std::string server_addr, int threadiness, std::string private_key, std::string cert_chain, std::string root_certs) @@ -170,25 +170,19 @@ void falco::grpc::server::run() ssl_opts.pem_root_certs = root_certs; ssl_opts.pem_key_cert_pairs.push_back(cert_pair); - // Setup server ::grpc::ServerBuilder builder; - // Listen on the given address without any authentication mechanism. builder.AddListeningPort(m_server_addr, ::grpc::SslServerCredentials(ssl_opts)); builder.RegisterService(&m_svc); - // builder.SetMaxSendMessageSize(GRPC_MAX_MESSAGE_SIZE); // testing max message size? - // builder.SetMaxReceiveMessageSize(GRPC_MAX_MESSAGE_SIZE); // testing max message size? - m_completion_queue = builder.AddCompletionQueue(); m_server = builder.BuildAndStart(); falco_logger::log(LOG_INFO, "Starting gRPC server at " + m_server_addr + "\n"); - // Create context for server threads // The number of contexts is multiple of the number of threads // This defines the number of simultaneous completion queue requests of the same type (service::AsyncService::Request##RPC) // For this approach to be sufficient server::IMPL have to be fast - int context_count = m_threadiness * 10; - PROCESS_STREAM(request, response, subscribe, subscribe, context_count) + int context_num = m_threadiness * 10; + REGISTER_STREAM(output::request, output::response, subscribe, subscribe, context_num) m_threads.resize(m_threadiness); int thread_idx = 0; diff --git a/userspace/falco/grpc_server.h b/userspace/falco/grpc_server.h index 461a77a0b82..5d2c2d526fd 100644 --- a/userspace/falco/grpc_server.h +++ b/userspace/falco/grpc_server.h @@ -48,7 +48,7 @@ class server : public server_impl void run(); void stop(); - service::AsyncService m_svc; + output::service::AsyncService m_svc; std::unique_ptr<::grpc::ServerCompletionQueue> m_completion_queue; private: @@ -97,7 +97,7 @@ class request_stream_context : public request_context_base void (server::*m_process_func)(const stream_context&, const Request&, Response&); // Pointer to function that requests the system to start processing given requests - void (service::AsyncService::*m_request_func)(::grpc::ServerContext*, Request*, ::grpc::ServerAsyncWriter*, ::grpc::CompletionQueue*, ::grpc::ServerCompletionQueue*, void*); + void (output::service::AsyncService::*m_request_func)(::grpc::ServerContext*, Request*, ::grpc::ServerAsyncWriter*, ::grpc::CompletionQueue*, ::grpc::ServerCompletionQueue*, void*); void start(server* srv); void process(server* srv); diff --git a/userspace/falco/grpc_server_impl.cpp b/userspace/falco/grpc_server_impl.cpp index 5b74dbdd1bd..f423b5af629 100644 --- a/userspace/falco/grpc_server_impl.cpp +++ b/userspace/falco/grpc_server_impl.cpp @@ -17,6 +17,7 @@ limitations under the License. */ #include "grpc_server_impl.h" +#include "falco_output_queue.h" bool falco::grpc::server_impl::is_running() { @@ -39,12 +40,12 @@ void falco::grpc::server_impl::subscribe(const stream_context& ctx, const output { // Start (or continue) streaming // ctx.m_status == stream_context::STREAMING - if(falco_output_queue::get().try_pop(res) && !req.keepalive()) + if(output::queue::get().try_pop(res) && !req.keepalive()) { ctx.m_has_more = true; return; } - while(is_running() && !falco_output_queue::get().try_pop(res) && req.keepalive()) + while(is_running() && !output::queue::get().try_pop(res) && req.keepalive()) { } diff --git a/userspace/falco/grpc_server_impl.h b/userspace/falco/grpc_server_impl.h index d1ff4fe0e3a..a430d976de0 100644 --- a/userspace/falco/grpc_server_impl.h +++ b/userspace/falco/grpc_server_impl.h @@ -19,8 +19,6 @@ limitations under the License. #pragma once #include - -#include "falco_output_queue.h" #include "falco_output.grpc.pb.h" #include "grpc_context.h" diff --git a/userspace/falco/schema.proto b/userspace/falco/schema.proto new file mode 100644 index 00000000000..262d096cf71 --- /dev/null +++ b/userspace/falco/schema.proto @@ -0,0 +1,42 @@ +syntax = "proto3"; + +package falco.schema; + +enum priority { + option allow_alias = true; + EMERGENCY = 0; + emergency = 0; + Emergency = 0; + ALERT = 1; + alert = 1; + Alert = 1; + CRITICAL = 2; + critical = 2; + Critical = 2; + ERROR = 3; + error = 3; + Error = 3; + WARNING = 4; + warning = 4; + Warning = 4; + NOTICE = 5; + notice = 5; + Notice = 5; + INFORMATIONAL = 6; + informational = 6; + Informational = 6; + DEBUG = 7; + debug = 7; + Debug = 7; +} + +enum source { + option allow_alias = true; + SYSCALL = 0; + syscall = 0; + Syscall = 0; + K8S_AUDIT = 1; + k8s_audit = 1; + K8s_audit = 1; + K8S_audit = 1; +} \ No newline at end of file From dee2a2b25b2000d530bf643bada0394b4413e82d Mon Sep 17 00:00:00 2001 From: Lorenzo Fontana Date: Tue, 24 Sep 2019 18:10:39 +0200 Subject: [PATCH 72/76] chore(userspace/falco): we don't support tags yet Co-Authored-By: Leonardo Di Donato Signed-off-by: Lorenzo Fontana --- userspace/falco/falco_output.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/userspace/falco/falco_output.proto b/userspace/falco/falco_output.proto index 44cbf72cb77..21a41df780f 100644 --- a/userspace/falco/falco_output.proto +++ b/userspace/falco/falco_output.proto @@ -22,5 +22,5 @@ message response { string rule = 4; string output = 5; map output_fields = 6; - repeated string tags = 7; + // repeated string tags = 7; // TODO(leodido,fntlnz): tags not supported yet, keeping for reference } \ No newline at end of file From 6e47f585cbad277e2d43ae8859db302a7c0bd4c2 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Tue, 24 Sep 2019 16:47:28 +0000 Subject: [PATCH 73/76] update(userspace/falco): specify go packages into protobuf Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/CMakeLists.txt | 20 +++++++++---------- userspace/falco/falco_output_queue.h | 2 +- userspace/falco/grpc_server_impl.h | 2 +- .../{falco_output.proto => output.proto} | 4 +++- userspace/falco/schema.proto | 4 +++- 5 files changed, 18 insertions(+), 14 deletions(-) rename userspace/falco/{falco_output.proto => output.proto} (90%) diff --git a/userspace/falco/CMakeLists.txt b/userspace/falco/CMakeLists.txt index 6b8d0f9a8a5..b95b79444ed 100644 --- a/userspace/falco/CMakeLists.txt +++ b/userspace/falco/CMakeLists.txt @@ -21,16 +21,16 @@ endif() configure_file("${SYSDIG_DIR}/userspace/sysdig/config_sysdig.h.in" config_sysdig.h) -add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/falco_output.grpc.pb.cc - ${CMAKE_CURRENT_BINARY_DIR}/falco_output.grpc.pb.h - ${CMAKE_CURRENT_BINARY_DIR}/falco_output.pb.cc - ${CMAKE_CURRENT_BINARY_DIR}/falco_output.pb.h +add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/output.grpc.pb.cc + ${CMAKE_CURRENT_BINARY_DIR}/output.grpc.pb.h + ${CMAKE_CURRENT_BINARY_DIR}/output.pb.cc + ${CMAKE_CURRENT_BINARY_DIR}/output.pb.h ${CMAKE_CURRENT_BINARY_DIR}/schema.pb.cc ${CMAKE_CURRENT_BINARY_DIR}/schema.pb.h - COMMENT "Generate gRPC code for falco_output" - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/falco_output.proto - COMMAND ${PROTOC} -I ${CMAKE_CURRENT_SOURCE_DIR} --cpp_out=. ${CMAKE_CURRENT_SOURCE_DIR}/falco_output.proto ${CMAKE_CURRENT_SOURCE_DIR}/schema.proto - COMMAND ${PROTOC} -I ${CMAKE_CURRENT_SOURCE_DIR} --grpc_out=. --plugin=protoc-gen-grpc=${GRPC_CPP_PLUGIN} ${CMAKE_CURRENT_SOURCE_DIR}/falco_output.proto + COMMENT "Generate gRPC code" + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/output.proto + COMMAND ${PROTOC} -I ${CMAKE_CURRENT_SOURCE_DIR} --cpp_out=. ${CMAKE_CURRENT_SOURCE_DIR}/output.proto ${CMAKE_CURRENT_SOURCE_DIR}/schema.proto + COMMAND ${PROTOC} -I ${CMAKE_CURRENT_SOURCE_DIR} --grpc_out=. --plugin=protoc-gen-grpc=${GRPC_CPP_PLUGIN} ${CMAKE_CURRENT_SOURCE_DIR}/output.proto WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) add_executable(falco @@ -46,8 +46,8 @@ add_executable(falco grpc_server_impl.cpp grpc_server.cpp utils.cpp - ${CMAKE_CURRENT_BINARY_DIR}/falco_output.grpc.pb.cc - ${CMAKE_CURRENT_BINARY_DIR}/falco_output.pb.cc + ${CMAKE_CURRENT_BINARY_DIR}/output.grpc.pb.cc + ${CMAKE_CURRENT_BINARY_DIR}/output.pb.cc ${CMAKE_CURRENT_BINARY_DIR}/schema.pb.cc) target_include_directories(falco PUBLIC diff --git a/userspace/falco/falco_output_queue.h b/userspace/falco/falco_output_queue.h index 639ba952867..101fa1cf7f0 100644 --- a/userspace/falco/falco_output_queue.h +++ b/userspace/falco/falco_output_queue.h @@ -18,7 +18,7 @@ limitations under the License. #pragma once -#include "falco_output.pb.h" +#include "output.pb.h" #include "tbb/concurrent_queue.h" namespace falco diff --git a/userspace/falco/grpc_server_impl.h b/userspace/falco/grpc_server_impl.h index a430d976de0..0c7adc7f190 100644 --- a/userspace/falco/grpc_server_impl.h +++ b/userspace/falco/grpc_server_impl.h @@ -19,7 +19,7 @@ limitations under the License. #pragma once #include -#include "falco_output.grpc.pb.h" +#include "output.grpc.pb.h" #include "grpc_context.h" namespace falco diff --git a/userspace/falco/falco_output.proto b/userspace/falco/output.proto similarity index 90% rename from userspace/falco/falco_output.proto rename to userspace/falco/output.proto index 21a41df780f..604c914dd44 100644 --- a/userspace/falco/falco_output.proto +++ b/userspace/falco/output.proto @@ -5,6 +5,8 @@ import "schema.proto"; package falco.output; +option go_package = "github.com/falcosecurity/client-go/pkg/api/output"; + service service { rpc subscribe(request) returns (stream response); } @@ -23,4 +25,4 @@ message response { string output = 5; map output_fields = 6; // repeated string tags = 7; // TODO(leodido,fntlnz): tags not supported yet, keeping for reference -} \ No newline at end of file +} diff --git a/userspace/falco/schema.proto b/userspace/falco/schema.proto index 262d096cf71..45967348e68 100644 --- a/userspace/falco/schema.proto +++ b/userspace/falco/schema.proto @@ -2,6 +2,8 @@ syntax = "proto3"; package falco.schema; +option go_package = "github.com/falcosecurity/client-go/pkg/api/schema"; + enum priority { option allow_alias = true; EMERGENCY = 0; @@ -39,4 +41,4 @@ enum source { k8s_audit = 1; K8s_audit = 1; K8S_audit = 1; -} \ No newline at end of file +} From f4f355650ef8295f04e88419f3f980756c41be5f Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Wed, 25 Sep 2019 09:19:08 +0000 Subject: [PATCH 74/76] docs(userspace/falco): document output proto messages and service Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/falco/output.proto | 8 ++++++++ userspace/falco/schema.proto | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/userspace/falco/output.proto b/userspace/falco/output.proto index 604c914dd44..6de1adf855f 100644 --- a/userspace/falco/output.proto +++ b/userspace/falco/output.proto @@ -7,16 +7,24 @@ package falco.output; option go_package = "github.com/falcosecurity/client-go/pkg/api/output"; +// The `subscribe` service defines the RPC call +// to perform an output `request` which will lead to obtain an output `response`. service service { rpc subscribe(request) returns (stream response); } +// The `request` message is the logical representation of the request model. +// It is the input of the `subscribe` service. +// It is used to configure the kind of subscription to the gRPC streaming server. message request { bool keepalive = 1; // string duration = 2; // TODO(leodido, fntlnz): not handled yet but keeping for reference. // repeated string tags = 3; // TODO(leodido, fntlnz): not handled yet but keeping for reference. } +// The `response` message is the logical representation of the output model. +// It contains all the elements that Falco emits in an output along with the +// definitions for priorities and source. message response { google.protobuf.Timestamp time = 1; falco.schema.priority priority = 2; diff --git a/userspace/falco/schema.proto b/userspace/falco/schema.proto index 45967348e68..bcd90f514c7 100644 --- a/userspace/falco/schema.proto +++ b/userspace/falco/schema.proto @@ -31,7 +31,7 @@ enum priority { debug = 7; Debug = 7; } - + enum source { option allow_alias = true; SYSCALL = 0; From f0e7a2fc4355c89806d23be6d39d41b2b60e6277 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Wed, 25 Sep 2019 11:35:14 +0000 Subject: [PATCH 75/76] chore(userspace): addressing review comments and typos Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- userspace/engine/formats.cpp | 1 - userspace/falco/configuration.cpp | 1 - userspace/falco/grpc_context.h | 5 ++--- userspace/falco/grpc_server.cpp | 31 ++++++++++++---------------- userspace/falco/grpc_server_impl.cpp | 5 +---- userspace/falco/lua/output.lua | 8 ++++--- 6 files changed, 21 insertions(+), 30 deletions(-) diff --git a/userspace/engine/formats.cpp b/userspace/engine/formats.cpp index bea9e5f02bc..5ae1a4713d9 100644 --- a/userspace/engine/formats.cpp +++ b/userspace/engine/formats.cpp @@ -290,7 +290,6 @@ int falco_formats::resolve_tokens(lua_State *ls) json_event_formatter json_formatter(s_engine->json_factory(), sformat); values = json_formatter.tomap((json_event*) evt); } - // todo(leodido, fntlnz) > check explicitly for k8s_audit, otherwise throw exception lua_newtable(ls); for(auto const& v : values) diff --git a/userspace/falco/configuration.cpp b/userspace/falco/configuration.cpp index bfe6512ee7c..ae87169044d 100644 --- a/userspace/falco/configuration.cpp +++ b/userspace/falco/configuration.cpp @@ -151,7 +151,6 @@ void falco_configuration::init(string conf_filename, list &cmdline_optio m_grpc_enabled = m_config->get_scalar("grpc", "enabled", false); m_grpc_bind_address = m_config->get_scalar("grpc", "bind_address", "0.0.0.0:5060"); m_grpc_threadiness = m_config->get_scalar("grpc", "threadiness", 8); // todo > limit it to avoid overshubscription? std::thread::hardware_concurrency() - // todo(fntlnz,leodido) > chose correct paths m_grpc_private_key = m_config->get_scalar("grpc", "private_key", "/etc/falco/certs/server.key"); m_grpc_cert_chain = m_config->get_scalar("grpc", "cert_chain", "/etc/falco/certs/server.crt"); m_grpc_root_certs = m_config->get_scalar("grpc", "root_certs", "/etc/falco/certs/ca.crt"); diff --git a/userspace/falco/grpc_context.h b/userspace/falco/grpc_context.h index bb8eec62dc0..b086792dd38 100644 --- a/userspace/falco/grpc_context.h +++ b/userspace/falco/grpc_context.h @@ -60,9 +60,8 @@ class stream_context : public context SUCCESS, ERROR } m_status = STREAMING; - // Request-specific stream data - mutable void* m_stream = nullptr; - // Are there more responses to stream? + + mutable void* m_stream = nullptr; // todo(fntlnz, leodido) > useful in the future mutable bool m_has_more = false; }; } // namespace grpc diff --git a/userspace/falco/grpc_server.cpp b/userspace/falco/grpc_server.cpp index a0a448ce203..2ec98c5d52d 100644 --- a/userspace/falco/grpc_server.cpp +++ b/userspace/falco/grpc_server.cpp @@ -27,6 +27,15 @@ limitations under the License. #include "grpc_context.h" #include "utils.h" +#define REGISTER_STREAM(req, res, svc, rpc, impl, num) \ + std::vector> rpc##_contexts(num); \ + for(request_stream_context & ctx : rpc##_contexts) \ + { \ + ctx.m_process_func = &server::impl; \ + ctx.m_request_func = &svc::AsyncService::Request##rpc; \ + ctx.start(this); \ + } + template<> void falco::grpc::request_stream_context::start(server* srv) { @@ -52,7 +61,7 @@ void falco::grpc::request_stream_context*m_process_func)(*m_stream_ctx, m_req, res); + (srv->*m_process_func)(*m_stream_ctx, m_req, res); // subscribe() // When there still are more responses to stream if(m_stream_ctx->m_has_more) @@ -86,14 +95,12 @@ void falco::grpc::request_stream_contextNext(&tag, &event_read_success)) { if(tag == nullptr) { - // TODO: empty tag returned, log "completion queue with empty tag" continue; } @@ -105,7 +112,6 @@ void falco::grpc::server::thread_process(int thread_index) { if(ctx->m_state != request_context_base::REQUEST) { - // todo > log "server completion queue failed to read event for tag `tag`" // End the context with error ctx->end(this, true); } @@ -127,24 +133,11 @@ void falco::grpc::server::thread_process(int thread_index) break; default: // todo > log "unkown completion queue event" - // todo > abort? break; } } } -// -// Create array of contexts and start processing streaming RPC request. -// -#define REGISTER_STREAM(REQ, RESP, RPC, IMPL, CONTEXT_NUM) \ - std::vector> RPC##_contexts(CONTEXT_NUM); \ - for(request_stream_context & ctx : RPC##_contexts) \ - { \ - ctx.m_process_func = &server::IMPL; \ - ctx.m_request_func = &output::service::AsyncService::Request##RPC; \ - ctx.start(this); \ - } - void falco::grpc::server::init(std::string server_addr, int threadiness, std::string private_key, std::string cert_chain, std::string root_certs) { m_server_addr = server_addr; @@ -182,7 +175,9 @@ void falco::grpc::server::run() // This defines the number of simultaneous completion queue requests of the same type (service::AsyncService::Request##RPC) // For this approach to be sufficient server::IMPL have to be fast int context_num = m_threadiness * 10; - REGISTER_STREAM(output::request, output::response, subscribe, subscribe, context_num) + REGISTER_STREAM(output::request, output::response, output::service, subscribe, subscribe, context_num) + + // register_stream(subscribe, context_num) m_threads.resize(m_threadiness); int thread_idx = 0; diff --git a/userspace/falco/grpc_server_impl.cpp b/userspace/falco/grpc_server_impl.cpp index f423b5af629..d4712290c9d 100644 --- a/userspace/falco/grpc_server_impl.cpp +++ b/userspace/falco/grpc_server_impl.cpp @@ -32,14 +32,11 @@ void falco::grpc::server_impl::subscribe(const stream_context& ctx, const output { if(ctx.m_status == stream_context::SUCCESS || ctx.m_status == stream_context::ERROR) { - // todo > logic - ctx.m_stream = nullptr; } else { - // Start (or continue) streaming - // ctx.m_status == stream_context::STREAMING + // Streaming if(output::queue::get().try_pop(res) && !req.keepalive()) { ctx.m_has_more = true; diff --git a/userspace/falco/lua/output.lua b/userspace/falco/lua/output.lua index 6a635d350ef..8c8454804b5 100644 --- a/userspace/falco/lua/output.lua +++ b/userspace/falco/lua/output.lua @@ -170,12 +170,14 @@ function mod.http_reopen() end function mod.grpc(event, rule, source, priority, priority_num, msg, format, options) - fields = formats.resolve_tokens(event, source, format) - c_outputs.handle_grpc(event, rule, source, priority, msg, fields, options) + if options.enabled == true then + fields = formats.resolve_tokens(event, source, format) + c_outputs.handle_grpc(event, rule, source, priority, msg, fields, options) + end end function mod.grpc_message(priority, priority_num, msg, options) - -- todo + -- todo(fntlnz, leodido) > gRPC does not support subscribing to dropped events yet end From e2bb1e4181d10f6e99ac72bcf8c8c2c1726dd86b Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Wed, 25 Sep 2019 11:35:48 +0000 Subject: [PATCH 76/76] docs: document gRPC server and gRPC output service config options Co-authored-by: Lorenzo Fontana Signed-off-by: Leonardo Di Donato --- falco.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/falco.yaml b/falco.yaml index 5ef62edc086..2ebacd127a0 100644 --- a/falco.yaml +++ b/falco.yaml @@ -168,6 +168,11 @@ http_output: enabled: false url: http://some.url +# gRPC server configuration. +# The gRPC server is secure by default (mutual TLS) so you need to generate certificates and update their paths here. +# By default the gRPC server is off. +# You can configure the address to bind and expose it. +# By modifying the threadiness configuration you can fine-tune the number of threads (and context) it will use. grpc: enabled: false bind_address: "0.0.0.0:5060" @@ -176,5 +181,8 @@ grpc: cert_chain: "/etc/falco/certs/server.crt" root_certs: "/etc/falco/certs/ca.crt" +# gRPC output service. +# By default it is off. +# By enabling this all the output events will be kept in memory until you read them with a gRPC client. grpc_output: - enabled: true \ No newline at end of file + enabled: false \ No newline at end of file