bpf: Improve publisher reliability

osquery/events/CMakeLists.txt

@@ -29,11 +29,13 @@ function(generateOsqueryEvents)
 
     if(OSQUERY_BUILD_BPF)
       list(APPEND source_files
+        linux/bpf/bpferrorstate.cpp
         linux/bpf/bpfeventpublisher.cpp
         linux/bpf/filesystem.cpp
         linux/bpf/processcontextfactory.cpp
         linux/bpf/setrlimit.cpp
         linux/bpf/systemstatetracker.cpp
+        linux/bpf/serializers.cpp
       )
     endif()
 
@@ -121,6 +123,7 @@ function(generateOsqueryEvents)
 
     if(OSQUERY_BUILD_BPF)
       list(APPEND platform_public_header_files
+        linux/bpf/bpferrorstate.h
         linux/bpf/bpfeventpublisher.h
         linux/bpf/filesystem.h
         linux/bpf/ifilesystem.h
@@ -129,6 +132,7 @@ function(generateOsqueryEvents)
         linux/bpf/processcontextfactory.h
         linux/bpf/setrlimit.h
         linux/bpf/systemstatetracker.h
+        linux/bpf/serializers.h
         linux/bpf/uniquedir.h
       )
     endif()

osquery/events/linux/bpf/bpferrorstate.cpp

@@ -0,0 +1,81 @@
+/**
+ * Copyright (c) 2014-present, The osquery authors
+ *
+ * This source code is licensed as defined by the LICENSE file found in the
+ * root directory of this source tree.
+ *
+ * SPDX-License-Identifier: (Apache-2.0 OR GPL-2.0-only)
+ */
+
+#include <osquery/events/linux/bpf/bpferrorstate.h>
+#include <osquery/logger/logger.h>
+
+namespace osquery {
+
+namespace ebpfpub = tob::ebpfpub;
+
+void updateBpfErrorState(
+    BPFErrorState& bpf_error_state,
+    const ebpfpub::IPerfEventReader::ErrorCounters& perf_error_counters) {
+  bpf_error_state.perf_error_counters.invalid_event +=
+      perf_error_counters.invalid_event;
+
+  bpf_error_state.perf_error_counters.lost_events +=
+      perf_error_counters.lost_events;
+
+  bpf_error_state.perf_error_counters.invalid_probe_output +=
+      perf_error_counters.invalid_probe_output;
+
+  bpf_error_state.perf_error_counters.invalid_event_data +=
+      perf_error_counters.invalid_event_data;
+}
+
+void reportAndClearBpfErrorState(BPFErrorState& bpf_error_state) {
+  if (bpf_error_state.perf_error_counters.invalid_probe_output != 0U) {
+    VLOG(1) << "Invalid BPF probe output error count: "
+            << bpf_error_state.perf_error_counters.invalid_probe_output;
+  }
+
+  if (bpf_error_state.perf_error_counters.invalid_event != 0U) {
+    VLOG(1) << "Invalid BPF probe event id count: "
+            << bpf_error_state.perf_error_counters.invalid_event;
+  }
+
+  if (bpf_error_state.perf_error_counters.invalid_event_data != 0U) {
+    VLOG(1) << "Invalid BPF event data count: "
+            << bpf_error_state.perf_error_counters.invalid_event_data;
+  }
+
+  if (bpf_error_state.perf_error_counters.lost_events != 0U) {
+    VLOG(1) << "Lost BPF event count: "
+            << bpf_error_state.perf_error_counters.lost_events;
+  }
+
+  if (bpf_error_state.probe_error_counter != 0U) {
+    VLOG(1) << "Buffers/strings that could not be captured by the probe: "
+            << bpf_error_state.probe_error_counter;
+  }
+
+  if (!bpf_error_state.errored_tracer_list.empty()) {
+    std::string tracer_list;
+
+    for (auto tracer_it = bpf_error_state.errored_tracer_list.begin();
+         tracer_it != bpf_error_state.errored_tracer_list.end();
+         ++tracer_it) {
+      if (tracer_it != bpf_error_state.errored_tracer_list.begin()) {
+        tracer_list += ", ";
+      }
+
+      auto tracer_id = *tracer_it;
+      tracer_list += std::to_string(tracer_id);
+    }
+
+    VLOG(1)
+        << "Failed to process one or more events from the following tracers: " +
+               tracer_list;
+  }
+
+  bpf_error_state = {};
+}
+
+} // namespace osquery

osquery/events/linux/bpf/bpferrorstate.h

@@ -0,0 +1,42 @@
+/**
+ * Copyright (c) 2014-present, The osquery authors
+ *
+ * This source code is licensed as defined by the LICENSE file found in the
+ * root directory of this source tree.
+ *
+ * SPDX-License-Identifier: (Apache-2.0 OR GPL-2.0-only)
+ */
+
+#pragma once
+
+#include <ebpfpub/iperfeventreader.h>
+
+#include <unordered_set>
+
+namespace osquery {
+
+/// A container for all the error counters the publisher works with
+struct BPFErrorState final {
+  /// perf_output related errors
+  tob::ebpfpub::IPerfEventReader::ErrorCounters perf_error_counters{};
+
+  /// Errors reported by the BPF probes we have loaded; this can be
+  /// a failure to capture a string (example: open syscall) or a buffer
+  /// (example: a sockaddr_in structure)
+  std::size_t probe_error_counter{};
+
+  /// A list of the tracers that have generated events we could not
+  /// process correctly. This is likely caused by either lost events
+  /// or probe errors (see above)
+  std::unordered_set<std::uint64_t> errored_tracer_list;
+};
+
+/// Updates the error state structure with the given perf error counters
+void updateBpfErrorState(
+    BPFErrorState& bpf_error_state,
+    const tob::ebpfpub::IPerfEventReader::ErrorCounters& perf_error_counters);
+
+/// Logs the error state structure and then clears it
+void reportAndClearBpfErrorState(BPFErrorState& bpf_error_state);
+
+} // namespace osquery