AuditNetlink: Added common rules, fixed audit id parsing (see details).

Moved the audit_reply pre-processing inside the getEvents() callback. This
means that the AuditNetlink class (the one that receives messages from the
kernel) is now only reading from the netlink, leaving all processing to
the consumer (the AuditEventPublisher, and our future class named
AuditFileEventsPublisher). Those event publisher are executed in the
context of an additional thread.

The --audit_debug=true switch has been improved, and will now show more
meaningful output.

I've also converted the audit id field from unsigned int to string (see the
issue list section), and we are now using audit_id + timestamp as primary
key.

Issues found in the original implementation:

1. The rules being added to the audit service were still being
   removed even if they already existed before launching osquery.
2. The audit id alone is being used as primary key and the
   implementation is handling subsequent records with the same
   id value as duplicated. This is wrong (see the attached links).
   This means I will probably have to add an additional field to keep
   supporting the "broken" implementation until we can fix it*.

* We are not going to use the AuditAssembler for our new table anymore because
it is too limited (so it's not an issue for us), but the existing process_events
and socket_events tables are probably tossing away valid events.

Links:
1. https://people.redhat.com/sgrubb/audit/audit-rt-events.txt
2. https://www.redhat.com/archives/linux-audit/2008-August/msg00115.html
3. https://github.com/torvalds/linux/blob/master/kernel/audit.c (see audit_serial)
4. https://www.redhat.com/archives/linux-audit/2006-April/msg00028.html

osquery/events/linux/audit.cpp

@@ -38,7 +38,7 @@ void AuditAssembler::start(size_t capacity,
   types_ = std::move(types);
 }
 
-boost::optional<AuditFields> AuditAssembler::add(AuditId id,
+boost::optional<AuditFields> AuditAssembler::add(const std::string &id,
                                                  size_t type,
                                                  const AuditFields& fields) {
   auto it = m_.find(id);
@@ -94,18 +94,18 @@ boost::optional<AuditFields> AuditAssembler::add(AuditId id,
   return boost::none;
 }
 
-void AuditAssembler::evict(AuditId id) {
+void AuditAssembler::evict(const std::string &id) {
   queue_.erase(std::remove(queue_.begin(), queue_.end(), id), queue_.end());
   mt_.erase(id);
   m_.erase(id);
 }
 
-void AuditAssembler::shuffle(AuditId id) {
+void AuditAssembler::shuffle(const std::string & id) {
   queue_.erase(std::remove(queue_.begin(), queue_.end(), id), queue_.end());
   queue_.push_back(id);
 }
 
-bool AuditAssembler::complete(AuditId id) {
+bool AuditAssembler::complete(const std::string & id) {
   // Is this type enough.
   const auto& types = mt_.at(id);
   for (const auto& t : types_) {

osquery/events/linux/audit.h

@@ -32,39 +32,6 @@ namespace osquery {
 #define AUDIT_TYPE_SYSCALL 1300
 #define AUDIT_TYPE_SOCKADDR 1306
 
-typedef std::pair<audit_reply, std::size_t> AuditReplyDescriptor;
-
-/**
- * @brief This is the context data used by the thread that receives the audit
- * events.
- */
-
-struct EventReaderTaskContext final {
-  /// The ::tearDown method of the publisher sets this to true when terminating.
-  std::atomic<bool> terminate{false};
-
-  /// The file descriptor to the auditd netlink.
-  int audit_handle{0};
-
-  /// The primary event queue. We have two because we swap them to avoid a
-  /// potentially slow copy.
-  std::vector<AuditReplyDescriptor> primary_audit_queue;
-
-  /// The secondary event queue; see primary_audit_queue.
-  std::vector<AuditReplyDescriptor> secondary_audit_queue;
-
-  /// This is the mutex that controls access to this structure.
-  std::mutex context_mutex;
-
-  /// The active audit queue is where new audit event are stored. It will either
-  /// point to the primary or secondary event queue.
-  std::vector<AuditReplyDescriptor>* active_audit_queue{nullptr};
-
-  /// The passive audit queue is where audit events are taken for processing. It
-  /// will either point to the primary or secondary event queue.
-  std::vector<AuditReplyDescriptor>* passive_audit_queue{nullptr};
-};
-
 /**
  * @brief A simple audit rule description that can be populated via a config.
  *
@@ -102,9 +69,6 @@ struct AuditRuleInternal {
   int action{0};
 };
 
-/// The audit ID is a smaller integer.
-using AuditId = size_t;
-
 /// Alias the field container so we can replace and improve with refactors.
 using AuditFields = std::map<std::string, std::string>;
 
@@ -145,30 +109,30 @@ class AuditAssembler : private boost::noncopyable {
   void start(size_t capacity, std::vector<size_t> types, AuditUpdate update);
 
   /// Add a message from audit.
-  boost::optional<AuditFields> add(AuditId id,
+  boost::optional<AuditFields> add(const std::string &id,
                                    size_t type,
                                    const AuditFields& fields);
 
   /// Allow the publisher to explicit-set fields.
-  void set(AuditId id, const std::string& key, const std::string& value) {
+  void set(const std::string &id, const std::string& key, const std::string& value) {
     m_[id][key] = value;
   }
 
   /// Remove an audit ID from the queue and clear associated messages/types.
-  void evict(AuditId id);
+  void evict(const std::string &id);
 
   /// Shuffle an audit ID to the front of the queue.
-  void shuffle(AuditId id);
+  void shuffle(const std::string & id);
 
   /// Check if the audit ID has completed each required message types.
-  bool complete(AuditId id);
+  bool complete(const std::string & id);
 
  private:
   /// A map of audit ID to aggregate message fields.
-  std::unordered_map<AuditId, AuditFields> m_;
+  std::unordered_map<std::string, AuditFields> m_;
 
   /// A map of audit ID to current set of types seen.
-  std::unordered_map<AuditId, std::vector<size_t>> mt_;
+  std::unordered_map<std::string, std::vector<size_t>> mt_;
 
   /// A functional callable to sanitize individual messages.
   AuditUpdate update_{nullptr};
@@ -177,7 +141,7 @@ class AuditAssembler : private boost::noncopyable {
   size_t capacity_{0};
 
   /// The in-order (by time) queue of audit IDs.
-  std::vector<AuditId> queue_;
+  std::vector<std::string> queue_;
 
   /// The set of required types.
   std::vector<size_t> types_;
@@ -242,7 +206,7 @@ struct AuditEventContext : public EventContext {
   AuditFields fields;
 
   /// Each message will contain the audit ID.
-  AuditId audit_id{0};
+  std::string audit_id{0};
 
   /// Each message will contain the event time.
   size_t time{0};