Adding new plugin for parsing ovn logs LOG-1377

fluentd/Dockerfile

@@ -71,7 +71,6 @@ RUN bash -c '. /source.jemalloc; echo jemalloc $JEMALLOC_VER >> /contents'
 
 COPY  ${upstream_code}/vendored_gem_src/ ${HOME}/vendored_gem_src/
 COPY  ${upstream_code}/lib/fluent-plugin-remote_syslog/ ${HOME}/vendored_gem_src/fluent-plugin-remote_syslog/
-COPY  ${upstream_code}/lib/fluent-plugin-collected/ ${HOME}/vendored_gem_src/fluent-plugin-collected/
 COPY  ${upstream_code}/lib/remote_syslog_sender/ ${HOME}/vendored_gem_src/remote_syslog_sender/
 COPY  ${upstream_code}/lib/syslog_protocol/ ${HOME}/vendored_gem_src/syslog_protocol/
 COPY  ${upstream_code}/install-gems.sh ${HOME}/vendored_gem_src/
@@ -99,6 +98,7 @@ COPY  ${upstream_code}/wait_for_es_version.sh ${HOME}/
 COPY  ${upstream_code}/lib/filter_parse_json_field/lib/*.rb /etc/fluent/plugin/
 COPY  ${upstream_code}/lib/filter_elasticsearch_genid_ext/lib/filter_elasticsearch_genid_ext.rb /etc/fluent/plugin/
 COPY  ${upstream_code}/lib/parser_viaq_host_audit/lib/*.rb /etc/fluent/plugin/
+COPY  ${upstream_code}/lib/parser_viaq_ovn_audit/lib/*.rb /etc/fluent/plugin/
 COPY  ${upstream_code}/lib/fluent-plugin-viaq_data_model/lib/fluent/plugin/*.rb /etc/fluent/plugin/
 COPY  ${upstream_code}/utils/ /usr/local/bin/
 

fluentd/Dockerfile.in

@@ -87,7 +87,6 @@ RUN bash -c '. /source.jemalloc; echo jemalloc $JEMALLOC_VER >> /contents'
 
 COPY --from=builder ${upstream_code}/vendored_gem_src/ ${HOME}/vendored_gem_src/
 COPY --from=builder ${upstream_code}/lib/fluent-plugin-remote_syslog/ ${HOME}/vendored_gem_src/fluent-plugin-remote_syslog/
-COPY --from=builder ${upstream_code}/lib/fluent-plugin-collected/ ${HOME}/vendored_gem_src/fluent-plugin-collected/
 COPY --from=builder ${upstream_code}/lib/remote_syslog_sender/ ${HOME}/vendored_gem_src/remote_syslog_sender/
 COPY --from=builder ${upstream_code}/lib/syslog_protocol/ ${HOME}/vendored_gem_src/syslog_protocol/
 COPY --from=builder ${upstream_code}/install-gems.sh ${HOME}/vendored_gem_src/
@@ -115,6 +114,7 @@ COPY --from=builder ${upstream_code}/wait_for_es_version.sh ${HOME}/
 COPY --from=builder ${upstream_code}/lib/filter_parse_json_field/lib/*.rb /etc/fluent/plugin/
 COPY --from=builder ${upstream_code}/lib/filter_elasticsearch_genid_ext/lib/filter_elasticsearch_genid_ext.rb /etc/fluent/plugin/
 COPY --from=builder ${upstream_code}/lib/parser_viaq_host_audit/lib/*.rb /etc/fluent/plugin/
+COPY --from=builder ${upstream_code}/lib/parser_viaq_ovn_audit/lib/*.rb /etc/fluent/plugin/
 COPY --from=builder ${upstream_code}/lib/fluent-plugin-viaq_data_model/lib/fluent/plugin/*.rb /etc/fluent/plugin/
 COPY --from=builder ${upstream_code}/utils/ /usr/local/bin/
 

fluentd/lib/parser_viaq_ovn_audit/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+gem 'codeclimate-test-reporter', :group => :test, :require => nil
+
+gemspec
+
+gem "webrick", "~> 1.7"

fluentd/lib/parser_viaq_ovn_audit/Rakefile

@@ -0,0 +1,11 @@
+#require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new do |t|
+    t.test_files = FileList['test/**/*_test.rb']
+    t.warning = false
+    #t.verbose = true
+end
+desc "Run tests"
+
+task default: :test

fluentd/lib/parser_viaq_ovn_audit/lib/parser_viaq_ovn_audit.rb

@@ -0,0 +1,34 @@
+require 'fluent/parser'
+require 'fluent/time'
+
+require_relative 'viaq_ovn_audit'
+
+module Fluent
+  class ViaqOvnAuditParser < Parser
+    Plugin.register_parser("viaq_ovn_audit", self)
+
+    def configure(conf={})
+      super
+      @audit_parser = ViaqOvnAudit.new()
+    end
+
+    def parse(text)
+      begin
+        parsed_json = @audit_parser.parse_audit_line text
+
+        if parsed_json.nil?
+          t = Time.now
+          time = Fluent::EventTime.new(t.to_i, t.nsec)
+        else
+          t = DateTime.parse(parsed_json['@timestamp']).to_time
+          time = Fluent::EventTime.new(t.to_i, t.nsec)
+        end
+
+        yield time, parsed_json
+      rescue Fluent::ViaqOvnAudit::ViaqOvnAuditParserException => e
+        log.error e.message
+        yield nil, nil
+      end
+    end
+  end
+end

fluentd/lib/parser_viaq_ovn_audit/lib/viaq_ovn_audit.rb

@@ -0,0 +1,57 @@
+require 'fluent/plugin/input'
+require 'time'
+
+# Parses audit log to format that fits Origin Aggregated Logging
+module Fluent
+  class ViaqOvnAudit
+
+    class ViaqOvnAuditParserException < StandardError
+    end
+
+    # Keys as found in raw audit.log messsages
+    IN_TYPE = 'type'
+    IN_MSG = 'msg'
+
+    # Keys used in Origin Aggregated Logging schema
+    OUT_HOST_TYPE = 'type'
+    OUT_HOST_HOSTNAME = 'hostname'
+
+    TIME = '@timestamp'
+    LEVEL = 'level'
+    ENV_HOSTNAME = 'NODE_NAME'
+    AUDIT_ENVELOPE = 'structured'
+
+    def initialize()
+      @@hostname = ENV[ENV_HOSTNAME].nil? ? nil : String.new(ENV[ENV_HOSTNAME])
+    end
+
+    # Takes one line from audit.log and returns hash
+    # that fits the OAL format.
+    def parse_audit_line(line)
+      puts line.inspect
+      event = {}
+      return normalize(event, line.split('|'))
+    end
+
+    private
+
+    # Parses metadata and extract key values
+    def normalize(target, metadata)
+      event = {}
+      event[TIME]              = metadata[0]
+      event[LEVEL]             = metadata[3].downcase
+      event[OUT_HOST_HOSTNAME] = @@hostname unless @@hostname.nil?
+
+      event[AUDIT_ENVELOPE] = {}
+      key_value =  metadata[4].split(',')
+
+      key_value.each do |pair|
+          key = pair.split('=')[0].strip
+          value = pair.split('=')[1].strip
+          event[AUDIT_ENVELOPE][key] = value
+      end
+      return event
+    end
+
+  end
+end

fluentd/lib/parser_viaq_ovn_audit/parser_viaq_ovn_audit.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+# can override for testing
+FLUENTD_VERSION = ENV['FLUENTD_VERSION'] || "1.12.0"
+
+Gem::Specification.new do |gem|
+  gem.name          = "parser_viaq_ovn_audit"
+  gem.version       = "0.0.1"
+  gem.authors       = ["Ajay Gupta"]
+  gem.summary       = %q{Parser plugin to read ovn audit records}
+
+  gem.required_ruby_version = '>= 2.0.0'
+
+  gem.add_runtime_dependency "fluentd", "~> #{FLUENTD_VERSION}"
+
+  gem.add_development_dependency "bundler"
+  gem.add_development_dependency("fluentd", "~> #{FLUENTD_VERSION}")
+  gem.add_development_dependency("rake", ["~> 13.0"])
+  gem.add_development_dependency("rr", ["~> 3.0"])
+  gem.add_development_dependency("test-unit", ["~> 3.2"])
+  gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
+end

fluentd/lib/parser_viaq_ovn_audit/test/parser_viaq_ovn_audit_test.rb

@@ -0,0 +1,33 @@
+require 'fluent/test'
+require 'test/unit/rr'
+require 'fluent/test/driver/parser'
+require 'json'
+
+require File.join(File.dirname(__FILE__), '..', 'lib/parser_viaq_ovn_audit')
+
+class ParserViaqOvnAuditTest < Test::Unit::TestCase
+  include Fluent
+
+  setup do
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf = '')
+    Fluent::Test::Driver::Parser.new(ViaqOvnAuditParser).configure(conf)
+  end
+
+  sub_test_case 'plugin will parse ovn audit messages' do
+    test 'ovn audit logs test' do
+      d = create_driver()
+      message = "2021-07-06T08:26:58.687Z|00004|acl_log(ovn_pinctrl0)|INFO|name=\"verify-audit-logging_deny-all\", verdict=drop, severity=alert:icmp,vlan_tci=0x0000,dl_src=0a:58:0a:81:02:12,dl_dst=0a:58:0a:81:02:14,nw_src=10.129.2.18,nw_dst=10.129.2.20,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0"
+      d.instance.parse(message) do |time, record|
+        assert_equal('2021-07-06T08:26:58.687Z', record['@timestamp'])
+        assert_equal('info', record['level'])
+        assert_equal("\"verify-audit-logging_deny-all\"", record['structured']['name'])
+        assert_equal("alert:icmp", record['structured']['severity'])
+        assert_true(time.instance_of? Fluent::EventTime)
+      end
+    end
+
+  end
+end