Skip to content

Commit 3b79b79

Browse files
committed
Merge vmdb-logger code from manageiq-gems-pending
Process for this described below: --- From within the manageiq-gems-pending repo --- 1. All versions of filenames to be were determined: $ git log --name-only --format=format: --follow -- spec/util/vmdb-logger_spec.rb | sort -u lib/gems/pending/spec/util/vmdb-logger_spec.rb spec/util/vmdb-logger_spec.rb $ git log --name-only --format=format: --follow -- lib/gems/pending/util/vmdb-logger.rb | sort -u lib/gems/pending/util/vmdb-logger.rb 2. Script created to build history for extracted files: mkdir -p vmdb-logger/spec/util/ mkdir -p vmdb-logger/lib/gems/pending/util/ mkdir -p vmdb-logger/lib/gems/pending/spec/util/ mv lib/gems/pending/util/vmdb-logger.rb vmdb-logger/lib/gems/pending/util/ 2>/dev/null mv lib/gems/pending/spec/util/vmdb-logger_spec.rb vmdb-logger/lib/gems/pending/spec/util/ 2>/dev/null mv spec/util/vmdb-logger_spec.rb vmdb-logger/spec/util/ 2>/dev/null true 3. Subtree for manageiq-gems-pending created: $ git checkout -b vmdb-logger-extraction $ git filter-branch -f --prune-empty --tree-filter /path/to/prune_vmdb_logger.sh HEAD Rewrite 5cc19a6b852a94ef8c6f6254bce6f426a0797ddf (1527/1531) (253 seconds passed, remaining 0 predicted) Ref 'refs/heads/vmdb-logger-extraction' was rewritten $ git filter-branch --prune-empty -f --subdirectory-filter vmdb-logger Rewrite 7476c176b854b236b73cc549ad6764a31b8e8c28 (15/15) (0 seconds passed, remaining 0 predicted) Ref 'refs/heads/vmdb-logger-extraction' was rewritten 4. Update the file locations out of gems-pending $ mkdir spec/lib/ $ mv lib/gems/pending/util/vmdb-logger.rb lib/ $ mv spec/util/vmdb-logger_spec.rb spec/lib/ $ rm -rf lib/gems spec/util $ git add -A $ git commit -m "Move vmdb-logger out gems/pending subdir" --- From the parent ("repos") directory --- 5. Create a new "local remote" for the new in manageiq-gems-pending code: $ cd .. $ git init --bare vmdb-logger.git --- From within the manageiq-gems-pending repo --- 6. Add this local remote as a remote, and push to it $ cd manageiq-gems-pending $ git remote add vmdb-logger /path/to/repos/vmdb-logger.git $ git push vmdb-logger vmdb-logger-extraction Counting objects: 122, done. Delta compression using up to 8 threads. Compressing objects: 100% (32/32), done. Writing objects: 100% (122/122), 13.32 KiB | 0 bytes/s, done. Total 122 (delta 17), reused 19 (delta 17) To /path/to/repos/vmdb-logger.git * [new branch] vmdb-logger-extraction -> vmdb-logger-extraction --- From the manageiq repo --- 7. Checkout a new branch to perform the code merge $ cd ../manageiq $ git checkout -b vmdb-logger-reinsert Switched to a new branch 'vmdb-logger-reinsert' 8. Add the local remote to the manageiq remotes $ git remote add vmdb-logger /Users/nicklamuro/code/redhat/vmdb-logger.git 9. Fetch and merge the contents from the vmdb-logger remote $ git fetch vmdb-logger $ git merge vmdb-logger/vmdb-logger-extraction
2 parents e73c175 + c764f65 commit 3b79b79

File tree

2 files changed

+330
-0
lines changed

2 files changed

+330
-0
lines changed

lib/vmdb-logger.rb

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
require 'logger'
2+
3+
class VMDBLogger < Logger
4+
def initialize(*args)
5+
super
6+
self.level = INFO
7+
8+
# HACK: ActiveSupport monkey patches the standard Ruby Logger#initialize
9+
# method to set @formatter to a SimpleFormatter.
10+
#
11+
# The ActiveSupport Logger patches are deprecated in Rails 3.1.1 in favor of
12+
# ActiveSupport::BufferedLogger, so this hack may not be needed in future
13+
# version of Rails.
14+
self.formatter = Formatter.new
15+
16+
# Allow for thread safe Logger level changes, similar to functionalities
17+
# provided by ActiveSupport::LoggerThreadSafeLevel
18+
@write_lock = Mutex.new
19+
@local_levels = {}
20+
end
21+
22+
# Silences the logger for the duration of the block.
23+
#
24+
# Taken from activesupport/logger_silence
25+
def silence(temporary_level = Logger::ERROR)
26+
old_local_level = local_level
27+
self.local_level = temporary_level
28+
29+
yield self
30+
ensure
31+
self.local_level = old_local_level
32+
end
33+
34+
attr_reader :logdev # Expose logdev
35+
36+
def logdev=(logdev)
37+
if @logdev
38+
shift_age = @logdev.instance_variable_get(:@shift_age)
39+
shift_size = @logdev.instance_variable_get(:@shift_size)
40+
@logdev.close
41+
else
42+
shift_age = 0
43+
shift_size = 1048576
44+
end
45+
46+
@logdev = LogDevice.new(logdev, :shift_age => shift_age, :shift_size => shift_size)
47+
end
48+
49+
def filename
50+
logdev.filename unless logdev.nil?
51+
end
52+
53+
alias_method :filename=, :logdev=
54+
55+
def self.contents(log, width = nil, last = 1000)
56+
return "" unless File.file?(log)
57+
58+
if last.nil?
59+
contents = File.open(log, "rb", &:read).split("\n")
60+
else
61+
require 'util/miq-system'
62+
contents = MiqSystem.tail(log, last)
63+
end
64+
return "" if contents.nil? || contents.empty?
65+
66+
results = []
67+
68+
# Wrap lines at width if passed
69+
contents.each do |line|
70+
while !width.nil? && line.length > width
71+
# Don't return lines containing invalid UTF8 byte sequences - see vmdb_logger_test.rb
72+
results.push(line[0...width]) if (line[0...width].unpack("U*") rescue nil)
73+
line = line[width..line.length]
74+
end
75+
# Don't return lines containing invalid UTF8 byte sequences - see vmdb_logger_test.rb
76+
results.push(line) if line.length && (line.unpack("U*") rescue nil)
77+
end
78+
79+
# Put back the utf-8 encoding which is the default for most rails libraries
80+
# after opening it as binary and getting rid of the invalid UTF8 byte sequences
81+
results.join("\n").force_encoding("utf-8")
82+
end
83+
84+
def contents(width = nil, last = 1000)
85+
self.class.contents(filename, width, last)
86+
end
87+
88+
def log_backtrace(err, level = :error)
89+
# Get the name of the method that called us unless it is a wrapped log_backtrace
90+
method_name = nil
91+
caller.each do |c|
92+
method_name = c[/`([^']*)'/, 1]
93+
break unless method_name == 'log_backtrace'
94+
end
95+
96+
# Log the error text
97+
send(level, "[#{err.class.name}]: #{err.message} Method:[#{method_name}]")
98+
99+
# Log the stack trace except for some specific exceptions
100+
unless (Object.const_defined?(:MiqException) && err.kind_of?(MiqException::Error)) ||
101+
(Object.const_defined?(:MiqAeException) && err.kind_of?(MiqAeException::Error))
102+
send(level, err.backtrace.nil? || err.backtrace.empty? ? "Backtrace is not available" : err.backtrace.join("\n"))
103+
end
104+
end
105+
106+
def self.log_hashes(logger, h, options = {})
107+
level = options[:log_level] || :info
108+
filter = Array(options[:filter]).flatten.compact.map(&:to_s) << "password"
109+
filter.uniq!
110+
111+
values = YAML.dump(h).gsub(MiqPassword::REGEXP, "[FILTERED]")
112+
values.split("\n").each do |l|
113+
next if l[0...3] == '---'
114+
if (key = filter.detect { |f| l.include?(f) })
115+
l.gsub!(/#{key}.*: (.+)/) { |m| m.gsub!($1, "[FILTERED]") }
116+
end
117+
logger.send(level, " #{l}")
118+
end
119+
end
120+
121+
def log_hashes(h, options = {})
122+
self.class.log_hashes(self, h, options)
123+
end
124+
125+
def level
126+
local_level || super
127+
end
128+
129+
private
130+
131+
def local_log_id
132+
Thread.current.object_id
133+
end
134+
135+
def local_level
136+
@local_levels[local_log_id]
137+
end
138+
139+
def local_level=(level)
140+
@write_lock.synchronize do
141+
if level
142+
@local_levels[local_log_id] = level
143+
else
144+
@local_levels.delete(local_log_id)
145+
end
146+
end
147+
end
148+
149+
class Formatter < Logger::Formatter
150+
FORMAT = "[----] %s, [%s#%d:%x] %5s -- %s: %s\n"
151+
152+
def call(severity, time, progname, msg)
153+
msg = msg2str(msg)
154+
155+
# Add task id to the message if a task is currently being worked on.
156+
if $_miq_worker_current_msg && !$_miq_worker_current_msg.task_id.nil?
157+
prefix = "Q-task_id([#{$_miq_worker_current_msg.task_id}])"
158+
msg = "#{prefix} #{msg}" unless msg.include?(prefix)
159+
end
160+
161+
FORMAT % [severity[0..0], format_datetime(time), $$, Thread.current.object_id, severity, progname, msg]
162+
end
163+
end
164+
end

spec/lib/vmdb-logger_spec.rb

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
require 'util/vmdb-logger'
2+
require 'util/miq-password'
3+
4+
describe VMDBLogger do
5+
describe "#log_hashes" do
6+
let(:buffer) { StringIO.new }
7+
let(:logger) { described_class.new(buffer) }
8+
9+
it "filters out passwords when keys are symbols" do
10+
hash = {:a => {:b => 1, :password => "pa$$w0rd"}}
11+
logger.log_hashes(hash)
12+
13+
buffer.rewind
14+
expect(buffer.read).to include(":password: [FILTERED]")
15+
end
16+
17+
it "filters out passwords when keys are strings" do
18+
hash = {"a" => {"b" => 1, "password" => "pa$$w0rd"}}
19+
logger.log_hashes(hash)
20+
21+
buffer.rewind
22+
expect(buffer.read).to include("password: [FILTERED]")
23+
end
24+
25+
it "with :filter option, filters out given keys and passwords" do
26+
hash = {:a => {:b => 1, :extra_key => "pa$$w0rd", :password => "pa$$w0rd"}}
27+
logger.log_hashes(hash, :filter => :extra_key)
28+
29+
buffer.rewind
30+
message = buffer.read
31+
expect(message).to include(':extra_key: [FILTERED]')
32+
expect(message).to include(':password: [FILTERED]')
33+
end
34+
35+
it "when :filter option is a Set object, filters out the given Set elements" do
36+
hash = {:a => {:b => 1, :bind_pwd => "pa$$w0rd", :amazon_secret => "pa$$w0rd", :password => "pa$$w0rd"}}
37+
logger.log_hashes(hash, :filter => %i(bind_pwd password amazon_secret).to_set)
38+
39+
buffer.rewind
40+
message = buffer.read
41+
expect(message).to include(':bind_pwd: [FILTERED]')
42+
expect(message).to include(':amazon_secret: [FILTERED]')
43+
expect(message).to include(':password: [FILTERED]')
44+
end
45+
46+
it "filters out encrypted value" do
47+
hash = {:a => {:b => 1, :extra_key => "v2:{c5qTeiuz6JgbBOiDqp3eiQ==}"}}
48+
logger.log_hashes(hash)
49+
50+
buffer.rewind
51+
expect(buffer.read).to include(':extra_key: [FILTERED]')
52+
end
53+
54+
it "filters out root_password" do
55+
hash = {"a" => {"b" => 1, "root_password" => "pa$$w0rd"}}
56+
logger.log_hashes(hash)
57+
58+
buffer.rewind
59+
expect(buffer.read).to include("root_password: [FILTERED]")
60+
end
61+
62+
it "filters out password_for_important_thing" do
63+
hash = {:a => {:b => 1, :password_for_important_thing => "pa$$w0rd"}}
64+
logger.log_hashes(hash)
65+
66+
buffer.rewind
67+
expect(buffer.read).to include(":password_for_important_thing: [FILTERED]")
68+
end
69+
end
70+
71+
it ".contents with no log returns empty string" do
72+
allow(File).to receive_messages(:file? => false)
73+
expect(VMDBLogger.contents("mylog.log")).to eq("")
74+
end
75+
76+
it ".contents with empty log returns empty string" do
77+
require 'util/miq-system'
78+
allow(MiqSystem).to receive_messages(:tail => "")
79+
80+
allow(File).to receive_messages(:file? => true)
81+
expect(VMDBLogger.contents("mylog.log")).to eq("")
82+
end
83+
84+
context "with evm log snippet with invalid utf8 byte sequence data" do
85+
before(:each) do
86+
@log = File.expand_path(File.join(File.dirname(__FILE__), "data/redundant_utf8_byte_sequence.log"))
87+
end
88+
89+
context "accessing the invalid data directly" do
90+
before(:each) do
91+
@data = File.read(@log)
92+
end
93+
94+
it "should have content with the invalid utf8 lines" do
95+
expect(@data).not_to be_nil
96+
expect(@data.kind_of?(String)).to be_truthy
97+
end
98+
99+
it "should unpack raw data as UTF-8 characters and raise ArgumentError" do
100+
expect { @data.unpack("U*") }.to raise_error(ArgumentError)
101+
end
102+
end
103+
104+
context "using VMDBLogger with no width" do
105+
before(:each) do
106+
logger = VMDBLogger.new(@log)
107+
@contents = logger.contents(nil, 1000)
108+
end
109+
110+
it "should have content but without the invalid utf8 lines" do
111+
expect(@contents).not_to be_nil
112+
expect(@contents.kind_of?(String)).to be_truthy
113+
end
114+
115+
it "should unpack logger.consents as UTF-8 characters and raise nothing" do
116+
expect { @contents.unpack("U*") }.not_to raise_error
117+
end
118+
end
119+
120+
context "using VMDBLogger with a provided width" do
121+
before(:each) do
122+
logger = VMDBLogger.new(@log)
123+
@contents = logger.contents(120, 5000)
124+
end
125+
126+
it "should have content but without the invalid utf8 lines" do
127+
expect(@contents).not_to be_nil
128+
expect(@contents.kind_of?(String)).to be_truthy
129+
end
130+
131+
it "should unpack logger.consents as UTF-8 characters and raise nothing" do
132+
expect { @contents.unpack("U*") }.not_to raise_error
133+
end
134+
end
135+
136+
context "using VMDBLogger no limit on lines read" do
137+
before(:each) do
138+
logger = VMDBLogger.new(@log)
139+
@contents = logger.contents(120, nil)
140+
end
141+
142+
it "should have content but without the invalid utf8 lines" do
143+
expect(@contents).not_to be_nil
144+
expect(@contents.kind_of?(String)).to be_truthy
145+
end
146+
147+
it "should unpack logger.consents as UTF-8 characters and raise nothing" do
148+
expect { @contents.unpack("U*") }.not_to raise_error
149+
end
150+
end
151+
152+
context "encoding" do
153+
it "with ascii file" do
154+
log = File.expand_path(File.join(File.dirname(__FILE__), "data/miq_ascii.log"))
155+
expect(VMDBLogger.new(log).contents.encoding.name).to eq("UTF-8")
156+
expect(VMDBLogger.new(log).contents(100, nil).encoding.name).to eq("UTF-8")
157+
end
158+
159+
it "with utf-8 file" do
160+
log = File.expand_path(File.join(File.dirname(__FILE__), "data/miq_utf8.log"))
161+
expect(VMDBLogger.new(log).contents.encoding.name).to eq("UTF-8")
162+
expect(VMDBLogger.new(log).contents(100, nil).encoding.name).to eq("UTF-8")
163+
end
164+
end
165+
end
166+
end

0 commit comments

Comments
 (0)