Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fixes #14447 - add cache around parser using YAML file
A cache of parsed modules can be configured with `:parser_cache_path` to skip calls out to kafo_parsers when loading Puppet modules. This helps when no parser is available for the current Puppet installation (e.g. under Puppet 4 with AIO at the time of writing), and may provide a small peformance benefit. The cache can be generated with `kafo-export-params -f parsercache --no-parser-cache`. The cache is skipped if the mtime of the manifest is greater than the mtime when the cache was generated, causing Kafo to use kafo_parsers as normal.
- Loading branch information
Showing
14 changed files
with
269 additions
and
31 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,46 @@ | ||
module Kafo | ||
class ParserCacheReader | ||
def self.new_from_file(cache_path) | ||
if cache_path.nil? || cache_path.empty? | ||
logger.debug "No parser cache configured in :parser_cache_path, skipping setup" | ||
return nil | ||
end | ||
|
||
unless File.exist?(cache_path) | ||
logger.warn "Parser cache configured at #{cache_path} is missing, skipping setup" | ||
return nil | ||
end | ||
|
||
parsed = YAML.load(File.read(cache_path)) | ||
if !parsed.is_a?(Hash) || parsed[:version] != 1 || !parsed[:files].is_a?(Hash) | ||
logger.warn "Parser cache is from a different version of Kafo, skipping setup" | ||
return nil | ||
end | ||
|
||
new(parsed) | ||
end | ||
|
||
def self.logger | ||
KafoConfigure.logger | ||
end | ||
|
||
def initialize(cache) | ||
@cache = cache | ||
end | ||
|
||
def logger | ||
KafoConfigure.logger | ||
end | ||
|
||
def get(key, manifest_path) | ||
return nil unless @cache[:files].has_key?(key) | ||
|
||
if @cache[:files][key][:mtime] && File.mtime(manifest_path).to_i > @cache[:files][key][:mtime] | ||
logger.debug "Parser cache for #{manifest_path} is outdated, ignoring cache entry" | ||
return nil | ||
end | ||
|
||
@cache[:files][key][:data] | ||
end | ||
end | ||
end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
module Kafo | ||
class ParserCacheWriter | ||
def self.write(modules) | ||
{ | ||
:version => 1, | ||
:files => Hash[modules.sort.map { |m| write_module(m) }] | ||
} | ||
end | ||
|
||
def self.write_module(mod) | ||
[mod.identifier, {:data => mod.raw_data, :mtime => File.mtime(mod.manifest_path).to_i}] | ||
end | ||
end | ||
end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,58 @@ | ||
require 'test_helper' | ||
require 'kafo/parser_cache_writer' | ||
|
||
module Kafo | ||
describe ParserCacheReader do | ||
subject { ParserCacheReader } | ||
|
||
describe ".new_from_file" do | ||
let(:valid_cache) do | ||
{:version => 1, :files => {}} | ||
end | ||
|
||
specify { subject.new_from_file(nil).must_be_nil } | ||
specify { subject.new_from_file('').must_be_nil } | ||
specify { subject.new_from_file('/non/existent/file').must_be_nil } | ||
|
||
describe "with empty file" do | ||
let(:cache) { ParserCacheFactory.build('') } | ||
specify { subject.new_from_file(cache.path).must_be_nil } | ||
end | ||
|
||
describe "with version other than 1" do | ||
let(:cache) { ParserCacheFactory.build(valid_cache.update(:version => 2)) } | ||
specify { subject.new_from_file(cache.path).must_be_nil } | ||
end | ||
|
||
describe "with missing files section" do | ||
let(:cache) { ParserCacheFactory.build(valid_cache.reject { |k,v| k == :files }) } | ||
specify { subject.new_from_file(cache.path).must_be_nil } | ||
end | ||
|
||
describe "with correct format" do | ||
let(:cache) { ParserCacheFactory.build(valid_cache) } | ||
specify { subject.new_from_file(cache.path).must_be_instance_of subject } | ||
end | ||
end | ||
|
||
describe "#get" do | ||
specify { subject.new({:files => {}}).get('test', '/test/file.pp').must_be_nil } | ||
specify { subject.new({:files => {'test' => {:data => {:parameters => []}}}}).get('test', '/test/file.pp').must_equal(:parameters => []) } | ||
specify { File.stub(:mtime, 1) { subject.new({:files => {'test' => {:mtime => 1, :data => :test}}}).get('test', '/test/file.pp').must_equal(:test) } } | ||
specify { File.stub(:mtime, 2) { subject.new({:files => {'test' => {:mtime => 1, :data => :test}}}).get('test', '/test/file.pp').must_be_nil } } | ||
end | ||
|
||
describe "compatibility with writer" do | ||
before do | ||
KafoConfigure.config = Configuration.new(ConfigFileFactory.build('basic', BASIC_CONFIGURATION).path) | ||
end | ||
|
||
let(:parser) { TestParser.new(BASIC_MANIFEST) } | ||
let(:mod) { PuppetModule.new('puppet', parser) } | ||
let(:writer) { ParserCacheWriter.write([mod]) } | ||
let(:cache) { ParserCacheFactory.build(writer) } | ||
|
||
specify { File.stub(:mtime, 1) { subject.new_from_file(cache.path).get('puppet', parser.manifest_file).must_equal mod.raw_data } } | ||
end | ||
end | ||
end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,32 @@ | ||
require 'test_helper' | ||
require 'kafo/parser_cache_writer' | ||
require 'tempfile' | ||
|
||
module Kafo | ||
describe ParserCacheWriter do | ||
subject { ParserCacheWriter } | ||
|
||
describe ".write" do | ||
specify { subject.write([])[:version].must_equal 1 } | ||
specify { subject.write([])[:files].must_equal({}) } | ||
|
||
describe "with a module" do | ||
let(:manifest) { Tempfile.new("#{subject}_manifest") } | ||
let(:mod) do | ||
mod = MiniTest::Mock.new | ||
mod.expect(:identifier, 'module') | ||
mod.expect(:manifest_path, manifest.path) | ||
mod.expect(:raw_data, {:parameters => [], :groups => []}) | ||
mod | ||
end | ||
let(:output) { subject.write([mod]) } | ||
|
||
specify { output[:files].keys.must_equal ['module'] } | ||
specify { output[:files]['module'].keys.sort.must_equal [:data, :mtime] } | ||
specify { output[:files]['module'][:mtime].must_equal File.mtime(manifest.path).to_i } | ||
specify { output[:files]['module'][:data].has_key?(:parameters).must_equal true } | ||
specify { output[:files]['module'][:data][:parameters].must_equal([]) } | ||
end | ||
end | ||
end | ||
end |
Oops, something went wrong.