(#177) add a shell script based data store

CHANGELOG.md

@@ -1,5 +1,6 @@
 |Date      |Issue |Description                                                                                              |
 |----------|------|---------------------------------------------------------------------------------------------------------|
+|2017/02/13|177   |Add a shell script based data store                                                                      |
 |2017/02/12|      |Release 0.0.22                                                                                           |
 |2017/02/11|181   |Add a registration plugin                                                                                |
 |2017/02/09|176   |Create the choria public certs in the right directory on windows                                         |

lib/mcollective/util/playbook/data_stores.rb

@@ -3,6 +3,7 @@
 require_relative "data_stores/environment_data_store"
 require_relative "data_stores/file_data_store"
 require_relative "data_stores/memory_data_store"
+require_relative "data_stores/shell_data_store"
 
 module MCollective
   module Util

lib/mcollective/util/playbook/data_stores/shell_data_store.rb

@@ -0,0 +1,102 @@
+require_relative "base"
+
+module MCollective
+  module Util
+    class Playbook
+      class DataStores
+        class ShellDataStore < Base
+          attr_reader :command, :timeout, :environment, :cwd
+
+          def write(key, value)
+            run("write", key, "CHORIA_DATA_VALUE" => value)
+
+            nil
+          end
+
+          def delete(key)
+            run("delete", key)
+
+            nil
+          end
+
+          def read(key)
+            run("read", key).stdout.chomp
+          end
+
+          def run(action, key, environment={})
+            validate_key(key)
+
+            command = "%s --%s" % [@command, action]
+            options = shell_options
+
+            options["environment"].merge!(
+              environment.merge(
+                "CHORIA_DATA_KEY" => key,
+                "CHORIA_DATA_ACTION" => action
+              )
+            )
+
+            shell = run_command(command, options)
+
+            unless shell.status.exitstatus == 0
+              Log.warn("While running command %s: %s" % [command, shell.stderr])
+              raise("Could not %s key %s, got exitcode %d" % [action, key, shell.status.exitstatus])
+            end
+
+            shell
+          end
+
+          def run_command(command, options)
+            shell = Shell.new(command, options)
+            shell.runcommand
+            shell
+          end
+
+          def validate_key(key)
+            raise("Valid keys must match ^[a-zA-Z0-9_-]+$") unless key =~ /^[a-zA-Z0-9_-]+$/
+            true
+          end
+
+          def from_hash(properties)
+            @command = properties["command"]
+            @timeout = properties.fetch("timeout", 10)
+            @environment = properties.fetch("environment", {})
+            @cwd = properties["cwd"]
+
+            self
+          end
+
+          def validate_configuration!
+            raise("A command is required") unless @command
+            raise("Command %s is not executable" % @command) unless File.executable?(@command)
+            raise("Timeout should be an integer") unless @timeout.to_i.to_s == @timeout.to_s
+
+            if @environment
+              raise("Environment should be a hash") unless @environment.is_a?(Hash)
+
+              all_strings = @environment.map {|k, v| k.is_a?(String) && v.is_a?(String)}.all?
+              raise("All keys and values in the environment must be strings") unless all_strings
+            end
+
+            if @cwd
+              raise("cwd %s does not exist" % @cwd) unless File.exist?(@cwd)
+              raise("cwd %s is not a directory" % @cwd) unless File.directory?(@cwd)
+            end
+          end
+
+          def shell_options
+            unless @__options
+              @__options = {}
+              @__options["cwd"] = @cwd if @cwd
+              @__options["environment"] = @environment
+              @__options["timeout"] = Integer(@timeout)
+            end
+
+            # bacause environment is being edited
+            Marshal.load(Marshal.dump(@__options))
+          end
+        end
+      end
+    end
+  end
+end

module/data/plugin.yaml

@@ -21,6 +21,7 @@ mcollective_choria::client_files:
  - util/playbook/data_stores/environment_data_store.rb
  - util/playbook/data_stores/file_data_store.rb
  - util/playbook/data_stores/memory_data_store.rb
+ - util/playbook/data_stores/shell_data_store.rb
  - util/playbook/data_stores.rb
  - util/playbook/inputs.rb
  - util/playbook/nodes/mcollective_nodes.rb

spec/fixtures/playbooks/shell_data.rb

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby
+
+@action = ENV.fetch("CHORIA_DATA_ACTION", "").downcase
+@key = ENV["CHORIA_DATA_KEY"]
+@value = ENV["CHORIA_DATA_VALUE"]
+
+abort("Unknown action '%s', valid actions are read, write and delete" % @action) unless ["read", "write", "delete"].include?(@action)
+abort("A key is required") unless @key
+abort("Writing requires a value") if @action == "write" && !@value
+
+abort("forced failure simulation") if @key == "force_fail"
+
+def read(key)
+  STDERR.puts("Reading %s" % [key])
+
+  if File.exist?("/tmp/shell_data_tmp")
+    puts File.read("/tmp/shell_data_tmp").chomp
+  else
+    abort("no value")
+  end
+end
+
+def write(key)
+  STDERR.puts("Writing %s" % [key])
+
+  open("/tmp/shell_data_tmp", "w") {|f| f.puts @value}
+
+  puts @value
+end
+
+def delete(key)
+  STDERR.puts("Deleting %s" % [key])
+
+  File.unlink("/tmp/shell_data_tmp") if File.exist?("/tmp/shell_data_tmp")
+
+  puts @key
+end
+
+send(@action, @key)

spec/unit/mcollective/util/playbook/data_stores/shell_data_store_spec.rb

@@ -0,0 +1,167 @@
+require "spec_helper"
+require "mcollective/util/playbook"
+
+module MCollective
+  module Util
+    class Playbook
+      class DataStores
+        describe ShellDataStore do
+          let(:ds) { ShellDataStore.new("rspec", stub) }
+          let(:fixture) { File.expand_path("spec/fixtures/playbooks/shell_data.rb") }
+
+          before(:each) do
+            ds.from_hash("command" => fixture)
+          end
+
+          describe "#integration" do
+            it "should produce correct commands" do
+              expect { ds.read("x") }.to raise_error("Could not read key x, got exitcode 1")
+              ds.write("x", "y")
+              expect(ds.read("x")).to eq("y")
+              ds.delete("x")
+              expect { ds.read("x") }.to raise_error("Could not read key x, got exitcode 1")
+            end
+          end
+
+          describe "#shell_options" do
+            it "should return the right options" do
+              expect(ds.shell_options).to eq("timeout" => 10, "environment" => {})
+            end
+
+            it "should return the right options with all settings set" do
+              ds.from_hash("cwd" => "/nonexisting_cwd", "environment" => {"rspec" => "1"}, "timeout" => 20)
+              expect(ds.shell_options).to eq("timeout" => 20, "environment" => {"rspec" => "1"}, "cwd" => "/nonexisting_cwd")
+            end
+          end
+
+          describe "#validate_configuration!" do
+            it "should validate the command" do
+              ds.from_hash({})
+              expect { ds.validate_configuration! }.to raise_error("A command is required")
+
+              ds.from_hash("command" => "/nonexisting")
+              expect { ds.validate_configuration! }.to raise_error("Command /nonexisting is not executable")
+            end
+
+            it "should validate the timeout" do
+              File.stubs(:executable?).with("/nonexisting").returns(true)
+
+              ds.from_hash("command" => "/nonexisting", "timeout" => "a")
+              expect { ds.validate_configuration! }.to raise_error("Timeout should be an integer")
+            end
+
+            it "should validate the environment" do
+              File.stubs(:executable?).with("/nonexisting").returns(true)
+
+              ds.from_hash("command" => "/nonexisting", "environment" => 1)
+              expect { ds.validate_configuration! }.to raise_error("Environment should be a hash")
+
+              ds.from_hash("command" => "/nonexisting", "environment" => {1 => 1})
+              expect { ds.validate_configuration! }.to raise_error("All keys and values in the environment must be strings")
+
+              ds.from_hash("command" => "/nonexisting", "environment" => {"a" => 1})
+              expect { ds.validate_configuration! }.to raise_error("All keys and values in the environment must be strings")
+
+              ds.from_hash("command" => "/nonexisting", "environment" => {1 => "a"})
+              expect { ds.validate_configuration! }.to raise_error("All keys and values in the environment must be strings")
+            end
+
+            it "should validate the cwd" do
+              File.stubs(:executable?).with("/nonexisting").returns(true)
+
+              ds.from_hash("command" => "/nonexisting", "cwd" => "/nonexisting_cwd")
+              expect { ds.validate_configuration! }.to raise_error("cwd /nonexisting_cwd does not exist")
+
+              ds.from_hash("command" => "/nonexisting", "cwd" => "/nonexisting_cwd")
+              File.stubs(:exist?).returns(true)
+              expect { ds.validate_configuration! }.to raise_error("cwd /nonexisting_cwd is not a directory")
+            end
+
+            it "should accept valid configs" do
+              expect(ds.validate_configuration!).to be_nil
+            end
+          end
+
+          describe "#from_hash" do
+            it "should set sane defaults" do
+              expect(ds.timeout).to be(10)
+              expect(ds.environment).to eq({})
+              expect(ds.cwd).to be_nil
+            end
+
+            it "should accept supplied values" do
+              ds.from_hash("command" => fixture, "timeout" => 20, "environment" => {"rspec" => 1}, "cwd" => "/nonexisting")
+              expect(ds.command).to eq(File.expand_path("spec/fixtures/playbooks/shell_data.rb"))
+              expect(ds.timeout).to be(20)
+              expect(ds.environment).to eq("rspec" => 1)
+              expect(ds.cwd).to eq("/nonexisting")
+            end
+          end
+
+          describe "#validate_key" do
+            it "should not accept invalid keys" do
+              expect { ds.validate_key("foo|bar") }.to raise_error("Valid keys must match ^[a-zA-Z0-9_-]+$")
+            end
+
+            it "should accept valid keys" do
+              %w(foo_bar FOO_BAR FOO_bar 1FOO_bar FOO_bar1 1FOO1bar1).each do |test|
+                expect(ds.validate_key(test)).to be(true)
+              end
+            end
+          end
+
+          describe "#run_command" do
+            it "should create and run a shell" do
+              Shell.expects(:new).with("/nonexisting/command", "stdin" => "rspec").returns(s = stub)
+              s.expects(:runcommand)
+              expect(ds.run_command("/nonexisting/command", "stdin" => "rspec")).to be(s)
+            end
+          end
+
+          describe "#run" do
+            it "should only accept valid keys" do
+              expect { ds.run("read", "foo|bar") }.to raise_error("Valid keys must match ^[a-zA-Z0-9_-]+$")
+            end
+
+            it "should support a supplied environment" do
+              ds.expects(:run_command).with("#{fixture} --write",
+                                            "timeout" => 10,
+                                            "environment" => {
+                                              "CHORIA_DATA_VALUE" => "hello world",
+                                              "CHORIA_DATA_KEY" => "rspec_test",
+                                              "CHORIA_DATA_ACTION" => "write"
+                                            }).returns(stub(:status => stub(:exitstatus => 0)))
+
+              ds.write("rspec_test", "hello world")
+            end
+
+            it "should detect command failures" do
+              expect { ds.run("read", "force_fail") }.to raise_error("Could not read key force_fail, got exitcode 1")
+            end
+          end
+
+          describe "#read" do
+            it "should read the key correctly" do
+              ds.expects(:run).with("read", "rspec_key").returns(stub(:stdout => "rspec_data"))
+              expect(ds.read("rspec_key")).to eq("rspec_data")
+            end
+          end
+
+          describe "#write" do
+            it "should write the key correctly" do
+              ds.expects(:run).with("write", "rspec_key", "CHORIA_DATA_VALUE" => "rspec value")
+              expect(ds.write("rspec_key", "rspec value")).to be_nil
+            end
+          end
+
+          describe "#delete" do
+            it "should delete the key correctly" do
+              ds.expects(:run).with("delete", "rspec_key")
+              expect(ds.delete("rspec_key")).to be_nil
+            end
+          end
+        end
+      end
+    end
+  end
+end