Add comma based multi-pathing (#69)

Co-authored-by: Marcin Pietraszek <mpietraszek@opera.com>

.github/workflows/test.yml

@@ -0,0 +1,20 @@
+name: Test
+on: [push, pull_request]
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    env:
+      BUNDLE_WITHOUT: development
+    steps:
+      - name: Install vault
+        run: |
+          curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add -
+          sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
+          sudo apt-get update && sudo apt-get install vault
+      - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          # Can't use jruby-9.1.17.0 due to issues with bundler 2.X: https://github.com/ruby/setup-ruby/issues/108
+          ruby-version: jruby-9.2.9.0
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+      - run: bundle exec rspec

.jrubyrc

@@ -0,0 +1 @@
+debug.fullTrace=true

.rspec

@@ -1,3 +1,4 @@
 --color
 --format documentation
 --backtrace
+--pattern 'spec/functions/**/*'

Gemfile

@@ -12,6 +12,7 @@ group :test do
 
   gem 'rake'
   gem 'puppet-lint'
+  gem 'rspec'
   gem 'rspec-puppet'
   gem 'puppet-syntax'
   gem 'puppetlabs_spec_helper'

Gemfile.lock

@@ -32,6 +32,7 @@ GEM
       faraday (~> 0.8)
     fast_gettext (1.1.2)
     ffi (1.9.25)
+    ffi (1.9.25-java)
     gettext (3.2.9)
       locale (>= 2.0.5)
       text (>= 1.3.0)
@@ -49,6 +50,7 @@ GEM
       retriable (~> 3.0)
     hiera (3.7.0)
     hitimes (1.3.0)
+    hitimes (1.3.0-java)
     hocon (1.3.1)
     http-accept (1.7.0)
     http-cookie (1.0.3)
@@ -57,7 +59,9 @@ GEM
     i18n (0.9.5)
       concurrent-ruby (~> 1.0)
     jaro_winkler (1.5.4)
+    jaro_winkler (1.5.4-java)
     json (2.3.0)
+    json (2.3.0-java)
     json-schema (2.8.0)
       addressable (>= 2.4)
     json_pure (2.1.0)
@@ -107,6 +111,10 @@ GEM
     pry (0.12.2)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
+    pry (0.12.2-java)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+      spoon (~> 0.0)
     public_suffix (4.0.1)
     puppet (6.21.1)
       concurrent-ruby (~> 1.0)
@@ -181,10 +189,13 @@ GEM
       terminal-table
     simplecov-html (0.10.2)
     spdx-licenses (1.2.0)
+    spoon (0.0.6)
+      ffi
     terminal-table (1.8.0)
       unicode-display_width (~> 1.1, >= 1.1.1)
     text (1.3.1)
     thread_safe (0.3.6)
+    thread_safe (0.3.6-java)
     tty-color (0.5.0)
     tty-cursor (0.7.0)
     tty-prompt (0.20.0)
@@ -203,14 +214,16 @@ GEM
       thread_safe (~> 0.1)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.6)
+    unf (0.1.4-java)
+    unf_ext (0.0.7.7)
     unicode-display_width (1.6.0)
     vault (0.13.0)
       aws-sigv4
     wisper (2.0.1)
 
 PLATFORMS
   ruby
+  universal-java-16
 
 DEPENDENCIES
   activesupport (< 5)
@@ -227,6 +240,7 @@ DEPENDENCIES
   puppetlabs_spec_helper
   rake
   rb-readline
+  rspec
   rspec-puppet
   rubocop
   rubocop-rspec
@@ -236,4 +250,4 @@ DEPENDENCIES
   vault (>= 0.13.0)
 
 BUNDLED WITH
-   2.0.1
+   2.2.24

README.md

@@ -12,7 +12,7 @@ For an example repo of it in action, check out the [hashicorp/webinar-vault-hier
 
 ### Compatibility
 
-* This module is only compatible with Hiera 5 (ships with Puppet 4.9+) and Vault KV engine version 2 (Vault 0.10+)
+- This module is only compatible with Hiera 5 (ships with Puppet 4.9+) and Vault KV engine version 2 (Vault 0.10+)
 
 ### Requirements
 
@@ -101,17 +101,16 @@ The following is an example Hiera 5 hiera.yaml configuration for use with hiera-
 
 ```yaml
 ---
-
 version: 5
 
 hierarchy:
   - name: "Hiera-vault lookup"
     lookup_key: hiera_vault
     options:
       confine_to_keys:
-        - '^vault_.*'
-        - '^.*_password$'
-        - '^password.*'
+        - "^vault_.*"
+        - "^.*_password$"
+        - "^password.*"
       ssl_verify: false
       address: https://vault.foobar.com:8200
       token: <insert-your-vault-token-here>
@@ -154,10 +153,14 @@ strip_from_keys:
 
 `default_field:`: The default field within data to return. If not present, the lookup will be the full contents of the secret data.
 
-`mounts:`: The list of mounts you want to do lookups against. This is treated as the backend hiearchy for lookup. It is recomended you use [Trusted Facts](https://puppet.com/docs/puppet/5.3/lang_facts_and_builtin_vars.html#trusted-facts) within the hierachy to ensure lookups are restricted to the correct hierachy points. See [Mounts](####Mounts)
+`mounts:`: The list of mounts you want to do lookups against. This is treated as the backend hiearchy for lookup. It is recomended you use [Trusted Facts](https://puppet.com/docs/puppet/5.3/lang_facts_and_builtin_vars.html#trusted-facts) within the hierachy to ensure lookups are restricted to the correct hierachy points. See [Mounts](#mounts).
 
 `:ssl_verify`: Specify whether to verify SSL certificates (default: true)
 
+`v1_lookup`: whether to lookup within kv v1 hierarchy (default `true`) - disable if you only use kv v2 :) See [Less lookups](#less-lookups).
+
+`v2_guess_mount`: whether to try to guess mount for KV v2 (default `true`) - add `data` after your mount and disable to minimize amount of misses. See [Less lookups](#less-lookups).
+
 ### Debugging
 
 ```
@@ -206,16 +209,88 @@ $cool_key = lookup({"name" => "cool_key", "default_value" => "No Vault Secret Fo
 ```
 
 Secrets will then be looked up with the following paths:
-* http://vault.foobar.com:8200/some_secret/foo.example.com/cool_key (for v1)
-* http://vault.foobar.com:8200/some_secret/foo.example.com/data/cool_key (for v2)
-* http://vault.foobar.com:8200/some_secret/common/cool_key (for v1)
-* http://vault.foobar.com:8200/some_secret/common/data/cool_key (for v2)
-* http://vault.foobar.com:8200/another_secret/foo.example.com/cool_key (for v1)
-* http://vault.foobar.com:8200/another_secret/foo.example.com/data/cool_key (for v2)
-* http://vault.foobar.com:8200/another_secret/common/cool_key (for v1)
-* http://vault.foobar.com:8200/another_secret/common/data/cool_key (for v2)
+
+- http://vault.foobar.com:8200/some_secret/foo.example.com/cool_key (for v1)
+- http://vault.foobar.com:8200/some_secret/foo.example.com/data/cool_key (for v2)
+- http://vault.foobar.com:8200/some_secret/data/foo.example.com/cool_key (for v2)
+- http://vault.foobar.com:8200/some_secret/common/cool_key (for v1)
+- http://vault.foobar.com:8200/some_secret/common/data/cool_key (for v2)
+- http://vault.foobar.com:8200/some_secret/data/common/cool_key (for v2)
+
+#### Less lookups
+
+You can use `v1_lookup` and `v2_guess_mount` to minimize misses in above lookups.
+
+Changing above configuration to
+
+```yaml
+v2_guess_mount: false
+v1_lookup: false
+mounts:
+  some_secret/data:
+    - %{::trusted.certname}
+    - common
+```
+
+would result in following lookups:
+
+- http://vault.foobar.com:8200/some_secret/data/foo.example.com/cool_key (for v2)
+- http://vault.foobar.com:8200/some_secret/data/common/cool_key (for v2)
+
+#### Multiple keys in trusted certname
+
+Often you want to whitelist multiple paths for each host (e.g. due to host having multiple roles). In this case simply add keys delimited with comma to trusted field. For example:
+
+```yaml
+mounts:
+  secret:
+    - "%{trusted.extensions.pp_role}"
+```
+
+and host configured with
+
+```yaml
+---
+extension_requests:
+  pp_role: api,ssl
+```
+
+would result in lookups in:
+
+- http://vault.foobar.com:8200/secret/api/cool_key (for v1)
+- http://vault.foobar.com:8200/secret/api/data/cool_key (for v2)
+- http://vault.foobar.com:8200/secret/data/api/cool_key (for v2)
+- http://vault.foobar.com:8200/secret/ssl/cool_key (for v1)
+- http://vault.foobar.com:8200/secret/ssl/data/cool_key (for v2)
+- http://vault.foobar.com:8200/secret/data/ssl/cool_key (for v2)
+
+#### More verbose paths in Hiera
+
+Often implicit path extension makes it hard to understand which exact paths are used for given host - as you need to inspect both Hiera and trusted field for each host.
+
+With above configuration and lookup `$cool_key = lookup({"name" => "cool_key"})` you cannot be sure whether `api/cool_key` or `ssl/cool_key` will be used (whichever happens to be first in lookup list).
+
+To alleviate this problem you can use full paths in Hiera, provided `v2_guess_mount: false` configuration is active. For example with:
+
+```yaml
+v2_guess_mount: false
+v1_lookup: false
+mounts:
+  secret/data:
+    - "%{trusted.extensions.pp_role}"
+```
+
+You can use `$cool_key = lookup({"name" => "ssl/cool_key"})` to ensure `http://vault.foobar.com:8200/secret/data/ssl/cool_key` will be used.
+
+And make yourself a favor and avoid `lookup` directly ;) Use
+
+```yaml
+profile::ssl_role::key: "%{alias('vault_storage::ssl/params.key')}"
+```
+
+to inject value from `key` inside `http://vault.foobar.com:8200/secret/data/ssl/params`.
 
 ### Author
 
-* Original - David Alden <dave@alden.name>
-* Transfered and maintained by Peter Souter
+- Original - David Alden <dave@alden.name>
+- Transfered and maintained by Peter Souter

lib/puppet/functions/hiera_vault.rb

@@ -130,19 +130,26 @@ def vault_get(key, options, context)
 
     # Only kv mounts supported so far
     kv_mounts.each_pair do |mount, paths|
-      paths.each do |path|
-
+      interpolate(context, paths).each do |path|
         secretpath = context.interpolate(File.join(mount, path))
 
         context.explain { "[hiera-vault] Looking in path #{secretpath} for #{key}" }
 
         secret = nil
 
-        [
-          [:v1, File.join(mount, path, key)],
-          [:v2, File.join(mount, path, 'data', key).chomp('/')],
-          [:v2, File.join(mount, 'data', path, key).chomp('/')],
-        ].each do |version_path|
+        paths = []
+
+        if options.fetch("v2_guess_mount", true)
+          paths << [:v2, File.join(mount, path, 'data', key).chomp('/')]
+          paths << [:v2, File.join(mount, 'data', path, key).chomp('/')]
+        else
+          paths << [:v2, File.join(mount, path, key).chomp('/')]
+          paths << [:v2, File.join(mount, key).chomp('/')] if key.start_with?(path)
+        end
+
+        paths << [:v1, File.join(mount, path, key)] if options.fetch("v1_lookup", true)
+
+        paths.each do |version_path|
           begin
             version, path = version_path[0], version_path[1]
             context.explain { "[hiera-vault] Checking path: #{path}" }
@@ -209,4 +216,31 @@ def stringify_keys(value)
       value
     end
   end
+
+  def interpolate(context, paths)
+    allowed_paths = []
+    paths.each do |path|
+      path = context.interpolate(path)
+      # TODO: Unify usage of '/' - File.join seems to be a mistake, since it won't work on Windows
+      # secret/puppet/scope1,scope2 => [[secret], [puppet], [scope1, scope2]]
+      segments = path.split('/').map { |segment| segment.split(',') }
+      allowed_paths += build_paths(segments) unless segments.empty?
+    end
+    allowed_paths
+  end
+
+  # [[secret], [puppet], [scope1, scope2]] => ['secret/puppet/scope1', 'secret/puppet/scope2']
+  def build_paths(segments)
+    paths = [[]]
+    segments.each do |segment|
+      p = paths.dup
+      paths.clear
+      segment.each do |option|
+        p.each do |path|
+          paths << path + [option]
+        end
+      end
+    end
+    paths.map { |p| File.join(*p) }
+  end
 end

spec/functions/hiera_vault_happy_path_v2_spec.rb

@@ -58,6 +58,7 @@ def vault_test_client
           vault_test_client.logical.write('puppetv2/data/common/broken_json_key', { "data" => { value: '[,'} } )
           vault_test_client.logical.write('puppetv2/data/common/confined_vault_key', { "data" => { value: 'find_me'} } )
           vault_test_client.logical.write('puppetv2/data/common/stripped_key', { "data" => { value: 'regexed_key'} } )
+          vault_test_client.logical.write('puppetv2/data/common/complex_structure_key', { "data" => { hash: {a: 1}, array: [1, 2], hash_with_array: {a: [1, 2]}, array_with_hash: [{a: 1}, {b: 2}]} } )
         end
 
         context 'configuring vault' do
@@ -130,6 +131,11 @@ def vault_test_client
               .to include('a' => 1, 'b' => 2, 'c' => 3)
           end
 
+          it 'should return a complex object' do
+            expect(function.lookup_key('complex_structure_key', vault_options, context))
+              .to include("hash" => {"a" => 1}, "array" => [1, 2], "hash_with_array" => {"a" => [1, 2]}, "array_with_hash" => [{"a" => 1}, {"b" => 2}])
+          end
+
           it 'should return an array parsed from json' do
             expect(function.lookup_key('array_key', {
               'default_field' => 'value',