Initial implementation of multiple template dirs support

Different approach with multiple renderers

rubocop

change ejson_secret_provisioner to handle multiple template dirs

exe/kubernetes-deploy

@@ -5,7 +5,7 @@ require 'kubernetes-deploy'
 require 'optparse'
 
 skip_wait = false
-template_dir = nil
+template_dirs = []
 allow_protected_ns = false
 prune = true
 bindings = {}
@@ -23,7 +23,7 @@ ARGV.options do |opts|
   opts.on("--allow-protected-ns", "Enable deploys to #{prot_ns}; requires --no-prune") { allow_protected_ns = true }
   opts.on("--no-prune", "Disable deletion of resources that do not appear in the template dir") { prune = false }
   opts.on("--template-dir=DIR", "Set the template dir (default: config/deploy/$ENVIRONMENT).") do |d|
-    template_dir = d
+    template_dirs << d
   end
   opts.on("--verbose-log-prefix", "Add [context][namespace] to the log prefix") { verbose_log_prefix = true }
   opts.on("--max-watch-seconds=seconds",
@@ -52,12 +52,12 @@ context = ARGV[1]
 logger = KubernetesDeploy::FormattedLogger.build(namespace, context, verbose_prefix: verbose_log_prefix)
 
 begin
-  KubernetesDeploy::OptionsHelper.with_validated_template_dir(template_dir) do |dir|
+  KubernetesDeploy::OptionsHelper.with_validated_template_dirs(template_dirs) do |dirs|
     runner = KubernetesDeploy::DeployTask.new(
       namespace: namespace,
       context: context,
       current_sha: ENV["REVISION"],
-      template_dir: dir,
+      template_dirs: dirs,
       bindings: bindings,
       logger: logger,
       max_watch_seconds: max_watch_seconds,

lib/kubernetes-deploy/deploy_task.rb

@@ -102,22 +102,24 @@ def server_version
       kubectl.server_version
     end
 
-    def initialize(namespace:, context:, current_sha:, template_dir:, logger:, kubectl_instance: nil, bindings: {},
+    def initialize(namespace:, context:, current_sha:, template_dirs:, logger:, kubectl_instance: nil, bindings: {},
       max_watch_seconds: nil, selector: nil)
       @namespace = namespace
       @namespace_tags = []
       @context = context
       @current_sha = current_sha
-      @template_dir = File.expand_path(template_dir)
+      @template_dirs = template_dirs.map { |template_dir| File.expand_path(template_dir) }
       @logger = logger
       @kubectl = kubectl_instance
       @max_watch_seconds = max_watch_seconds
-      @renderer = KubernetesDeploy::Renderer.new(
-        current_sha: @current_sha,
-        template_dir: @template_dir,
-        logger: @logger,
-        bindings: bindings,
-      )
+      @renderers = @template_dirs.map do |template_dir|
+        KubernetesDeploy::Renderer.new(
+          current_sha: @current_sha,
+          template_dir: template_dir,
+          logger: @logger,
+          bindings: bindings,
+        )
+      end
       @selector = selector
     end
 
@@ -204,15 +206,17 @@ def cluster_resource_discoverer
       )
     end
 
-    def ejson_provisioner
-      @ejson_provisioner ||= EjsonSecretProvisioner.new(
-        namespace: @namespace,
-        context: @context,
-        template_dir: @template_dir,
-        logger: @logger,
-        statsd_tags: @namespace_tags,
-        selector: @selector,
-      )
+    def ejson_provisioners
+      @ejson_provisioners ||= @template_dirs.map do |template_dir|
+        EjsonSecretProvisioner.new(
+          namespace: @namespace,
+          context: @context,
+          template_dir: template_dir,
+          logger: @logger,
+          statsd_tags: @namespace_tags,
+          selector: @selector,
+        )
+      end
     end
 
     def deploy_has_priority_resources?(resources)
@@ -267,21 +271,23 @@ def check_initial_status(resources)
     measure_method(:check_initial_status, "initial_status.duration")
 
     def secrets_from_ejson
-      ejson_provisioner.resources
+      ejson_provisioners.flat_map(&:resources)
     end
 
     def discover_resources
       resources = []
       crds = cluster_resource_discoverer.crds.group_by(&:kind)
       @logger.info("Discovering templates:")
 
-      TemplateDiscovery.new(@template_dir).templates.each do |filename|
-        split_templates(filename) do |r_def|
-          crd = crds[r_def["kind"]]&.first
-          r = KubernetesResource.build(namespace: @namespace, context: @context, logger: @logger, definition: r_def,
-            statsd_tags: @namespace_tags, crd: crd)
-          resources << r
-          @logger.info("  - #{r.id}")
+      @template_dirs.each do |template_dir|
+        TemplateDiscovery.new(template_dir).templates.each do |filename|
+          split_templates(template_dir, filename) do |r_def|
+            crd = crds[r_def["kind"]]&.first
+            r = KubernetesResource.build(namespace: @namespace, context: @context, logger: @logger, definition: r_def,
+              statsd_tags: @namespace_tags, crd: crd)
+            resources << r
+            @logger.info("  - #{r.id}")
+          end
         end
       end
       secrets_from_ejson.each do |secret|
@@ -296,9 +302,11 @@ def discover_resources
     end
     measure_method(:discover_resources)
 
-    def split_templates(filename)
-      file_content = File.read(File.join(@template_dir, filename))
-      rendered_content = @renderer.render_template(filename, file_content)
+    def split_templates(template_dir, filename)
+      file_content = File.read(File.join(template_dir, filename))
+      rendered_content = @renderers.find do |renderer|
+        renderer.template_dir == template_dir
+      end.render_template(filename, file_content)
       YAML.load_stream(rendered_content, "<rendered> #{filename}") do |doc|
         next if doc.blank?
         unless doc.is_a?(Hash)
@@ -333,10 +341,12 @@ def validate_configuration(allow_protected_ns:, prune:)
       errors = []
       errors += kubeclient_builder.validate_config_files
 
-      if !File.directory?(@template_dir)
-        errors << "Template directory `#{@template_dir}` doesn't exist"
-      elsif Dir.entries(@template_dir).none? { |file| file =~ /(\.ya?ml(\.erb)?)$|(secrets\.ejson)$/ }
-        errors << "`#{@template_dir}` doesn't contain valid templates (secrets.ejson or postfix .yml, .yml.erb)"
+      @template_dirs.each do |template_dir|
+        if !File.directory?(template_dir)
+          errors << "Template directory `#{template_dir}` doesn't exist"
+        elsif Dir.entries(template_dir).none? { |file| file =~ /(\.ya?ml(\.erb)?)$|(secrets\.ejson)$/ }
+          errors << "`#{template_dir}` doesn't contain valid templates (secrets.ejson or postfix .yml, .yml.erb)"
+        end
       end
 
       if @namespace.blank?
@@ -568,7 +578,7 @@ def namespace_definition
 
     # make sure to never prune the ejson-keys secret
     def confirm_ejson_keys_not_prunable
-      secret = ejson_provisioner.ejson_keys_secret
+      secret = ejson_provisioners.first.ejson_keys_secret
       return unless secret.dig("metadata", "annotations", KubernetesResource::LAST_APPLIED_ANNOTATION)
 
       @logger.error("Deploy cannot proceed because protected resource " \

lib/kubernetes-deploy/options_helper.rb

@@ -6,22 +6,35 @@ class OptionsError < StandardError; end
 
     STDIN_TEMP_FILE = "from_stdin.yml.erb"
     class << self
-      def with_validated_template_dir(template_dir)
-        if template_dir == '-'
+      def with_validated_template_dirs(template_dirs)
+        if template_dirs.select { |dir| dir == "-" }.length > 2
+          raise OptionsError, "Cannot specify stdin as a template directory more than once"
+        end
+
+        dirs = []
+        if template_dirs.empty?
+          dirs << default_template_dir
+        else
+          template_dirs.each do |template_dir|
+            next if template_dir == '-'
+            dirs << template_dir
+          end
+        end
+
+        if template_dirs.include?("-")
           Dir.mktmpdir("kubernetes-deploy") do |dir|
             template_dir_from_stdin(temp_dir: dir)
-            yield dir
+            dirs << dir
+            yield dirs
           end
-        elsif template_dir
-          yield template_dir
         else
-          yield default_template_dir(template_dir)
+          yield dirs
         end
       end
 
       private
 
-      def default_template_dir(template_dir)
+      def default_template_dir
         if ENV.key?("ENVIRONMENT")
           template_dir = File.join("config", "deploy", ENV['ENVIRONMENT'])
         end

lib/kubernetes-deploy/renderer.rb

@@ -15,6 +15,7 @@ def initialize(msg, parents: [], content: nil, filename:)
       end
     end
     class PartialNotFound < InvalidTemplateError; end
+    attr_reader :template_dir
 
     def initialize(current_sha:, template_dir:, logger:, bindings: {})
       @current_sha = current_sha