(PE-28877) Teach ASTCompiler about bolt

documentation/puppet-api/v3/compile.markdown

@@ -36,7 +36,9 @@ The request body must look like:
   "transaction_uuid": "<uuid string>",
   "job_id": "<id string>",
   "options": { "capture_logs": <true/false>,
-               "log_level": <one of debug/info/warning/err>}
+               "log_level": <one of debug/info/warning/err>
+               "bolt": <true/false> Whether or not to attempt to load bolt
+               "boltlib": <String> Path to bolt-modules to prepend to modulepath}
 }
 ```
 

src/clj/puppetlabs/services/jruby/jruby_puppet_schemas.clj

@@ -53,7 +53,11 @@
 
     * :use-legacy-auth-conf - Whether to use the legacy core Puppet auth.conf
         (true) or trapperkeeper-authorization (false) to authorize requests
-        being made to core Puppet endpoints."
+        being made to core Puppet endpoints.
+
+    * :botlib-path - Optional array containng path(s) to bolt modules. This path
+         will be prepended to AST compilation modulepath. This is required for
+         compiling AST that contains bolt types"
   {:master-conf-dir (schema/maybe schema/Str)
    :master-code-dir (schema/maybe schema/Str)
    :master-var-dir (schema/maybe schema/Str)
@@ -67,5 +71,6 @@
    :http-client-idle-timeout-milliseconds schema/Int
    :http-client-metrics-enabled schema/Bool
    :max-requests-per-instance schema/Int
-   :use-legacy-auth-conf schema/Bool})
+   :use-legacy-auth-conf schema/Bool
+   (schema/optional-key :botlib-path) [schema/Str]})
 

src/clj/puppetlabs/services/jruby/jruby_puppet_service.clj

@@ -85,8 +85,8 @@
    (.compileCatalog jruby-instance request-options))
 
   (compile-ast
-    [this jruby-instance compile-options]
-    (.compileAST jruby-instance compile-options))
+    [this jruby-instance compile-options boltlib-path]
+    (.compileAST jruby-instance compile-options boltlib-path))
 
   (get-environment-module-info
     [this jruby-instance env-name]

src/clj/puppetlabs/services/legacy_routes/legacy_routes_service.clj

@@ -36,6 +36,9 @@
                                 master-ns
                                 config)
           jruby-service (tk-services/get-service this :JRubyPuppetService)
+          boltlib-path (get-in config
+                               [:jruby-puppet :boltlib-path]
+                               [""])
           master-routes (comidi/context path
                                         (master-core/root-routes handle-request
                                                                  (partial identity)
@@ -45,7 +48,8 @@
                                                                    (throw (IllegalStateException.
                                                                             (i18n/trs "Versioned code not supported."))))
                                                                  (constantly nil)
-                                                                 false))
+                                                                 false
+                                                                 boltlib-path))
           master-route-handler (comidi/routes->handler master-routes)
           master-mount (master-core/get-master-mount
                         master-ns

src/clj/puppetlabs/services/master/master_core.clj

@@ -890,13 +890,15 @@
   {(schema/required-key "certname") schema/Str
    (schema/required-key "environment") schema/Str
    (schema/required-key "code_ast") schema/Str
-   (schema/required-key "trusted_facts") {(schema/required-key "values") {schema/Str schema/Any}}
-   (schema/required-key "facts") {(schema/required-key "values") {schema/Str schema/Any}}
-   (schema/required-key "variables") {(schema/required-key "values") {schema/Str schema/Any}}
+   (schema/required-key "trusted_facts") {(schema/required-key "values") schema/Str}
+   (schema/required-key "facts") {(schema/required-key "values") schema/Str}
+   (schema/required-key "variables") {(schema/required-key "values") schema/Str}
    (schema/optional-key "job_id") schema/Str
    (schema/optional-key "transaction_uuid") schema/Str
    (schema/optional-key "options") {(schema/optional-key "capture_logs") schema/Bool
-                                    (schema/optional-key "log_level") (schema/enum "debug" "info" "warning" "error")}})
+                                    (schema/optional-key "log_level") (schema/enum "debug" "info" "warning" "error")
+                                    (schema/optional-key "bolt") schema/Bool
+                                    (schema/optional-key "boltlib_path") [schema/Str]}})
 
 (defn validated-body
   [body schema]
@@ -929,7 +931,8 @@
 
 (schema/defn ^:always-validate compile-fn :- IFn
   [jruby-service :- (schema/protocol jruby-protocol/JRubyPuppetService)
-   current-code-id-fn :- IFn]
+   current-code-id-fn :- IFn
+   boltlib-path :- [schema/Str]]
   (fn [request]
     (let [env (jruby-request/get-environment-from-request request)
           request-options (-> request
@@ -944,7 +947,8 @@
        :body (json/encode
                (jruby-protocol/compile-ast jruby-service
                                            (:jruby-instance request)
-                                           compile-options))})))
+                                           compile-options
+                                           boltlib-path))})))
 
 (schema/defn ^:always-validate
   v4-catalog-handler :- IFn
@@ -960,8 +964,9 @@
   compile-handler :- IFn
   [jruby-service :- (schema/protocol jruby-protocol/JRubyPuppetService)
    wrap-with-jruby-queue-limit :- IFn
-   current-code-id-fn :- IFn]
-  (-> (compile-fn jruby-service current-code-id-fn)
+   current-code-id-fn :- IFn
+   boltlib-path :- [schema/Str]]
+  (-> (compile-fn jruby-service current-code-id-fn boltlib-path)
       (jruby-request/wrap-with-jruby-instance jruby-service)
       wrap-with-jruby-queue-limit
       jruby-request/wrap-with-error-handling))
@@ -1023,7 +1028,8 @@
    get-code-content-fn :- IFn
    current-code-id-fn :- IFn
    cache-enabled :- schema/Bool
-   wrap-with-jruby-queue-limit :- IFn]
+   wrap-with-jruby-queue-limit :- IFn
+   boltlib-path :- [schema/Str]]
   (let [class-handler (create-cacheable-info-handler-with-middleware
                         (fn [jruby env]
                           (some-> jruby-service
@@ -1057,7 +1063,8 @@
         compile-handler' (compile-handler
                           jruby-service
                           wrap-with-jruby-queue-limit
-                          current-code-id-fn)]
+                          current-code-id-fn
+                          boltlib-path)]
     (comidi/routes
       (comidi/POST "/compile" request
                    (compile-handler' request))
@@ -1107,15 +1114,17 @@
    get-code-content-fn :- IFn
    current-code-id-fn :- IFn
    environment-class-cache-enabled :- schema/Bool
-   wrap-with-jruby-queue-limit :- IFn]
+   wrap-with-jruby-queue-limit :- IFn
+   boltlib-path :- [schema/Str]]
   (comidi/context "/v3"
                   (v3-ruby-routes ruby-request-handler)
                   (comidi/wrap-routes
                    (v3-clojure-routes jruby-service
                                       get-code-content-fn
                                       current-code-id-fn
                                       environment-class-cache-enabled
-                                      wrap-with-jruby-queue-limit)
+                                      wrap-with-jruby-queue-limit
+                                      boltlib-path)
                    clojure-request-wrapper)))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1216,15 +1225,17 @@
    wrap-with-jruby-queue-limit :- IFn
    get-code-content-fn :- IFn
    current-code-id-fn :- IFn
-   environment-class-cache-enabled :- schema/Bool]
+   environment-class-cache-enabled :- schema/Bool
+   boltlib-path :- [schema/Str]]
   (comidi/routes
    (v3-routes ruby-request-handler
               clojure-request-wrapper
               jruby-service
               get-code-content-fn
               current-code-id-fn
               environment-class-cache-enabled
-              wrap-with-jruby-queue-limit)
+              wrap-with-jruby-queue-limit
+              boltlib-path)
    (v4-routes clojure-request-wrapper
               jruby-service
               wrap-with-jruby-queue-limit
@@ -1301,7 +1312,8 @@
    handle-request :- IFn
    wrap-with-authorization-check :- IFn
    wrap-with-jruby-queue-limit :- IFn
-   environment-class-cache-enabled :- schema/Bool]
+   environment-class-cache-enabled :- schema/Bool
+   boltlib-path :- [schema/Str]]
   (let [ruby-request-handler (get-wrapped-handler handle-request
                                                   wrap-with-authorization-check
                                                   puppet-version
@@ -1317,7 +1329,8 @@
                  wrap-with-jruby-queue-limit
                  get-code-content
                  current-code-id
-                 environment-class-cache-enabled)))
+                 environment-class-cache-enabled
+                 boltlib-path)))
 
 (def MasterStatusV1
   {(schema/optional-key :experimental) {:http-metrics [http-metrics/RouteSummary]

src/clj/puppetlabs/services/master/master_service.clj

@@ -122,6 +122,10 @@
                                            max-retry-delay))
                                        identity)
 
+         boltlib-path (get-in config
+                              [:jruby-puppet :boltlib-path]
+                              [""])
+
          ring-app (comidi/routes
                    (core/construct-root-routes puppet-version
                                                use-legacy-auth-conf
@@ -131,7 +135,8 @@
                                                handle-request
                                                (get-auth-handler)
                                                wrap-with-jruby-queue-limit
-                                               environment-class-cache-enabled))
+                                               environment-class-cache-enabled
+                                               boltlib-path))
          routes (comidi/context path ring-app)
          route-metadata (comidi/route-metadata routes)
          comidi-handler (comidi/routes->handler routes)

src/clj/puppetlabs/services/protocols/jruby_puppet.clj

@@ -89,7 +89,7 @@
     "Compile the catalog for a given certname")
 
   (compile-ast
-    [this jruby-instance compile-options]
+    [this jruby-instance compile-options boltlib-path]
     "Compiles arbitrary Puppet ASTs into mini catalogs")
 
   (get-environment-module-info

src/java/com/puppetlabs/puppetserver/JRubyPuppet.java

@@ -24,7 +24,7 @@ public interface JRubyPuppet {
     List getTransportInfoForEnvironment(String environment);
     List getModuleInfoForEnvironment(String environment);
     Map compileCatalog(Map requestBody);
-    Map compileAST(Map compileOptions);
+    Map compileAST(Map compileOptions, List boltlibPath);
     Map getModuleInfoForAllEnvironments();
     JRubyPuppetResponse handleRequest(Map request);
     Object getSetting(String setting);

src/ruby/puppetserver-lib/puppet/server/ast_compiler.rb

@@ -5,45 +5,118 @@
 module Puppet
   module Server
     class ASTCompiler
-      def self.compile(compile_options)
+      def self.compile(compile_options, boltlib_path)
         options = compile_options['options'] || {}
 
         log_level = options['log_level']
         code = JSON.parse(compile_options['code_ast'])
 
         if options['capture_logs']
           catalog, logs = Logging.capture_logs(log_level) do
-            compile_ast(code, compile_options)
+            compile_ast(code, compile_options, boltlib_path)
           end
 
           { catalog: catalog, logs: logs }
         else
-          catalog = compile_ast(code, compile_options)
+          catalog = compile_ast(code, compile_options, boltlib_path)
           { catalog: catalog }
         end
       end
 
-      def self.compile_ast(code, compile_options)
-        Puppet[:node_name_value] = compile_options['certname']
-
-        # Use the existing environment with the requested name
-        Puppet::Pal.in_environment(compile_options['environment'],
-                                   envpath: Puppet[:environmentpath],
-                                   facts: compile_options['facts']['values'],
-                                   variables: compile_options['variables']['values']) do |pal|
-          Puppet.lookup(:pal_current_node).trusted_data = compile_options['trusted_facts']['values']
-
-          # This compiler has been configured with a node containing
-          # the requested environment, facts, and variables, and is used
-          # to compile a catalog in that context from the supplied AST.
-          pal.with_catalog_compiler do |compiler|
-            # We have to parse the AST inside the compiler block, because it
-            # initializes the necessary type loaders for us.
-            ast = Puppet::Pops::Serialization::FromDataConverter.convert(code)
-
-            compiler.evaluate(ast)
-            compiler.compile_additions
-            compiler.catalog_data_hash
+      def self.compile_ast(code, compile_options, boltlib_path)
+        # If the request requires that bolt be loaded we assume we are in a properly 
+        # configured PE environment
+        if compile_options.dig('options', 'bolt')
+          require 'bolt'
+          require 'bolt/target'
+          require 'bolt/apply_inventory'
+          require 'bolt/apply_target'
+          require 'bolt/apply_result'
+          require 'addressable'
+
+          Puppet[:rich_data] = true
+          Puppet[:node_name_value] = compile_options['certname']
+          json_deserialized_facts = JSON.parse(compile_options['facts']['values'])
+          json_deserialized_trusted_facts = JSON.parse(compile_options['trusted_facts']['values'])
+          json_deserialzed_vars = JSON.parse(compile_options['variables']['values'])
+          Puppet.warning("boltib_path: #{boltlib_path}")
+          # Use the existing environment with the requested name
+          Puppet::Pal.in_environment(compile_options['environment'],
+                                     pre_modulepath: boltlib_path,
+                                     envpath: Puppet[:environmentpath],
+                                     facts: json_deserialized_facts) do |pal|
+            # TODO: We need to decide on an appropriate config for inventory. Given we 
+            # hide this from plan authors this current iteration has only the "required"
+            # data for now. This is being discussed on https://github.com/puppetlabs/bolt/pull/1770
+            fake_config = {
+              'transport' => 'redacted',
+              'transports' => {
+                'redacted' => 'redacted'
+              }
+            }
+            bolt_inv = Bolt::ApplyInventory.new(fake_config)
+            Puppet.override(bolt_inventory: bolt_inv) do
+              Puppet.lookup(:pal_current_node).trusted_data = json_deserialized_trusted_facts
+              # This compiler has been configured with a node containing
+              # the requested environment, facts, and variables, and is used
+              # to compile a catalog in that context from the supplied AST.
+              pal.with_catalog_compiler do |compiler|
+                #TODO: Should we update these in PAL? They are the defaults in Puppet
+                Puppet[:strict] = :warning
+                Puppet[:strict_variables] = false
+                vars = Puppet::Pops::Serialization::FromDataConverter.convert(json_deserialzed_vars)
+
+                json_deserialized_facts.keys.each {|fact_name| vars.delete(fact_name)}
+                # TODO: Refactor PAL api such that we do not need to use private methods
+                pal.send(:add_variables, compiler.send(:topscope), vars)
+
+                # We have to parse the AST inside the compiler block, because it
+                # initializes the necessary type loaders for us.
+                ast = Puppet::Pops::Serialization::FromDataConverter.convert(code)
+                # Node definitions must be at the top level of the apply block.
+                # That means the apply body either a) consists of just a
+                # NodeDefinition, b) consists of a BlockExpression which may
+                # contain NodeDefinitions, or c) doesn't contain NodeDefinitions.
+                definitions = if ast.is_a?(Puppet::Pops::Model::BlockExpression)
+                                ast.statements.select { |st| st.is_a?(Puppet::Pops::Model::NodeDefinition) }
+                              elsif ast.is_a?(Puppet::Pops::Model::NodeDefinition)
+                                [ast]
+                              else
+                                []
+                              end
+                ast = Puppet::Pops::Model::Factory.PROGRAM(ast, definitions, ast.locator).model
+
+                compiler.evaluate(ast)
+                compiler.instance_variable_get(:@internal_compiler).send(:evaluate_ast_node)
+                compiler.compile_additions
+                compiler.catalog_data_hash
+              end
+            end
+          end
+        else
+          # When we do not need to load bolt we assume there are no Bolt types to resolve in
+          # AST compilation.
+          Puppet[:node_name_value] = compile_options['certname']
+
+          # Use the existing environment with the requested name
+          Puppet::Pal.in_environment(compile_options['environment'],
+                                     envpath: Puppet[:environmentpath],
+                                     facts: compile_options['facts']['values'],
+                                     variables: compile_options['variables']['values']) do |pal|
+            Puppet.lookup(:pal_current_node).trusted_data = compile_options['trusted_facts']['values']
+
+            # This compiler has been configured with a node containing
+            # the requested environment, facts, and variables, and is used
+            # to compile a catalog in that context from the supplied AST.
+            pal.with_catalog_compiler do |compiler|
+              # We have to parse the AST inside the compiler block, because it
+              # initializes the necessary type loaders for us.
+              ast = Puppet::Pops::Serialization::FromDataConverter.convert(code)
+
+              compiler.evaluate(ast)
+              compiler.compile_additions
+              compiler.catalog_data_hash
+            end
           end
         end
       end

src/ruby/puppetserver-lib/puppet/server/master.rb

@@ -91,8 +91,14 @@ def compileCatalog(request_data)
     end
   end
 
-  def compileAST(compile_options)
-    Puppet::Server::ASTCompiler.compile(convert_java_args_to_ruby(compile_options))
+  def compileAST(compile_options, boltlib_path)
+    translated_compile_options = convert_java_args_to_ruby(compile_options)
+    translated_boltlib_path = if boltlib_path.java_kind_of?(Java::JavaUtil::List)
+                                boltlib_path.to_a
+                              else
+                                Array("")
+                              end
+    Puppet::Server::ASTCompiler.compile(translated_compile_options, translated_boltlib_path)
   end
 
   def create_recorder