[Fleet] Support input-level vars & templates (#83878) (#84359)

* Fix bug creating new policy on the fly * Adjust UI for input with vars but no streams * Revert "Fix bug creating new policy on the fly" This reverts commit 34f7014. * Add `compiled_input` field and compile input template, if any. Make compilation method names more generic (instead of only for streams). Add testts * Add compiled input to generated agent yaml * Don't return empty streams in agent yaml when there aren't any * Update missed assertion Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
elastic · Nov 25, 2020 · ef1c42c · ef1c42c
1 parent ad815fd
commit ef1c42c
Show file tree

Hide file tree

Showing 15 changed files with 306 additions and 83 deletions.
diff --git a/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.test.ts b/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.test.ts
@@ -100,7 +100,7 @@ describe('Fleet - storedPackagePoliciesToAgentInputs', () => {
     ).toEqual([]);
   });
 
-  it('returns agent inputs', () => {
+  it('returns agent inputs with streams', () => {
     expect(
       storedPackagePoliciesToAgentInputs([
         {
@@ -143,6 +143,46 @@ describe('Fleet - storedPackagePoliciesToAgentInputs', () => {
     ]);
   });
 
+  it('returns agent inputs without streams', () => {
+    expect(
+      storedPackagePoliciesToAgentInputs([
+        {
+          ...mockPackagePolicy,
+          package: {
+            name: 'mock-package',
+            title: 'Mock package',
+            version: '0.0.0',
+          },
+          inputs: [
+            {
+              ...mockInput,
+              compiled_input: {
+                inputVar: 'input-value',
+              },
+              streams: [],
+            },
+          ],
+        },
+      ])
+    ).toEqual([
+      {
+        id: 'some-uuid',
+        name: 'mock-package-policy',
+        revision: 1,
+        type: 'test-logs',
+        data_stream: { namespace: 'default' },
+        use_output: 'default',
+        meta: {
+          package: {
+            name: 'mock-package',
+            version: '0.0.0',
+          },
+        },
+        inputVar: 'input-value',
+      },
+    ]);
+  });
+
   it('returns agent inputs without disabled streams', () => {
     expect(
       storedPackagePoliciesToAgentInputs([

diff --git a/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.ts b/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.ts
@@ -33,20 +33,25 @@ export const storedPackagePoliciesToAgentInputs = (
           acc[key] = value;
           return acc;
         }, {} as { [k: string]: any }),
-        streams: input.streams
-          .filter((stream) => stream.enabled)
-          .map((stream) => {
-            const fullStream: FullAgentPolicyInputStream = {
-              id: stream.id,
-              data_stream: stream.data_stream,
-              ...stream.compiled_stream,
-              ...Object.entries(stream.config || {}).reduce((acc, [key, { value }]) => {
-                acc[key] = value;
-                return acc;
-              }, {} as { [k: string]: any }),
-            };
-            return fullStream;
-          }),
+        ...(input.compiled_input || {}),
+        ...(input.streams.length
+          ? {
+              streams: input.streams
+                .filter((stream) => stream.enabled)
+                .map((stream) => {
+                  const fullStream: FullAgentPolicyInputStream = {
+                    id: stream.id,
+                    data_stream: stream.data_stream,
+                    ...stream.compiled_stream,
+                    ...Object.entries(stream.config || {}).reduce((acc, [key, { value }]) => {
+                      acc[key] = value;
+                      return acc;
+                    }, {} as { [k: string]: any }),
+                  };
+                  return fullStream;
+                }),
+            }
+          : {}),
       };
 
       if (packagePolicy.package) {

diff --git a/x-pack/plugins/fleet/common/types/models/agent_policy.ts b/x-pack/plugins/fleet/common/types/models/agent_policy.ts
@@ -49,7 +49,7 @@ export interface FullAgentPolicyInput {
     package?: Pick<PackagePolicyPackage, 'name' | 'version'>;
     [key: string]: unknown;
   };
-  streams: FullAgentPolicyInputStream[];
+  streams?: FullAgentPolicyInputStream[];
   [key: string]: any;
 }
 

diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts
@@ -121,6 +121,7 @@ export interface RegistryInput {
   title: string;
   description?: string;
   vars?: RegistryVarsEntry[];
+  template_path?: string;
 }
 
 export interface RegistryStream {

diff --git a/x-pack/plugins/fleet/common/types/models/package_policy.ts b/x-pack/plugins/fleet/common/types/models/package_policy.ts
@@ -42,6 +42,7 @@ export interface NewPackagePolicyInput {
 
 export interface PackagePolicyInput extends Omit<NewPackagePolicyInput, 'streams'> {
   streams: PackagePolicyInputStream[];
+  compiled_input?: any;
 }
 
 export interface NewPackagePolicy {

diff --git a/...ctions/agent_policy/create_package_policy_page/components/package_policy_input_config.tsx b/...ctions/agent_policy/create_package_policy_page/components/package_policy_input_config.tsx
@@ -27,13 +27,15 @@ const FlexItemWithMaxWidth = styled(EuiFlexItem)`
 `;
 
 export const PackagePolicyInputConfig: React.FunctionComponent<{
+  hasInputStreams: boolean;
   packageInputVars?: RegistryVarsEntry[];
   packagePolicyInput: NewPackagePolicyInput;
   updatePackagePolicyInput: (updatedInput: Partial<NewPackagePolicyInput>) => void;
   inputVarsValidationResults: PackagePolicyConfigValidationResults;
   forceShowErrors?: boolean;
 }> = memo(
   ({
+    hasInputStreams,
     packageInputVars,
     packagePolicyInput,
     updatePackagePolicyInput,
@@ -82,15 +84,19 @@ export const PackagePolicyInputConfig: React.FunctionComponent<{
                   />
                 </h4>
               </EuiText>
-              <EuiSpacer size="s" />
-              <EuiText color="subdued" size="s">
-                <p>
-                  <FormattedMessage
-                    id="xpack.fleet.createPackagePolicy.stepConfigure.inputSettingsDescription"
-                    defaultMessage="The following settings are applicable to all inputs below."
-                  />
-                </p>
-              </EuiText>
+              {hasInputStreams ? (
+                <>
+                  <EuiSpacer size="s" />
+                  <EuiText color="subdued" size="s">
+                    <p>
+                      <FormattedMessage
+                        id="xpack.fleet.createPackagePolicy.stepConfigure.inputSettingsDescription"
+                        defaultMessage="The following settings are applicable to all inputs below."
+                      />
+                    </p>
+                  </EuiText>
+                </>
+              ) : null}
             </EuiFlexItem>
           </EuiFlexGroup>
         </EuiFlexItem>

diff --git a/...ections/agent_policy/create_package_policy_page/components/package_policy_input_panel.tsx b/...ections/agent_policy/create_package_policy_page/components/package_policy_input_panel.tsx
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import React, { useState, Fragment, memo } from 'react';
+import React, { useState, Fragment, memo, useMemo } from 'react';
 import styled from 'styled-components';
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n/react';
@@ -85,16 +85,23 @@ export const PackagePolicyInputPanel: React.FunctionComponent<{
     const errorCount = countValidationErrors(inputValidationResults);
     const hasErrors = forceShowErrors && errorCount;
 
-    const inputStreams = packageInputStreams
-      .map((packageInputStream) => {
-        return {
-          packageInputStream,
-          packagePolicyInputStream: packagePolicyInput.streams.find(
-            (stream) => stream.data_stream.dataset === packageInputStream.data_stream.dataset
-          ),
-        };
-      })
-      .filter((stream) => Boolean(stream.packagePolicyInputStream));
+    const hasInputStreams = useMemo(() => packageInputStreams.length > 0, [
+      packageInputStreams.length,
+    ]);
+    const inputStreams = useMemo(
+      () =>
+        packageInputStreams
+          .map((packageInputStream) => {
+            return {
+              packageInputStream,
+              packagePolicyInputStream: packagePolicyInput.streams.find(
+                (stream) => stream.data_stream.dataset === packageInputStream.data_stream.dataset
+              ),
+            };
+          })
+          .filter((stream) => Boolean(stream.packagePolicyInputStream)),
+      [packageInputStreams, packagePolicyInput.streams]
+    );
 
     return (
       <>
@@ -179,13 +186,14 @@ export const PackagePolicyInputPanel: React.FunctionComponent<{
         {isShowingStreams && packageInput.vars && packageInput.vars.length ? (
           <Fragment>
             <PackagePolicyInputConfig
+              hasInputStreams={hasInputStreams}
               packageInputVars={packageInput.vars}
               packagePolicyInput={packagePolicyInput}
               updatePackagePolicyInput={updatePackagePolicyInput}
               inputVarsValidationResults={{ vars: inputValidationResults.vars }}
               forceShowErrors={forceShowErrors}
             />
-            <ShortenedHorizontalRule margin="m" />
+            {hasInputStreams ? <ShortenedHorizontalRule margin="m" /> : <EuiSpacer size="l" />}
           </Fragment>
         ) : null}
 

diff --git a/x-pack/plugins/fleet/server/mocks.ts b/x-pack/plugins/fleet/server/mocks.ts
@@ -25,7 +25,7 @@ export const createAppContextStartContractMock = (): FleetAppContext => {
 
 export const createPackagePolicyServiceMock = () => {
   return {
-    assignPackageStream: jest.fn(),
+    compilePackagePolicyInputs: jest.fn(),
     buildPackagePolicyFromPackage: jest.fn(),
     bulkCreate: jest.fn(),
     create: jest.fn(),

diff --git a/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts b/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts
@@ -22,7 +22,7 @@ jest.mock('../../services/package_policy', (): {
 } => {
   return {
     packagePolicyService: {
-      assignPackageStream: jest.fn((packageInfo, dataInputs) => Promise.resolve(dataInputs)),
+      compilePackagePolicyInputs: jest.fn((packageInfo, dataInputs) => Promise.resolve(dataInputs)),
       buildPackagePolicyFromPackage: jest.fn(),
       bulkCreate: jest.fn(),
       create: jest.fn((soClient, callCluster, newData) =>

diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts
@@ -242,6 +242,7 @@ const getSavedObjectTypes = (
             enabled: { type: 'boolean' },
             vars: { type: 'flattened' },
             config: { type: 'flattened' },
+            compiled_input: { type: 'flattened' },
             streams: {
               type: 'nested',
               properties: {

diff --git a/x-pack/plugins/fleet/server/services/epm/agent/agent.test.ts b/x-pack/plugins/fleet/server/services/epm/agent/agent.test.ts
@@ -4,9 +4,9 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { createStream } from './agent';
+import { compileTemplate } from './agent';
 
-describe('createStream', () => {
+describe('compileTemplate', () => {
   it('should work', () => {
     const streamTemplate = `
 input: log
@@ -27,7 +27,7 @@ hidden_password: {{password}}
       password: { type: 'password', value: '' },
     };
 
-    const output = createStream(vars, streamTemplate);
+    const output = compileTemplate(vars, streamTemplate);
     expect(output).toEqual({
       input: 'log',
       paths: ['/usr/local/var/log/nginx/access.log'],
@@ -67,7 +67,7 @@ foo: bar
       password: { type: 'password', value: '' },
     };
 
-    const output = createStream(vars, streamTemplate);
+    const output = compileTemplate(vars, streamTemplate);
     expect(output).toEqual({
       input: 'redis/metrics',
       metricsets: ['key'],
@@ -114,7 +114,7 @@ hidden_password: {{password}}
         tags: { value: ['foo', 'bar', 'forwarded'] },
       };
 
-      const output = createStream(vars, streamTemplate);
+      const output = compileTemplate(vars, streamTemplate);
       expect(output).toEqual({
         input: 'log',
         paths: ['/usr/local/var/log/nginx/access.log'],
@@ -133,7 +133,7 @@ hidden_password: {{password}}
         tags: { value: ['foo', 'bar'] },
       };
 
-      const output = createStream(vars, streamTemplate);
+      const output = compileTemplate(vars, streamTemplate);
       expect(output).toEqual({
         input: 'log',
         paths: ['/usr/local/var/log/nginx/access.log'],
@@ -157,7 +157,7 @@ input: logs
       },
     };
 
-    const output = createStream(vars, streamTemplate);
+    const output = compileTemplate(vars, streamTemplate);
     expect(output).toEqual({
       input: 'logs',
     });

diff --git a/x-pack/plugins/fleet/server/services/epm/agent/agent.ts b/x-pack/plugins/fleet/server/services/epm/agent/agent.ts
@@ -10,27 +10,30 @@ import { PackagePolicyConfigRecord } from '../../../../common';
 
 const handlebars = Handlebars.create();
 
-export function createStream(variables: PackagePolicyConfigRecord, streamTemplate: string) {
-  const { vars, yamlValues } = buildTemplateVariables(variables, streamTemplate);
+export function compileTemplate(variables: PackagePolicyConfigRecord, templateStr: string) {
+  const { vars, yamlValues } = buildTemplateVariables(variables, templateStr);
 
-  const template = handlebars.compile(streamTemplate, { noEscape: true });
-  let stream = template(vars);
-  stream = replaceRootLevelYamlVariables(yamlValues, stream);
+  const template = handlebars.compile(templateStr, { noEscape: true });
+  let compiledTemplate = template(vars);
+  compiledTemplate = replaceRootLevelYamlVariables(yamlValues, compiledTemplate);
 
-  const yamlFromStream = safeLoad(stream, {});
+  const yamlFromCompiledTemplate = safeLoad(compiledTemplate, {});
 
   // Hack to keep empty string ('') values around in the end yaml because
   // `safeLoad` replaces empty strings with null
-  const patchedYamlFromStream = Object.entries(yamlFromStream).reduce((acc, [key, value]) => {
-    if (value === null && typeof vars[key] === 'string' && vars[key].trim() === '') {
-      acc[key] = '';
-    } else {
-      acc[key] = value;
-    }
-    return acc;
-  }, {} as { [k: string]: any });
+  const patchedYamlFromCompiledTemplate = Object.entries(yamlFromCompiledTemplate).reduce(
+    (acc, [key, value]) => {
+      if (value === null && typeof vars[key] === 'string' && vars[key].trim() === '') {
+        acc[key] = '';
+      } else {
+        acc[key] = value;
+      }
+      return acc;
+    },
+    {} as { [k: string]: any }
+  );
 
-  return replaceVariablesInYaml(yamlValues, patchedYamlFromStream);
+  return replaceVariablesInYaml(yamlValues, patchedYamlFromCompiledTemplate);
 }
 
 function isValidKey(key: string) {
@@ -54,7 +57,7 @@ function replaceVariablesInYaml(yamlVariables: { [k: string]: any }, yaml: any)
   return yaml;
 }
 
-function buildTemplateVariables(variables: PackagePolicyConfigRecord, streamTemplate: string) {
+function buildTemplateVariables(variables: PackagePolicyConfigRecord, templateStr: string) {
   const yamlValues: { [k: string]: any } = {};
   const vars = Object.entries(variables).reduce((acc, [key, recordEntry]) => {
     // support variables with . like key.patterns