Adding kustomize templates for k8s onboarding

.buildkite/scripts/steps/sync-k8s.sh

@@ -26,4 +26,7 @@ make ci-clone-kibana-repository
 cp Makefile ./kibana
 cd kibana
 echo "--- Create Kibana PR"
-make ci-create-kubernetes-templates-pull-request
+make ci-create-kubernetes-templates-pull-request
+
+echo "--- [File Update] Kustomize-Tempates"
+GENERATEKUSTOMIZE=true make ci-create-kustomize 

changelog/fragments/1715773969-kustomize-onboarding.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: enhancement
+
+# Change summary; a 80ish characters long description of the change.
+summary: kustomize templates using default manifests for k8s onboarding 
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+#pr: https://github.com/owner/repo/1234
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

deploy/kubernetes/Makefile

@@ -16,6 +16,11 @@ KUSTOMIZE=elastic-agent-kustomize
 KUSTOMIZE_DEFAULT=elastic-agent-kustomize/default
 KUSTOMIZE_KSM_AUTOSHARDING=elastic-agent-kustomize/ksm-autosharding
 
+# variable for processor for elastic-agent-standalone
+define ELASTIC_PROCESSOR
+processors:\n            - add_fields:\n                fields:\n                  onboarding_id: '%ONBOARDING_ID%'
+endef
+
 .PHONY: generate-k8s $(ALL)
 generate-k8s: $(ALL)
 
@@ -95,14 +100,15 @@ else
 endif
 
 
-## ci-create-kustomize-default : Create default kustomize folder 
+## ci-create-kustomize : Create default kustomize folder 
 .PHONY: ci-create-kustomize $(ALL)
 ci-create-kustomize: $(ALL)
 
 ifdef GENERATEKUSTOMIZE
+export ELASTIC_PROCESSOR
 $(ALL):
 	@echo "Generating $@ kustomize-default files"
-	@for f in $(shell ls $@/*.yaml); do \
+	@for f in $(shell ls $@/*.yaml | grep -v elastic-agent-standalone-daemonset-configmap); do \
 		cp -r $$f  $(KUSTOMIZE_DEFAULT)/$@/base; \
 	done
 
@@ -115,9 +121,13 @@ $(ALL):
 
 	mkdir -p $(KUSTOMIZE_KSM_AUTOSHARDING)/$@/extra/
 	sed -e "s/%VERSION%/${BEAT_VERSION}/g" -e "s/%BRANCH%/${BRANCH_VERSION}/g" -e "/name: elastic-agent-state/,+1 s/^/#/" -e "/path: \/var\/lib\/$@\/kube-system\/state/,+1 s/^/#/" $@/$@-daemonset.yaml > $(KUSTOMIZE_KSM_AUTOSHARDING)/$@/base/$@-daemonset.yaml
+	sed -e "s/%VERSION%/${BEAT_VERSION}/g" -e "s/%BRANCH%/${BRANCH_VERSION}/g" -e "/name: elastic-agent-state/,+1 s/^/#/" -e "/path: \/var\/lib\/$@\/kube-system\/state/,+1 s/^/#/" $@/$@-daemonset.yaml > $(KUSTOMIZE_KSM_AUTOSHARDING)/$@/base/$@-daemonset.yaml
+
 	sed -e "s/%VERSION%/${BEAT_VERSION}/g" -e "s/%BRANCH%/${BRANCH_VERSION}/g" -e "s/hostNetwork: true/hostNetwork: false/g" -e "s/DaemonSet/StatefulSet/g" -e "s/agent-node-datastreams/agent-ksm-datastreams/g" -e "/name: elastic-agent-state/,+1 s/^/#/" -e "/path: \/var\/lib\/$@\/kube-system\/state/,+1 s/^/#/" $@/$@-daemonset.yaml > $(KUSTOMIZE_KSM_AUTOSHARDING)/$@/extra/$@-statefulset.yaml
 
+	@echo "Generating processor $$ELASTIC_PROCESSOR"
+	sed -e "s/#<processors_placeholder>/$$ELASTIC_PROCESSOR/g"  elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml > $(KUSTOMIZE_DEFAULT)/elastic-agent-standalone/base/elastic-agent-standalone-daemonset-configmap.yaml
+
 else
 	echo "No KSM templates generated. Please run: GENERATEKUSTOMIZE=true make ci-create-kustomize "
-
-endif
+endif

deploy/kubernetes/elastic-agent-kustomize/default/README.md

@@ -0,0 +1,71 @@
+# Kustomize Templates
+
+The list below includes the official [kustomize](https://github.com/kubernetes-sigs/kustomize) templates to run them in Kubernetes:
+
+Agent Scenario | Description
+---- | ----
+[Elastic Agent managed - Default ](./elastic-agent-managed/) | Default Elastic Agent managed by Fleet setup. Kube-state-metrics (KSM) is installed automatically.
+[Elastic Agent standalone Default ](./elastic-agent-standalone/) | Default Standalone Elastic Agent setup. Kube-state-metrics (KSM) is installed automatically.
+
+## Using above templates
+
+Users can clone this repository to use the provided kustomize templates.
+
+For *Managed Elastic Agent*, please update the following variables inside main kustomization.yaml:
+
+- %FLEET_URL%: Fleet Server URL to enroll the Elastic Agent into. FLEET_URL can be found in Kibana, go to Management > Fleet > Settings
+- %ENROLLMENT_TOKEN%: Elasticsearch API key used to [enroll Elastic Agents](https://www.elastic.co/guide/en/fleet/current/fleet-enrollment-tokens.html#fleet-enrollment-tokens) in Fleet. *This should be encoded as base64 value because it will be stored as Kubernetes secret*
+
+Eg.
+
+```yaml
+secretGenerator:
+  - name: elastic-agent-creds
+    literals:
+      - enrollment_token=%ENROLLMENT_TOKEN%
+```
+
+For *Standalone Elastic Agent*, please update the following secrets inside main [kustomization.yaml](./elastic-agent-managed/kustomization.yaml):
+
+- %ES_HOST%: The Elasticsearch host to communicate with
+- %API_KEY: The API Key with access privileges to connect to Elasticsearch. See [create-api-key-standalone-agent](https://www.elastic.co/guide/en/fleet/current/grant-access-to-elasticsearch.html#create-api-key-standalone-agent). *This should be encoded as base64 value because it will be stored as Kubernetes secret*
+- %CA_TRUSTED%: The ssl.ca_trusted_fingerprint in order the elastic agent to be able to trust the certificate authority of the Elasticsearch output.
+- %ONBOARDING_ID%: A string that will be added as a new field and will denote a specific installation. *By default, this will be added to state_pod dataset.*
+
+## Remote usage of kustomize templates
+
+Users can use following commands:
+
+Managed Elastic Agent:
+
+```bash
+❯ kubectl https://github.com/elastic/elastic-agent/deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-maanged\?ref\=main | sed -e "s/JUVOUk9MTE1FTlRfVE9LRU4l/base64_ENCODED_ENROLLMENT_TOKEN/g" -e "s/%FLEET_URL%/https:\/\/localhost:9200/g" | kubectl apply -f-
+
+```
+
+Standalone Elastic Agent:
+
+```bash
+kubectl kustomize https://github.com/elastic/elastic-agent/deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-standalone\?ref\=main | sed -e "s/JUFQSV9LRVkl/<base64_encoded_APIKEY>/g" -e "s/%ES_HOST%/https:\/\/localhost:9200/g" -e "s/%CA_TRUSTED%/ca_trusted_fingerprint/g" -e "s/%ONBOARDING_ID%/12345/g" | kubectl apply -f-
+```
+
+Examples of Base64 encoded values:
+
+```bash
+❯ echo -n %API_KEY% | base64
+JUFQSV9LRVkl
+
+echo -n %ENROLLMENT_TOKEN% | base64
+JUVOUk9MTE1FTlRfVE9LRU4l
+
+❯ echo -n JUVOUk9MTE1FTlRfVE9LRU4l | base64 -D
+%ENROLLMENT_TOKEN%%
+```
+
+NOTE: `echo -n` flag needs to be provided in order to have correct base64 encoding. The echo command adds an extra line by default which needs to be avoided.
+
+## Updating kustomize templates
+
+The included kustomize templates are being produced based on [Makefile](../../Makefile) by running: `GENERATEKUSTOMIZE=true make ci-create-kustomize`
+
+The current templates are using patches as defined [here](https://github.com/elastic/elastic-agent/blob/main/deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-managed/kustomization.yaml)

deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-managed/base/elastic-agent-managed-daemonset.yaml

@@ -30,7 +30,7 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       containers:
         - name: elastic-agent
-          image: docker.elastic.co/beats/elastic-agent:8.8.1
+          image: docker.elastic.co/beats/elastic-agent:8.15.0
           env:
             # Set to 1 for enrollment into Fleet server. If not set, Elastic Agent is run in standalone mode
             - name: FLEET_ENROLL

deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-managed/base/kustomization.yaml

@@ -7,4 +7,4 @@ resources:
   - elastic-agent-managed-daemonset.yaml
   - elastic-agent-managed-role-binding.yaml
   - elastic-agent-managed-role.yaml
-  - elastic-agent-managed-service-account.yaml
+  - elastic-agent-managed-service-account.yaml

deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-managed/environmental-variables-remove.yaml

@@ -0,0 +1,23 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: elastic-agent
+  namespace: kube-system
+  labels:
+    app: elastic-agent
+spec:
+  selector:
+    matchLabels:
+      app: elastic-agent
+  template:
+      metadata:
+        labels:
+          app: elastic-agent
+      spec:
+        containers:
+          - name: elastic-agent
+            env:
+              - $patch: delete
+                name: FLEET_ENROLLMENT_TOKEN
+              - $patch: delete
+                name: FLEET_URL

deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-managed/fleet-enrollment-token-patch.yaml

@@ -0,0 +1,19 @@
+- op: add
+  path: /spec/template/spec/containers/0/env/-
+  value: 
+    name: FLEET_ENROLLMENT_TOKEN
+    valueFrom:
+      secretKeyRef:
+        name: elastic-agent-creds
+        key: enrollment_token
+
+
+- op: add
+  path: /spec/template/spec/containers/0/env/-
+  value: 
+    name: FLEET_URL
+    valueFrom:
+      configMapKeyRef:
+        name: elastic-agent-configs
+        key: host
+

deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-managed/kustomization.yaml

@@ -3,6 +3,25 @@ kind: Kustomization
 
 namespace: kube-system
 
+secretGenerator:
+  - name: elastic-agent-creds
+    literals:
+      - enrollment_token=%ENROLLMENT_TOKEN%
+
+configMapGenerator:
+- name: elastic-agent-configs
+  literals:
+  - host=%FLEET_URL%
+
 resources:
   - ./base
   - https://github.com/kubernetes/kube-state-metrics
+
+patches:
+- path: environmental-variables-remove.yaml
+- target:
+    group: apps
+    version: v1
+    kind: DaemonSet
+    name: elastic-agent
+  path: fleet-enrollment-token-patch.yaml

deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-standalone/api-key-patch.yaml

@@ -0,0 +1,27 @@
+- op: add
+  path: /spec/template/spec/containers/0/env/-
+  value: 
+    name: API_KEY
+    valueFrom:
+      secretKeyRef:
+        name: elastic-agent-creds
+        key: api_key
+
+
+- op: add
+  path: /spec/template/spec/containers/0/env/-
+  value: 
+    name: ES_HOST
+    valueFrom:
+      configMapKeyRef:
+        name: elastic-agent-configs
+        key: host
+
+- op: add
+  path: /spec/template/spec/containers/0/env/-
+  value: 
+    name: CA_TRUSTED
+    valueFrom:
+      configMapKeyRef:
+        name: elastic-agent-configs
+        key: ca_trusted

deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-standalone/base/elastic-agent-standalone-daemonset-configmap.yaml

@@ -14,8 +14,11 @@ data:
         hosts:
           - >-
             ${ES_HOST}
-        username: ${ES_USERNAME}
-        password: ${ES_PASSWORD}
+        api_key: ${API_KEY}
+        ssl.ca_trusted_fingerprint: ${CA_TRUSTED}
+        # Uncomment username/password and remove api_key if you want to use alternative authentication method
+        # username: ${ES_USERNAME}
+        # password: ${ES_PASSWORD}
     agent:
       monitoring:
         enabled: true
@@ -201,6 +204,10 @@ data:
             hosts:
               - 'kube-state-metrics:8080'
             period: 10s
+            processors:
+            - add_fields:
+                fields:
+                  onboarding_id: '%ONBOARDING_ID%'
             # Openshift:
             # if to access 'kube-state-metrics' are used third party tools, like kube-rbac-proxy or similar, that perform RBAC authorization
             # and/or tls termination, then configuration below should be considered:

deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-standalone/base/elastic-agent-standalone-daemonset.yaml

@@ -28,22 +28,27 @@ spec:
       # Uncomment if using hints feature
       #initContainers:
       #  - name: k8s-templates-downloader
-      #    image: busybox:1.28
-      #    command: ['sh']
+      #    image: docker.elastic.co/beats/elastic-agent:8.15.0
+      #    command: ['bash']
       #    args:
       #      - -c
       #      - >-
-      #        mkdir -p /etc/elastic-agent/inputs.d &&
-      #        wget -O - https://github.com/elastic/elastic-agent/archive/main.tar.gz | tar xz -C /etc/elastic-agent/inputs.d --strip=5 "elastic-agent-main/deploy/kubernetes/elastic-agent-standalone/templates.d"
+      #        mkdir -p /usr/share/elastic-agent/state/inputs.d &&
+      #        curl -sL https://github.com/elastic/elastic-agent/archive/8.15.tar.gz | tar xz -C /usr/share/elastic-agent/state/inputs.d --strip=5 "elastic-agent-8.15/deploy/kubernetes/elastic-agent-standalone/templates.d"
+      #    securityContext:
+      #      runAsUser: 0
       #    volumeMounts:
-      #      - name: external-inputs
-      #        mountPath: /etc/elastic-agent/inputs.d
+      #      - name: elastic-agent-state
+      #        mountPath: /usr/share/elastic-agent/state
       containers:
         - name: elastic-agent-standalone
-          image: docker.elastic.co/beats/elastic-agent:8.8.1
+          image: docker.elastic.co/beats/elastic-agent:8.15.0
           args: ["-c", "/etc/elastic-agent/agent.yml", "-e"]
           env:
-            # The basic authentication username used to connect to Elasticsearch
+            # The API Key with access privilleges to connect to Elasticsearch. https://www.elastic.co/guide/en/fleet/current/grant-access-to-elasticsearch.html#create-api-key-standalone-agent
+            - name: API_KEY
+              value: ""
+            # The basic authentication username used to connect to Elasticsearch. Alternative to API_KEY access.
             # This user needs the privileges required to publish events to Elasticsearch.
             - name: ES_USERNAME
               value: "elastic"
@@ -61,9 +66,7 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.name
-            - name: STATE_PATH
-              value: "/etc/elastic-agent"
-            # The following ELASTIC_NETINFO:false variable will disable the netinfo.enabled option of add-host-metadata processor. This will remove fields host.ip and host.mac.
+            # The following ELASTIC_NETINFO:false variable will disable the netinfo.enabled option of add-host-metadata processor. This will remove fields host.ip and host.mac.  
             # For more info: https://www.elastic.co/guide/en/beats/metricbeat/current/add-host-metadata.html
             - name: ELASTIC_NETINFO
               value: "false"
@@ -96,9 +99,6 @@ spec:
               mountPath: /etc/elastic-agent/agent.yml
               readOnly: true
               subPath: agent.yml
-            # Uncomment if using hints feature
-            #- name: external-inputs
-            #  mountPath: /etc/elastic-agent/inputs.d
             - name: proc
               mountPath: /hostfs/proc
               readOnly: true
@@ -129,9 +129,6 @@ spec:
           configMap:
             defaultMode: 0640
             name: agent-node-datastreams
-        # Uncomment if using hints feature
-        #- name: external-inputs
-        #  emptyDir: {}
         - name: proc
           hostPath:
             path: /proc

deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-standalone/environmental-variables-remove.yaml

@@ -0,0 +1,23 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: elastic-agent-standalone
+  namespace: kube-system
+  labels:
+    app: elastic-agent-standalone
+spec:
+  selector:
+    matchLabels:
+      app: elastic-agent-standalone
+  template:
+    metadata:
+      labels:
+        app: elastic-agent-standalone
+    spec:
+      containers:
+        - name: elastic-agent-standalone
+          env:
+          - $patch: delete
+            name: API_KEY
+          - $patch: delete
+            name: ES_HOST