Initial implementation

.cruft.json

@@ -4,10 +4,10 @@
   "checkout": "main",
   "context": {
     "cookiecutter": {
-      "name": "A high-performance observability data pipeline.",
+      "name": "Vector",
       "slug": "vector",
       "parameter_key": "vector",
-      "test_cases": "defaults",
+      "test_cases": "defaults resources",
       "add_lib": "n",
       "add_pp": "n",
       "add_golden": "y",

.github/workflows/test.yaml

@@ -33,6 +33,7 @@ jobs:
       matrix:
         instance:
           - defaults
+          - resources
     defaults:
       run:
         working-directory: ${{ env.COMPONENT_NAME }}
@@ -48,6 +49,7 @@ jobs:
       matrix:
         instance:
           - defaults
+          - resources
     defaults:
       run:
         working-directory: ${{ env.COMPONENT_NAME }}

Makefile.vars.mk

@@ -50,4 +50,4 @@ KUBENT_IMAGE    ?= ghcr.io/doitintl/kube-no-trouble:latest
 KUBENT_DOCKER   ?= $(DOCKER_CMD) $(DOCKER_ARGS) $(root_volume) --entrypoint=/app/kubent $(KUBENT_IMAGE)
 
 instance ?= defaults
-test_instances = tests/defaults.yml
+test_instances = tests/defaults.yml tests/resources.yml

README.md

@@ -1,6 +1,6 @@
-# Commodore Component: A high-performance observability data pipeline.
+# Commodore Component: Vector
 
-This is a [Commodore][commodore] Component for A high-performance observability data pipeline..
+This is a [Commodore][commodore] Component for Vector.
 
 This repository is part of Project Syn.
 For documentation on Project Syn and this component, see [syn.tools](https://syn.tools).

class/defaults.yml

@@ -2,4 +2,17 @@ parameters:
   vector:
     =_metadata:
       multi_tenant: true
-    namespace: syn-vector
+    namespace:
+      name: syn-vector
+      labels: {}
+      annotations: {}
+
+    # Cluster identifier exposed for pipelines that label or partition events per cluster.
+    tenant_id: ${cluster:name}
+
+    charts:
+      vector:
+        source: https://helm.vector.dev
+        version: 0.52.0
+
+    helm_values: {}

class/vector.yml

@@ -1,5 +1,11 @@
 parameters:
   kapitan:
+    dependencies:
+      - type: helm
+        source: ${vector:charts:vector:source}
+        version: ${vector:charts:vector:version}
+        chart_name: vector
+        output_path: ${_base_directory}/helmcharts/vector/${vector:charts:vector:version}
     compile:
       - input_paths:
           - ${_base_directory}/component/app.jsonnet
@@ -9,3 +15,12 @@ parameters:
           - ${_base_directory}/component/main.jsonnet
         input_type: jsonnet
         output_path: vector/
+      # vector Helm chart
+      - input_paths:
+          - ${_base_directory}/helmcharts/vector/${vector:charts:vector:version}
+        input_type: helm
+        helm_values: ${vector:helm_values}
+        helm_params:
+          name: vector
+          namespace: ${vector:namespace:name}
+        output_path: vector/01_vector

component/app.jsonnet

@@ -3,7 +3,7 @@ local inv = kap.inventory();
 local params = inv.parameters.vector;
 local argocd = import 'lib/argocd.libjsonnet';
 
-local app = argocd.App('vector', params.namespace);
+local app = argocd.App('vector', params.namespace.name);
 
 local appPath =
   local project = std.get(std.get(app, 'spec', {}), 'project', 'syn');

component/main.jsonnet

@@ -7,4 +7,10 @@ local params = inv.parameters.vector;
 
 // Define outputs below
 {
+  '00_namespace': kube.Namespace(params.namespace.name) {
+    metadata+: {
+      annotations+: params.namespace.annotations,
+      labels+: params.namespace.labels,
+    },
+  },
 }

docs/antora.yml

@@ -1,5 +1,5 @@
 name: vector
-title: A high-performance observability data pipeline.
+title: Vector
 version: master
 start_page: ROOT:index.adoc
 nav:

docs/modules/ROOT/pages/how-tos/talos.adoc

@@ -0,0 +1,52 @@
+= Collect Talos host logs
+
+This guide shows how to forward Talos service and kernel logs into a Vector `socket` source running as a DaemonSet `Agent` on each node.
+
+== Talos MachineConfig
+
+Forward Talos service logs to the loopback UDP socket Vector listens on:
+
+[source,yaml]
+----
+machine:
+  logging:
+    destinations:
+      - endpoint: "udp://127.0.0.1:5140/"
+        format: "json_lines"
+        extraTags:
+          cluster: <cluster-name>
+----
+
+Forward kernel ring buffer (`kmsg`) entries to the same socket:
+
+[source,yaml]
+----
+apiVersion: v1alpha1
+kind: KmsgLogConfig
+name: remote-kernel-log
+url: udp://127.0.0.1:5140/
+----
+
+== Vector helm_values
+
+The Vector Agent must share the host network namespace so the loopback datagrams arrive:
+
+[source,yaml]
+----
+parameters:
+  vector:
+    helm_values:
+      role: Agent
+      podHostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      customConfig:
+        sources:
+          talos_logs:
+            type: socket
+            mode: udp
+            address: 0.0.0.0:5140
+            decoding:
+              codec: json
+----
+
+See `tests/resources.yml` in this repo for a fuller pipeline example that ships the parsed logs to Loki.

docs/modules/ROOT/pages/index.adoc

@@ -1,5 +1,6 @@
-= A high-performance observability data pipeline.
+= Vector
 
-vector is a Commodore component to manage A high-performance observability data pipeline..
+vector is a Commodore component to manage https://vector.dev[Vector].
 
 See the xref:references/parameters.adoc[parameters] reference for further details.
+See xref:how-tos/talos.adoc[Collect Talos host logs] for an example.

docs/modules/ROOT/pages/references/parameters.adoc

@@ -2,18 +2,72 @@
 
 The parent key for all of the following parameters is `vector`.
 
+
 == `namespace`
 
+[horizontal]
+type:: dictionary
+default:: https://github.com/projectsyn/component-vector/blob/master/class/defaults.yml[See `class/defaults.yml`]
+
+Configures the namespace in which to deploy this component.
+
+=== `namespace.name`
+
 [horizontal]
 type:: string
 default:: `syn-vector`
 
-The namespace in which to deploy this component.
+The namespace name.
+
+=== `namespace.labels`
+
+[horizontal]
+type:: dictionary
+default:: `{}`
+
+Additional labels to add to the component's namespace.
+
+=== `namespace.annotations`
+
+[horizontal]
+type:: dictionary
+default:: `{}`
+
+Additional annotations to add to the component's namespace.
+
+
+== `tenant_id`
+
+[horizontal]
+type:: string
+default:: `${cluster:name}`
+
+Cluster identifier exposed for pipelines that need to label or partition events per cluster.
+Reference it from `helm_values` as `${vector:tenant_id}` to inject the same value into env vars, sink tenant fields, or transform expressions.
+
+
+== `charts`
+
+[horizontal]
+type:: dictionary
+default:: See https://github.com/projectsyn/component-vector/blob/master/class/defaults.yml[`class/defaults.yml`]
+
+The Helm chart source and version for the Vector chart.
+
+
+== `helm_values`
+
+[horizontal]
+type:: dictionary
+default:: `{}`
 
+Helm values to pass to the Vector Helm chart.
+The component does not set any defaults beyond the chart's own.
+See the https://github.com/vectordotdev/helm-charts/blob/develop/charts/vector/values.yaml[upstream `values.yaml`] for available options.
 
-== Example
+Typical fields a consumer will set:
 
-[source,yaml]
-----
-namespace: example-namespace
-----
+* `role` (`Agent`, `Aggregator`, or `Stateless-Aggregator`).
+* `podHostNetwork` (for example required when reading Talos logs from `127.0.0.1:5140`).
+* `customConfig` (the full Vector pipeline definition).
+* `env` (downward-API entries such as `VECTOR_NODE_NAME`).

docs/modules/ROOT/partials/nav.adoc

@@ -1,2 +1,4 @@
 * xref:index.adoc[Home]
 * xref:references/parameters.adoc[Parameters]
+* How-tos
+** xref:how-tos/talos.adoc[Collect Talos host logs]

tests/golden/defaults/vector/vector/00_namespace.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  annotations: {}
+  labels:
+    name: syn-vector
+  name: syn-vector

tests/golden/defaults/vector/vector/01_vector/vector/templates/configmap.yaml

@@ -0,0 +1,55 @@
+apiVersion: v1
+data:
+  aggregator.yaml: |
+    data_dir: /vector-data-dir
+    api:
+      enabled: true
+      address: 0.0.0.0:8686
+    sources:
+      datadog_agent:
+        address: 0.0.0.0:8282
+        type: datadog_agent
+      fluent:
+        address: 0.0.0.0:24224
+        type: fluent
+      internal_metrics:
+        type: internal_metrics
+      logstash:
+        address: 0.0.0.0:5044
+        type: logstash
+      splunk_hec:
+        address: 0.0.0.0:8080
+        type: splunk_hec
+      statsd:
+        address: 0.0.0.0:8125
+        mode: tcp
+        type: statsd
+      syslog:
+        address: 0.0.0.0:9000
+        mode: tcp
+        type: syslog
+      vector:
+        address: 0.0.0.0:6000
+        type: vector
+        version: "2"
+    sinks:
+      prom_exporter:
+        type: prometheus_exporter
+        inputs: [internal_metrics]
+        address: 0.0.0.0:9090
+      stdout:
+        type: console
+        inputs: [datadog_agent, fluent, logstash, splunk_hec, statsd, syslog, vector]
+        encoding:
+          codec: json
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/component: Aggregator
+    app.kubernetes.io/instance: vector
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: vector
+    app.kubernetes.io/version: 0.55.0-distroless-libc
+    helm.sh/chart: vector-0.52.0
+  name: vector
+  namespace: syn-vector

tests/golden/defaults/vector/vector/01_vector/vector/templates/service-headless.yaml

@@ -0,0 +1,45 @@
+apiVersion: v1
+kind: Service
+metadata:
+  annotations: null
+  labels:
+    app.kubernetes.io/component: Aggregator
+    app.kubernetes.io/instance: vector
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: vector
+    app.kubernetes.io/version: 0.55.0-distroless-libc
+    helm.sh/chart: vector-0.52.0
+  name: vector-headless
+  namespace: syn-vector
+spec:
+  clusterIP: None
+  ports:
+    - name: datadog-agent
+      port: 8282
+      protocol: TCP
+    - name: fluent
+      port: 24224
+      protocol: TCP
+    - name: logstash
+      port: 5044
+      protocol: TCP
+    - name: splunk-hec
+      port: 8080
+      protocol: TCP
+    - name: statsd
+      port: 8125
+      protocol: TCP
+    - name: syslog
+      port: 9000
+      protocol: TCP
+    - name: vector
+      port: 6000
+      protocol: TCP
+    - name: prom-exporter
+      port: 9090
+      protocol: TCP
+  selector:
+    app.kubernetes.io/component: Aggregator
+    app.kubernetes.io/instance: vector
+    app.kubernetes.io/name: vector
+  type: ClusterIP