diff --git a/generated.tf b/generated.tf index c00c21c9b9..84c0ddda65 100644 --- a/generated.tf +++ b/generated.tf @@ -390,6 +390,11 @@ module "fluent-bit" { target_repository = "${var.target_repository}/fluent-bit" } +module "fluent-bit-plugin-loki" { + source = "./images/fluent-bit-plugin-loki" + target_repository = "${var.target_repository}/fluent-bit-plugin-loki" +} + module "fluentd" { source = "./images/fluentd" target_repository = "${var.target_repository}/fluentd" @@ -1870,6 +1875,10 @@ output "summary_fluent-bit" { value = module.fluent-bit.summary } +output "summary_fluent-bit-plugin-loki" { + value = module.fluent-bit-plugin-loki.summary +} + output "summary_fluentd" { value = module.fluentd.summary } diff --git a/images/fluent-bit-plugin-loki/README.md b/images/fluent-bit-plugin-loki/README.md new file mode 100644 index 0000000000..c4ed1fd2dc --- /dev/null +++ b/images/fluent-bit-plugin-loki/README.md @@ -0,0 +1,53 @@ + +# fluent-bit-plugin-loki +| | | +| - | - | +| **OCI Reference** | `cgr.dev/chainguard/fluent-bit-plugin-loki` | + + +* [View Image in Chainguard Academy](https://edu.chainguard.dev/chainguard/chainguard-images/reference/fluent-bit-plugin-loki/overview/) +* [View Image Catalog](https://console.enforce.dev/images/catalog) for a full list of available tags. +* [Contact Chainguard](https://www.chainguard.dev/chainguard-images) for enterprise support, SLAs, and access to older tags.* + +--- + + + +The Fluent Bit Loki plugin allows you to send your log or events to a Loki service. + + + +## Download this Image +The image is available on `cgr.dev`: + +``` +docker pull cgr.dev/chainguard/fluent-bit-plugin-loki:latest +``` + + + +## Usage + +### Docker + +You can run a Fluent Bit container with the Loki output plugin pre-installed using the following command: + +```bash +docker run -v /var/log:/var/log \ + -e LOG_PATH="/var/log/*.log" -e LOKI_URL="http://localhost:3100/loki/api/v1/push" \ + cgr.dev/chainguard/fluent-bit-plugin-loki:latest +``` +The [GitHub repository](https://github.com/grafana/loki/tree/main/clients/cmd/fluent-bit) has more details on running the fluent-bit plugin. +**Note:** The default loki plugin config can be found at `/fluent-bit/etc/fluent-bit-loki.conf`. + +### Helm +The plugin can also be installed using the fluent-bit helm chart by configuring the image values: + +```yaml +image: + repository: cgr.dev/chainguard/fluent-bit-plugin-loki + tag: latest +``` +For detailed instructions on setting up the Fluent Bit Loki plugin, refer to the [Loki documentation](https://grafana.com/docs/loki/latest/send-data/fluentbit/). + + diff --git a/images/fluent-bit-plugin-loki/config/main.tf b/images/fluent-bit-plugin-loki/config/main.tf new file mode 100644 index 0000000000..b524d76793 --- /dev/null +++ b/images/fluent-bit-plugin-loki/config/main.tf @@ -0,0 +1,23 @@ +terraform { + required_providers { + apko = { source = "chainguard-dev/apko" } + } +} + +variable "extra_packages" { + description = "The additional packages to install" + default = [ + "fluent-bit", + "fluent-bit-compat", + "fluent-bit-plugin-loki" + ] +} + +data "apko_config" "this" { + config_contents = file("${path.module}/template.apko.yaml") + extra_packages = var.extra_packages +} + +output "config" { + value = jsonencode(data.apko_config.this.config) +} diff --git a/images/fluent-bit-plugin-loki/config/template.apko.yaml b/images/fluent-bit-plugin-loki/config/template.apko.yaml new file mode 100644 index 0000000000..42c66dc760 --- /dev/null +++ b/images/fluent-bit-plugin-loki/config/template.apko.yaml @@ -0,0 +1,15 @@ +contents: + packages: [] + +accounts: + groups: + - groupname: nonroot + gid: 65532 + users: + - username: nonroot + uid: 65532 + gid: 65532 + run-as: 65532 + +entrypoint: + command: "fluent-bit -e /fluent-bit/bin/out_grafana_loki.so -c /fluent-bit/etc/fluent-bit-loki.conf" diff --git a/images/fluent-bit-plugin-loki/generated.tf b/images/fluent-bit-plugin-loki/generated.tf new file mode 100644 index 0000000000..be0fca74cf --- /dev/null +++ b/images/fluent-bit-plugin-loki/generated.tf @@ -0,0 +1,13 @@ +# DO NOT EDIT - this file is autogenerated by tfgen + +output "summary" { + value = merge( + { + basename(path.module) = { + "ref" = module.fluent-bit-plugin-loki.image_ref + "config" = module.fluent-bit-plugin-loki.config + "tags" = ["latest"] + } + }) +} + diff --git a/images/fluent-bit-plugin-loki/main.tf b/images/fluent-bit-plugin-loki/main.tf new file mode 100644 index 0000000000..a6c882e394 --- /dev/null +++ b/images/fluent-bit-plugin-loki/main.tf @@ -0,0 +1,39 @@ +terraform { + required_providers { + oci = { source = "chainguard-dev/oci" } + } +} + +variable "target_repository" { + description = "The docker repo into which the image and attestations should be published." +} + +module "config" { source = "./config" } + +module "fluent-bit-plugin-loki" { + source = "../../tflib/publisher" + name = basename(path.module) + target_repository = var.target_repository + config = module.config.config + + build-dev = true + +} + +module "test" { + source = "./tests" + digest = module.fluent-bit-plugin-loki.image_ref +} + +resource "oci_tag" "latest" { + depends_on = [module.test] + digest_ref = module.fluent-bit-plugin-loki.image_ref + tag = "latest" +} + +resource "oci_tag" "latest-dev" { + depends_on = [module.test] + digest_ref = module.fluent-bit-plugin-loki.dev_ref + tag = "latest-dev" +} + diff --git a/images/fluent-bit-plugin-loki/metadata.yaml b/images/fluent-bit-plugin-loki/metadata.yaml new file mode 100644 index 0000000000..0318e7caa2 --- /dev/null +++ b/images/fluent-bit-plugin-loki/metadata.yaml @@ -0,0 +1,12 @@ +name: fluent-bit-plugin-loki +image: cgr.dev/chainguard/fluent-bit-plugin-loki +logo: https://storage.googleapis.com/chainguard-academy/logos/fluent-bit-plugin-loki.svg +endoflife: "" +console_summary: "" +short_description: "The Fluent Bit Loki plugin allows you to send your log or events to a Loki service." +compatibility_notes: "" +readme_file: README.md +upstream_url: "https://github.com/grafana/loki" +keywords: + - analytics + - application diff --git a/images/fluent-bit-plugin-loki/tests/main.tf b/images/fluent-bit-plugin-loki/tests/main.tf new file mode 100644 index 0000000000..9e8b594f8c --- /dev/null +++ b/images/fluent-bit-plugin-loki/tests/main.tf @@ -0,0 +1,196 @@ +terraform { + required_providers { + oci = { source = "chainguard-dev/oci" } + imagetest = { source = "chainguard-dev/imagetest" } + } +} + +variable "digest" { + description = "The image digest to run tests over." +} + +data "oci_string" "ref" { input = var.digest } + +data "imagetest_inventory" "this" {} + + +resource "imagetest_harness_k3s" "this" { + name = "fluent-bit-plugin-loki" + inventory = data.imagetest_inventory.this + + sandbox = { + mounts = [ + { + source = path.module + destination = "/tests" + } + ], + } +} + +variable "loki_url" { + type = string + default = "http://loki-gateway.loki.svc.cluster.local/loki/api/v1/push" +} + +module "helm-fluent-bit" { + source = "../../../tflib/imagetest/helm" + + name = "fluent-bit" + namespace = "fluent-bit" + repo = "https://fluent.github.io/helm-charts" + chart = "fluent-bit" + + values = { + image = { + repository = data.oci_string.ref.registry_repo + tag = data.oci_string.ref.pseudo_tag + }, + args = [ + "-e", + "/fluent-bit/bin/out_grafana_loki.so", + "--workdir=/fluent-bit/etc", + "--config=/fluent-bit/etc/conf/fluent-bit.conf" + ], + env = [{ + name = "FLUENT_LOKI_URL", + value = var.loki_url + }], + daemonSetVolumes = [ + { + name = "varlog", + hostPath = { + path = "/var/log" + } + }, + { + name = "varlibdockercontainers", + hostPath = { + path = "/var/lib/docker/containers" + } + } + ], + daemonSetVolumeMounts = [ + { + name = "varlog", + mountPath = "/var/log" + }, + { + name = "varlibdockercontainers", + mountPath = "/var/lib/docker/containers", + readOnly = true + } + ], + config = { + inputs = "[INPUT]\n Name tail\n Tag kube.*\n Path /var/log/containers/*.log\n multiline.parser docker, cri\n Mem_Buf_Limit 5MB\n Skip_Long_Lines On\n", + outputs = "[Output]\n Name grafana-loki\n Match kube.*\n Url ${var.loki_url}\n Labels {job=\"fluent-bit\"}\n LabelKeys level,app\n BatchWait 1\n BatchSize 1001024\n LineFormat json\n LogLevel info\n AutoKubernetesLabels true\n" + }, + podSecurityContext = { + fsGroup = 2000 + }, + securityContext = { + allowPrivilegeEscalation = false, + readOnlyRootFilesystem = false, + runAsUser = 0, + capabilities = { + drop = ["ALL"], + add = ["DAC_READ_SEARCH"] + } + } + } +} + +module "helm-loki" { + source = "../../../tflib/imagetest/helm" + + name = "loki" + namespace = "loki" + repo = "https://grafana.github.io/helm-charts" + chart = "loki" + values = { + deploymentMode = "SingleBinary", + loki = { + auth_enabled = false, + commonConfig = { + ring = { + instance_addr = "127.0.0.1", + kvstore = { + store = "inmemory" + } + }, + replication_factor = 1, + path_prefix = "/tmp/loki" + }, + storage = { + type = "filesystem" + }, + schemaConfig = { + configs = [ + { + from = "2024-01-01", + store = "tsdb", + index = { + prefix = "loki_index_", + period = "24h" + }, + object_store = "filesystem", + schema = "v13" + } + ] + }, + storage_config = { + filesystem = { + directory = "/tmp/loki/chunks" + } + } + }, + singleBinary = { + replicas = 1 + }, + read = { + replicas = 0 + }, + backend = { + replicas = 0 + }, + write = { + replicas = 0 + }, + chunksCache = { + enabled = false + }, + resultsCache = { + enabled = false + } + } +} + + +resource "imagetest_feature" "basic" { + harness = imagetest_harness_k3s.this + name = "basic" + description = "Functionality test for fluent-bit-plugin-loki installed in a Kubernetes cluster" + + steps = [ + { + name = "Install loki" + cmd = module.helm-loki.install_cmd + }, + + { + name = "Install fluent-bit" + cmd = module.helm-fluent-bit.install_cmd + }, + { + name = "Query loki for fluent-bit logs" + workdir = "/tests" + cmd = <