add jitsucom-bulker image: bulker, ingest, syncctl (#2631)

images/jitsucom-bulker/README.md

@@ -0,0 +1,48 @@
+<!--monopod:start-->
+# jitsucom-bulker
+| | |
+| - | - |
+| **OCI Reference** | `cgr.dev/chainguard/jitsucom-bulker` |
+
+
+* [View Image in Chainguard Academy](https://edu.chainguard.dev/chainguard/chainguard-images/reference/jitsucom-bulker/overview/)
+* [View Image Catalog](https://console.enforce.dev/images/catalog) for a full list of available tags.
+* [Contact Chainguard](https://www.chainguard.dev/chainguard-images) for enterprise support, SLAs, and access to older tags.*
+
+---
+<!--monopod:end-->
+
+<!--overview:start-->
+Service for bulk-loading data to databases with automatic schema management (Redshift, Snowflake, BigQuery, ClickHouse, Postgres, MySQL)
+<!--overview:end-->
+
+<!--getting:start-->
+## Download this Image
+The image is available on `cgr.dev`:
+
+```
+docker pull cgr.dev/chainguard/jitsucom-bulker:latest
+```
+<!--getting:end-->
+
+<!--body:start-->
+## Usage
+
+```sh
+helm upgrade --install jitsucom-bulker oci://registry-1.docker.io/stafftasticcharts/jitsu \
+    -n jitsucom-bulker \
+    --create-namespace \
+    --set bulker.image.repository=cgr.dev/chainguard/jitsucom-bulker \
+    --set bulker.image.tag=latest \
+    --set ingest.image.repository=cgr.dev/chainguard/jitsucom-ingest \
+    --set ingest.image.tag=latest \
+    --set syncctl.image.repository=cgr.dev/chainguard/jitsucom-syncctl \
+    --set syncctl.image.tag=latest \
+    --set tokenGenerator.image.tag=1.30.0 # `:latest` tag doesn't not exist, so set to _latest_ version
+```
+
+* Refer to [values.yaml](https://github.com/stafftastic/jitsu-chart/blob/main/values.yaml) file for more configuration options.
+
+> [!WARNING]
+> The Helm Chart we used in the tests is the official, stable and is used to run Jitsu in production, according to maintainer's [comment](https://github.com/jitsucom/jitsu/issues/880#issuecomment-1987928495).
+<!--body:end-->

images/jitsucom-bulker/config/main.tf

@@ -0,0 +1,32 @@
+variable "extra_packages" {
+  description = "Additional packages to install."
+  type        = list(string)
+  default     = []
+}
+
+variable "main_package" {
+  description = "The main package to install."
+  type        = string
+}
+
+module "accts" { source = "../../../tflib/accts" }
+
+output "config" {
+  value = jsonencode({
+    contents = {
+      packages = concat(["jitsucom-bulker-${var.main_package}", "jitsucom-bulker-${var.main_package}-compat"], var.extra_packages)
+    }
+    accounts = module.accts.block
+    entrypoint = {
+      command = "/app/${var.main_package}"
+    }
+    paths = [{
+      path        = "~/.${var.main_package}" # https://github.com/jitsucom/bulker/blob/315237cd54707f2d9395db087ec2d4b2ed1297f4/jitsubase/appbase/app_base.go#L33
+      type        = "directory"
+      uid         = module.accts.uid
+      gid         = module.accts.gid
+      permissions = 509
+      recursive   = true
+    }]
+  })
+}

images/jitsucom-bulker/main.tf

@@ -0,0 +1,56 @@
+terraform {
+  required_providers {
+    oci = { source = "chainguard-dev/oci" }
+  }
+}
+
+variable "target_repository" {
+  description = "The docker repo into which the image and attestations should be published."
+  default     = "jitsucom"
+}
+
+locals {
+  components = toset(["bulker", "ingest", "syncctl"])
+
+  // Normally the repository is named like "jitsucom-{component}".
+  repositories = {
+    for k, v in local.components : k => "${var.target_repository}-${k}"
+  }
+}
+
+module "config" {
+  for_each     = local.components
+  source       = "./config"
+  main_package = each.key
+}
+
+module "latest" {
+  for_each = local.components
+  source   = "../../tflib/publisher"
+
+  name              = basename(path.module)
+  target_repository = local.repositories[each.key]
+  config            = module.config[each.key].config
+  build-dev         = true
+}
+
+module "test" {
+  source  = "./tests"
+  digests = { for k, v in module.latest : k => v.image_ref }
+}
+
+resource "oci_tag" "latest" {
+  for_each = local.components
+
+  digest_ref = module.latest[each.key].image_ref
+  tag        = "latest"
+  depends_on = [module.test]
+}
+
+resource "oci_tag" "latest-dev" {
+  for_each = local.components
+
+  digest_ref = module.latest[each.key].dev_ref
+  tag        = "latest-dev"
+  depends_on = [module.test]
+}

images/jitsucom-bulker/metadata.yaml

@@ -0,0 +1,12 @@
+name: jitsucom-bulker
+image: cgr.dev/chainguard/jitsucom-bulker
+logo: https://storage.googleapis.com/chainguard-academy/logos/jitsucom-bulker.svg
+endoflife: ""
+console_summary: "Bulker is a tool for streaming and batching large amount of semi-structured data into data warehouses."
+short_description: Service for bulk-loading data to databases with automatic schema management (Redshift, Snowflake, BigQuery, ClickHouse, Postgres, MySQL)
+compatibility_notes: ""
+readme_file: README.md
+upstream_url: https://github.com/jitsucom/bulker
+keywords:
+  - application
+  - kafka

images/jitsucom-bulker/tests/log-checker.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+set -o errexit -o nounset -o errtrace -o pipefail -x
+
+# First argument: Kubernetes deployment name
+K8S_NAME="$1"
+
+# Second argument: String of expected logs separated by '|'
+input_logs="$2"
+
+# Split the input string into an array using '|' as delimiter
+IFS='|' read -r -a expected_logs <<< "$input_logs"
+
+missing_logs=()
+
+RETRIES=5
+RETRY_DELAY_SECONDS=15
+
+TEST_validate_container_logs() {
+  for ((i=1; i<=${RETRIES}; i++)); do
+    local logs=$(kubectl logs "deploy/${K8S_NAME}" -n "${K8S_NAMESPACE}" 2>&1)
+    local logs_found=true
+
+    # Search the container logs for our expected log lines.
+    for log in "${expected_logs[@]}"; do
+      if ! echo "$logs" | grep -Fq "$log"; then
+        logs_found=false
+      fi
+    done
+
+    if $logs_found; then
+      return 0
+    elif [[ $i -lt ${RETRIES} ]]; then
+      echo "Some expected logs were missing. Retrying in ${RETRY_DELAY_SECONDS} seconds..."
+      sleep ${RETRY_DELAY_SECONDS}
+    fi
+  done
+
+  # After all retries, record the missing logs
+  for log in "${expected_logs[@]}"; do
+    if ! echo "${logs}" | grep -Fq "$log"; then
+      missing_logs+=("${log}")
+    fi
+  done
+
+  echo "FAILED: The following log lines where not found:"
+  printf '%s\n' "${missing_logs[@]}"
+  echo "Full logs:"
+  kubectl logs "deploy/${K8S_NAME}" -n "${K8S_NAMESPACE}"
+  exit 1
+}
+
+TEST_validate_container_logs

images/jitsucom-bulker/tests/main.tf

@@ -0,0 +1,115 @@
+terraform {
+  required_providers {
+    oci       = { source = "chainguard-dev/oci" }
+    imagetest = { source = "chainguard-dev/imagetest" }
+  }
+}
+
+variable "digests" {
+  description = "The image digests to run tests over."
+  type = object({
+    bulker  = string
+    ingest  = string
+    syncctl = string
+  })
+}
+
+data "oci_string" "ref" {
+  for_each = var.digests
+  input    = each.value
+}
+
+variable "namespace" {
+  default = "jitsucom-bulker"
+}
+
+data "imagetest_inventory" "this" {}
+
+resource "imagetest_harness_k3s" "this" {
+  name      = "jitsucom-bulker"
+  inventory = data.imagetest_inventory.this
+
+  sandbox = {
+    mounts = [
+      {
+        source      = path.module
+        destination = "/tests"
+      }
+    ]
+    envs = {
+      "K8S_NAMESPACE" = var.namespace
+    }
+  }
+}
+
+module "helm" {
+  source = "../../../tflib/imagetest/helm"
+
+  name      = "jitsucom-bulker"
+  namespace = var.namespace
+  chart     = "oci://registry-1.docker.io/stafftasticcharts/jitsu"
+
+  values = {
+    bulker = {
+      image = {
+        repository = data.oci_string.ref["bulker"].registry_repo
+        tag        = data.oci_string.ref["bulker"].pseudo_tag
+      }
+    }
+    ingest = {
+      image = {
+        repository = data.oci_string.ref["ingest"].registry_repo
+        tag        = data.oci_string.ref["ingest"].pseudo_tag
+      }
+    }
+    syncctl = {
+      image = {
+        repository = data.oci_string.ref["syncctl"].registry_repo
+        tag        = data.oci_string.ref["syncctl"].pseudo_tag
+      }
+    }
+    tokenGenerator = {
+      image = {
+        tag = "1.30.0" # Latest is not exist.
+      }
+    }
+  }
+}
+
+resource "imagetest_feature" "basic" {
+  name        = "Basic"
+  description = "Basic Helm install for bank-vaults"
+  harness     = imagetest_harness_k3s.this
+
+  steps = [
+    {
+      name = "Helm install"
+      cmd  = module.helm.install_cmd
+    },
+    {
+      name  = "Test bulker"
+      cmd   = <<EOF
+ /tests/log-checker.sh "jitsucom-bulker-bulker" "[metrics_server] Starting metrics server|Starting http server|[topic-manager] Created topic: destination-messages"
+       EOF
+      retry = { attempts = 3, delay = "5s" }
+    },
+    {
+      name  = "Test ingest"
+      cmd   = <<EOF
+ /tests/log-checker.sh "jitsucom-bulker-ingest" "[metrics_server] Starting metrics server|Starting http server|[router] Data hosts"
+       EOF
+      retry = { attempts = 3, delay = "5s" }
+    },
+    {
+      name  = "Test syncctl"
+      cmd   = <<EOF
+ /tests/log-checker.sh "jitsucom-bulker-syncctl" "Starting http server|Generated instance id"
+       EOF
+      retry = { attempts = 3, delay = "5s" }
+    },
+  ]
+
+  labels = {
+    type = "k8s"
+  }
+}