diff --git a/generated.tf b/generated.tf index 5914ca004d..791c581fbd 100644 --- a/generated.tf +++ b/generated.tf @@ -560,6 +560,11 @@ module "jenkins" { target_repository = "${var.target_repository}/jenkins" } +module "jitsucom-bulker" { + source = "./images/jitsucom-bulker" + target_repository = "${var.target_repository}/jitsucom-bulker" +} + module "jre" { source = "./images/jre" target_repository = "${var.target_repository}/jre" @@ -1971,6 +1976,10 @@ output "summary_jenkins" { value = module.jenkins.summary } +output "summary_jitsucom-bulker" { + value = module.jitsucom-bulker.summary +} + output "summary_jre" { value = module.jre.summary } diff --git a/images/jitsucom-bulker/README.md b/images/jitsucom-bulker/README.md new file mode 100644 index 0000000000..c2d45e389f --- /dev/null +++ b/images/jitsucom-bulker/README.md @@ -0,0 +1,48 @@ + +# jitsucom-bulker +| | | +| - | - | +| **OCI Reference** | `cgr.dev/chainguard/jitsucom-bulker` | + + +* [View Image in Chainguard Academy](https://edu.chainguard.dev/chainguard/chainguard-images/reference/jitsucom-bulker/overview/) +* [View Image Catalog](https://console.enforce.dev/images/catalog) for a full list of available tags. +* [Contact Chainguard](https://www.chainguard.dev/chainguard-images) for enterprise support, SLAs, and access to older tags.* + +--- + + + +Service for bulk-loading data to databases with automatic schema management (Redshift, Snowflake, BigQuery, ClickHouse, Postgres, MySQL) + + + +## Download this Image +The image is available on `cgr.dev`: + +``` +docker pull cgr.dev/chainguard/jitsucom-bulker:latest +``` + + + +## Usage + +```sh +helm upgrade --install jitsucom-bulker oci://registry-1.docker.io/stafftasticcharts/jitsu \ + -n jitsucom-bulker \ + --create-namespace \ + --set bulker.image.repository=cgr.dev/chainguard/jitsucom-bulker \ + --set bulker.image.tag=latest \ + --set ingest.image.repository=cgr.dev/chainguard/jitsucom-ingest \ + --set ingest.image.tag=latest \ + --set syncctl.image.repository=cgr.dev/chainguard/jitsucom-syncctl \ + --set syncctl.image.tag=latest \ + --set tokenGenerator.image.tag=1.30.0 # `:latest` tag doesn't not exist, so set to _latest_ version +``` + +* Refer to [values.yaml](https://github.com/stafftastic/jitsu-chart/blob/main/values.yaml) file for more configuration options. + +> [!WARNING] +> The Helm Chart we used in the tests is the official, stable and is used to run Jitsu in production, according to maintainer's [comment](https://github.com/jitsucom/jitsu/issues/880#issuecomment-1987928495). + diff --git a/images/jitsucom-bulker/config/main.tf b/images/jitsucom-bulker/config/main.tf new file mode 100644 index 0000000000..5cb42a4e8f --- /dev/null +++ b/images/jitsucom-bulker/config/main.tf @@ -0,0 +1,32 @@ +variable "extra_packages" { + description = "Additional packages to install." + type = list(string) + default = [] +} + +variable "main_package" { + description = "The main package to install." + type = string +} + +module "accts" { source = "../../../tflib/accts" } + +output "config" { + value = jsonencode({ + contents = { + packages = concat(["jitsucom-bulker-${var.main_package}", "jitsucom-bulker-${var.main_package}-compat"], var.extra_packages) + } + accounts = module.accts.block + entrypoint = { + command = "/app/${var.main_package}" + } + paths = [{ + path = "~/.${var.main_package}" # https://github.com/jitsucom/bulker/blob/315237cd54707f2d9395db087ec2d4b2ed1297f4/jitsubase/appbase/app_base.go#L33 + type = "directory" + uid = module.accts.uid + gid = module.accts.gid + permissions = 509 + recursive = true + }] + }) +} diff --git a/images/jitsucom-bulker/generated.tf b/images/jitsucom-bulker/generated.tf new file mode 100644 index 0000000000..e3506998fc --- /dev/null +++ b/images/jitsucom-bulker/generated.tf @@ -0,0 +1,13 @@ +# DO NOT EDIT - this file is autogenerated by tfgen + +output "summary" { + value = merge( + { + for k, v in module.latest : k => { + "ref" = v.image_ref + "config" = v.config + "tags" = ["latest"] + } + }) +} + diff --git a/images/jitsucom-bulker/main.tf b/images/jitsucom-bulker/main.tf new file mode 100644 index 0000000000..33049f6115 --- /dev/null +++ b/images/jitsucom-bulker/main.tf @@ -0,0 +1,56 @@ +terraform { + required_providers { + oci = { source = "chainguard-dev/oci" } + } +} + +variable "target_repository" { + description = "The docker repo into which the image and attestations should be published." + default = "jitsucom" +} + +locals { + components = toset(["bulker", "ingest", "syncctl"]) + + // Normally the repository is named like "jitsucom-{component}". + repositories = { + for k, v in local.components : k => "${var.target_repository}-${k}" + } +} + +module "config" { + for_each = local.components + source = "./config" + main_package = each.key +} + +module "latest" { + for_each = local.components + source = "../../tflib/publisher" + + name = basename(path.module) + target_repository = local.repositories[each.key] + config = module.config[each.key].config + build-dev = true +} + +module "test" { + source = "./tests" + digests = { for k, v in module.latest : k => v.image_ref } +} + +resource "oci_tag" "latest" { + for_each = local.components + + digest_ref = module.latest[each.key].image_ref + tag = "latest" + depends_on = [module.test] +} + +resource "oci_tag" "latest-dev" { + for_each = local.components + + digest_ref = module.latest[each.key].dev_ref + tag = "latest-dev" + depends_on = [module.test] +} diff --git a/images/jitsucom-bulker/metadata.yaml b/images/jitsucom-bulker/metadata.yaml new file mode 100644 index 0000000000..e95e972a5d --- /dev/null +++ b/images/jitsucom-bulker/metadata.yaml @@ -0,0 +1,12 @@ +name: jitsucom-bulker +image: cgr.dev/chainguard/jitsucom-bulker +logo: https://storage.googleapis.com/chainguard-academy/logos/jitsucom-bulker.svg +endoflife: "" +console_summary: "Bulker is a tool for streaming and batching large amount of semi-structured data into data warehouses." +short_description: Service for bulk-loading data to databases with automatic schema management (Redshift, Snowflake, BigQuery, ClickHouse, Postgres, MySQL) +compatibility_notes: "" +readme_file: README.md +upstream_url: https://github.com/jitsucom/bulker +keywords: + - application + - kafka diff --git a/images/jitsucom-bulker/tests/log-checker.sh b/images/jitsucom-bulker/tests/log-checker.sh new file mode 100755 index 0000000000..14e695b054 --- /dev/null +++ b/images/jitsucom-bulker/tests/log-checker.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +set -o errexit -o nounset -o errtrace -o pipefail -x + +# First argument: Kubernetes deployment name +K8S_NAME="$1" + +# Second argument: String of expected logs separated by '|' +input_logs="$2" + +# Split the input string into an array using '|' as delimiter +IFS='|' read -r -a expected_logs <<< "$input_logs" + +missing_logs=() + +RETRIES=5 +RETRY_DELAY_SECONDS=15 + +TEST_validate_container_logs() { + for ((i=1; i<=${RETRIES}; i++)); do + local logs=$(kubectl logs "deploy/${K8S_NAME}" -n "${K8S_NAMESPACE}" 2>&1) + local logs_found=true + + # Search the container logs for our expected log lines. + for log in "${expected_logs[@]}"; do + if ! echo "$logs" | grep -Fq "$log"; then + logs_found=false + fi + done + + if $logs_found; then + return 0 + elif [[ $i -lt ${RETRIES} ]]; then + echo "Some expected logs were missing. Retrying in ${RETRY_DELAY_SECONDS} seconds..." + sleep ${RETRY_DELAY_SECONDS} + fi + done + + # After all retries, record the missing logs + for log in "${expected_logs[@]}"; do + if ! echo "${logs}" | grep -Fq "$log"; then + missing_logs+=("${log}") + fi + done + + echo "FAILED: The following log lines where not found:" + printf '%s\n' "${missing_logs[@]}" + echo "Full logs:" + kubectl logs "deploy/${K8S_NAME}" -n "${K8S_NAMESPACE}" + exit 1 +} + +TEST_validate_container_logs diff --git a/images/jitsucom-bulker/tests/main.tf b/images/jitsucom-bulker/tests/main.tf new file mode 100644 index 0000000000..3a0400328b --- /dev/null +++ b/images/jitsucom-bulker/tests/main.tf @@ -0,0 +1,115 @@ +terraform { + required_providers { + oci = { source = "chainguard-dev/oci" } + imagetest = { source = "chainguard-dev/imagetest" } + } +} + +variable "digests" { + description = "The image digests to run tests over." + type = object({ + bulker = string + ingest = string + syncctl = string + }) +} + +data "oci_string" "ref" { + for_each = var.digests + input = each.value +} + +variable "namespace" { + default = "jitsucom-bulker" +} + +data "imagetest_inventory" "this" {} + +resource "imagetest_harness_k3s" "this" { + name = "jitsucom-bulker" + inventory = data.imagetest_inventory.this + + sandbox = { + mounts = [ + { + source = path.module + destination = "/tests" + } + ] + envs = { + "K8S_NAMESPACE" = var.namespace + } + } +} + +module "helm" { + source = "../../../tflib/imagetest/helm" + + name = "jitsucom-bulker" + namespace = var.namespace + chart = "oci://registry-1.docker.io/stafftasticcharts/jitsu" + + values = { + bulker = { + image = { + repository = data.oci_string.ref["bulker"].registry_repo + tag = data.oci_string.ref["bulker"].pseudo_tag + } + } + ingest = { + image = { + repository = data.oci_string.ref["ingest"].registry_repo + tag = data.oci_string.ref["ingest"].pseudo_tag + } + } + syncctl = { + image = { + repository = data.oci_string.ref["syncctl"].registry_repo + tag = data.oci_string.ref["syncctl"].pseudo_tag + } + } + tokenGenerator = { + image = { + tag = "1.30.0" # Latest is not exist. + } + } + } +} + +resource "imagetest_feature" "basic" { + name = "Basic" + description = "Basic Helm install for bank-vaults" + harness = imagetest_harness_k3s.this + + steps = [ + { + name = "Helm install" + cmd = module.helm.install_cmd + }, + { + name = "Test bulker" + cmd = <