From 09d571e1cba48b4f639a80b36d5962ee1536524a Mon Sep 17 00:00:00 2001 From: Aarthy Adityan Date: Wed, 19 Feb 2025 23:41:18 -0500 Subject: [PATCH 1/6] fix(docker): empty version env var in image --- deploy/docker-bake.hcl | 5 +++++ deploy/testgen.dockerfile | 11 +++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/deploy/docker-bake.hcl b/deploy/docker-bake.hcl index da0e9c02..808afb01 100644 --- a/deploy/docker-bake.hcl +++ b/deploy/docker-bake.hcl @@ -1,11 +1,15 @@ variable "TESTGEN_LABELS" {} variable "TESTGEN_BASE_LABEL" {} variable "TESTGEN_VERSION" {} +variable "TESTGEN_DOCKER_HUB_REPO" { + "default": "datakitchen/dataops-testgen" +} target "testgen-release" { args = { TESTGEN_VERSION = "${TESTGEN_VERSION}" TESTGEN_BASE_LABEL = "${TESTGEN_BASE_LABEL}" + TESTGEN_DOCKER_HUB_REPO = "${TESTGEN_DOCKER_HUB_REPO}" } context = "." dockerfile = "deploy/testgen.dockerfile" @@ -17,6 +21,7 @@ target "testgen-qa" { args = { TESTGEN_VERSION = "${TESTGEN_VERSION}" TESTGEN_BASE_LABEL = "${TESTGEN_BASE_LABEL}" + TESTGEN_DOCKER_HUB_REPO = "${TESTGEN_DOCKER_HUB_REPO}" } context = "." dockerfile = "deploy/testgen.dockerfile" diff --git a/deploy/testgen.dockerfile b/deploy/testgen.dockerfile index b488e7d8..054de771 100644 --- a/deploy/testgen.dockerfile +++ b/deploy/testgen.dockerfile @@ -1,7 +1,6 @@ -ARG TESTGEN_BASE_LABEL -ARG TESTGEN_VERSION +ARG TESTGEN_BASE_LABEL=v1 -FROM datakitchen/dataops-testgen-base:${TESTGEN_BASE_LABEL} as build-image +FROM datakitchen/dataops-testgen-base:${TESTGEN_BASE_LABEL} AS build-image # Now install everything COPY . /tmp/dk/ @@ -9,6 +8,10 @@ RUN python3 -m pip install --prefix=/dk /tmp/dk FROM python:3.12.7-alpine3.20 AS release-image +# Args have to be set in current build stage: https://github.com/moby/moby/issues/37345 +ARG TESTGEN_VERSION +ARG TESTGEN_DOCKER_HUB_REPO + RUN addgroup -S testgen && adduser -S testgen -G testgen COPY --from=build-image --chown=testgen:testgen /dk/ /dk @@ -25,7 +28,7 @@ ENV PATH="$PATH:/dk/bin:/opt/mssql-tools/bin/" ENV TESTGEN_VERSION=${TESTGEN_VERSION} ENV TG_RELEASE_CHECK=docker -ENV TESTGEN_DOCKER_HUB_REPO=datakitchen/dataops-testgen-enterprise +ENV TESTGEN_DOCKER_HUB_REPO=${TESTGEN_DOCKER_HUB_REPO} ENV STREAMLIT_SERVER_MAX_UPLOAD_SIZE=200 RUN mkdir /var/lib/testgen && chown testgen:testgen /var/lib/testgen From 82efe24b83d11a0858aedab426f7d87f6a3fd81e Mon Sep 17 00:00:00 2001 From: Aarthy Adityan Date: Thu, 20 Feb 2025 00:19:56 -0500 Subject: [PATCH 2/6] fix(scores): hide empty scorecards and make breakdown filter conditional --- .../frontend/js/components/score_breakdown.js | 15 ++++++++++----- testgen/ui/views/quality_dashboard.py | 2 +- testgen/ui/views/score_details.py | 4 +++- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/testgen/ui/components/frontend/js/components/score_breakdown.js b/testgen/ui/components/frontend/js/components/score_breakdown.js index 8edb10e4..28c329e5 100644 --- a/testgen/ui/components/frontend/js/components/score_breakdown.js +++ b/testgen/ui/components/frontend/js/components/score_breakdown.js @@ -28,11 +28,16 @@ const ScoreBreakdown = (score, breakdown, category, scoreType, onViewDetails) => }, span('for'), () => { + const scoreValue = getValue(score); const selectedScoreType = getValue(scoreType); + const scoreTypeOptions = ['score', 'cde_score'].filter((s) => scoreValue[s]) + if (!scoreTypeOptions.length) { + scoreTypeOptions.push('score'); + } return Select({ label: '', value: selectedScoreType, - options: ['score', 'cde_score'].map((s) => ({ label: SCORE_TYPE_LABEL[s], value: s, selected: s === scoreType })), + options: scoreTypeOptions.map((s) => ({ label: SCORE_TYPE_LABEL[s], value: s, selected: s === scoreType })), onChange: (value) => emitEvent('ScoreTypeChanged', { payload: value }), }); }, @@ -41,7 +46,7 @@ const ScoreBreakdown = (score, breakdown, category, scoreType, onViewDetails) => ), () => div( { class: 'table-header breakdown-columns flex-row' }, - getValue(breakdown)?.columns?.map(column => span({ + getValue(breakdown)?.columns?.map(column => span({ style: `flex: ${BREAKDOWN_COLUMNS_SIZES[column]};` }, getReadableColumn(column, getValue(scoreType)), )), @@ -64,7 +69,7 @@ const ScoreBreakdown = (score, breakdown, category, scoreType, onViewDetails) => /** * Translate the column names for the table. - * + * * @param {Array} columns * @param {('table_name' | 'column_name' | 'semantic_data_type' | 'dq_dimension')} category * @param {('score' | 'cde_score')} scoreType @@ -82,7 +87,7 @@ function getReadableColumn(column, scoreType) { } /** - * + * * @param {object} row * @param {string} column * @returns {} @@ -205,4 +210,4 @@ stylesheet.replace(` } `); -export { ScoreBreakdown }; \ No newline at end of file +export { ScoreBreakdown }; diff --git a/testgen/ui/views/quality_dashboard.py b/testgen/ui/views/quality_dashboard.py index d3fd6a82..6510ff61 100644 --- a/testgen/ui/views/quality_dashboard.py +++ b/testgen/ui/views/quality_dashboard.py @@ -31,7 +31,7 @@ def render(self, *, project_code: str, **_kwargs) -> None: "table_groups_count": int(project_summary["table_groups_ct"]), "profiling_runs_count": int(project_summary["profiling_runs_ct"]), }, - "scores": [format_score_card(score) for score in get_all_score_cards(project_code)], + "scores": [format_score_card(score) for score in get_all_score_cards(project_code) if score["score"] or score["cde_score"] or score["categories"]], }, on_change_handlers={ "RefreshData": refresh_data, diff --git a/testgen/ui/views/score_details.py b/testgen/ui/views/score_details.py index 8ba4a34d..d91bc37c 100644 --- a/testgen/ui/views/score_details.py +++ b/testgen/ui/views/score_details.py @@ -29,7 +29,7 @@ def render( *, definition_id: str, category: str = "table_name", - score_type: str = "score", + score_type: str | None = None, drilldown: str | None = None, **_kwargs ): @@ -57,6 +57,8 @@ def render( with st.spinner(text="Loading data ..."): user_can_edit = authentication_service.current_user_has_edit_role() score_card = format_score_card(score_definition.as_score_card()) + if not score_type: + score_type = "cde_score" if score_card["cde_score"] and not score_card["score"] else "score" if not drilldown: score_breakdown = ScoreDefinitionBreakdownItem.filter( definition_id=definition_id, From caae9e63bfc82ebcb956be53d8e9f940dcb33a39 Mon Sep 17 00:00:00 2001 From: Ricardo Boni Date: Wed, 26 Feb 2025 16:35:19 -0500 Subject: [PATCH 3/6] feat: Adding Helm charts --- .gitignore | 1 + deploy/charts/README.md | 134 ++++++++++++++++++ deploy/charts/testgen-app/Chart.yaml | 24 ++++ .../testgen-app/templates/_environment.yaml | 40 ++++++ .../charts/testgen-app/templates/_helpers.tpl | 62 ++++++++ .../testgen-app/templates/deployment.yaml | 58 ++++++++ deploy/charts/testgen-app/templates/job.yaml | 57 ++++++++ .../charts/testgen-app/templates/secrets.yaml | 14 ++ .../charts/testgen-app/templates/service.yaml | 15 ++ .../testgen-app/templates/serviceaccount.yaml | 13 ++ deploy/charts/testgen-app/values.yaml | 81 +++++++++++ deploy/charts/testgen-services/Chart.lock | 6 + deploy/charts/testgen-services/Chart.yaml | 29 ++++ deploy/charts/testgen-services/values.yaml | 7 + deploy/charts_values/values-app-dev.yaml | 13 ++ 15 files changed, 554 insertions(+) create mode 100644 deploy/charts/README.md create mode 100644 deploy/charts/testgen-app/Chart.yaml create mode 100644 deploy/charts/testgen-app/templates/_environment.yaml create mode 100644 deploy/charts/testgen-app/templates/_helpers.tpl create mode 100644 deploy/charts/testgen-app/templates/deployment.yaml create mode 100644 deploy/charts/testgen-app/templates/job.yaml create mode 100644 deploy/charts/testgen-app/templates/secrets.yaml create mode 100644 deploy/charts/testgen-app/templates/service.yaml create mode 100644 deploy/charts/testgen-app/templates/serviceaccount.yaml create mode 100644 deploy/charts/testgen-app/values.yaml create mode 100644 deploy/charts/testgen-services/Chart.lock create mode 100644 deploy/charts/testgen-services/Chart.yaml create mode 100644 deploy/charts/testgen-services/values.yaml create mode 100644 deploy/charts_values/values-app-dev.yaml diff --git a/.gitignore b/.gitignore index bded8361..139decb6 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ share/python-wheels/ .installed.cfg *.egg MANIFEST +deploy/charts/*/charts # Installer logs pip-log.txt diff --git a/deploy/charts/README.md b/deploy/charts/README.md new file mode 100644 index 00000000..43237d22 --- /dev/null +++ b/deploy/charts/README.md @@ -0,0 +1,134 @@ +# Overview + +These are the Helm charts to install DataOps TestGen on a Kubernetes +environment. This README includes instructions for installing TestGen on a +minikube instance, including the application database engine, added by the +`testgen-services` charts. The application itself is installed by the +`testgen-app` charts. When installing on cloud, it's likely that the +application database will be provisioned by a cloud provider, and therefore +installing the `testgen-services` charts is not necessary. + +# Preparing the Environment + +Here are the instructions to create a local minikube cluster where DataOps +TestGen and its services will be installed. When deploying to a cloud / +production environment, this step should be adjusted accordingly. + +Assuming that minikube is installed, the following command will create a +cluster named dk-testgen and configure the local kubectl tool to issue commands +against the cluster. When running helm commands against an existing cluster, +it's important to make sure that the kubectl tool is pointing to the correct +profile. + +```shell +minikube start -p dk-testgen --namespace datakitchen +``` + +# Configuration + +Whenever a helm command is used to generate manifests from templates +(`install`, `upgrade`), it will rely on pre-configured values to do so. Helm +provides a variety of ways to define these values. You can find the complete +configuration set that is also used as a default in each charts’ folder, in the +values.yaml file. + +No additional configuration is recommended for the `testgen-services` charts. + +The `testgen-app` charts require some configuration to be fine tuned. The best +way to do that is to save it in a values file, so that the same configuration +set can be easily used on the first install and future upgrades. + +The following configuration is recommended for experimental installations, but +you're free to adjust it for your needs. The next installation steps assumes +that a file named tg-values.yaml exists with this configuration. + +```yaml +testgen: + + # Password that will be assigned to the 'admin' user during the database preparation + uiPassword: "admin" + + # Whether to trust the target database certificates. + trustTargetDatabaseCertificate: true + + # Whether to run the SSL certificate verifications when connecting to DataOps Observability + observabilityVerifySsl: false + +image: + + # DataOps TestGen version to be installed / upgraded + tag: v3.0 +``` + +# Installing + +The following command will install the required services for DataOps TestGen, +which currently is the Postgres database. This step is not needed when the +database is provisioned in a cloud environment. + +```shell +helm install --create-namespace --wait testgen-services deploy/charts/testgen-services/ +``` + +The following command will install the application. As part of the process, a +one-time database configuration will be automatically performed. Note that the +custom configuration values are being used. + +```shell +helm install --wait -f tg-values.yaml testgen-app deploy/charts/testgen-app/ +``` + +At this point you should have a fully functional instance of DataOps TestGen +installed. If you're looking to secure the UI password, you can delete it from +the values file. + +# Accessing the Application + +In order to use your TestGen instance, you have to use your web browser to +login to it. Your username will be admin and the password the one you +configured earlier. + +It may be needed that you forward the application HTTP port in order to be able +to access it. This is especially necessary if you are using the docker driver +for minikube installed into a mac OS. The following command forwards the TesGen +“http” port to the host's 8501 port. + +```shell +kubectl port-forward svc/testgen-app 8501:http +``` + +# Upgrading + +When you already have a running instance of TestGen and want to upgrade it to a +newer version, you can do so without losing any data by running the +`upgrade-system-version` TestGen command from the target version's image. The +helm charts will do this automatically when you upgrade. You should edit your +values file to update the image tag value to the desired version, and then run +the following command. + +```shell +helm upgrade --wait -f tg-values.yaml testgen-app deploy/charts/testgen-app/ +``` + +# Uninstalling + +The following commands will uninstall almost everything that was created +through the helm templates. It usually preserves data, which is helpful to +avoid unplanned data loss, but will likely break a future install attempt. + +```shell +helm uninstall testgen-app +``` + +```shell +helm uninstall testgen-services +``` + +If you're uninstalling to re-install from scratch, and the data previously +generated can be purged, issue the following command to delete the database +volume so that the next installation will re-create and re-populate it +successfully. + +```shell +kubectl delete pvc data-testgen-services-postgresql-0 +``` diff --git a/deploy/charts/testgen-app/Chart.yaml b/deploy/charts/testgen-app/Chart.yaml new file mode 100644 index 00000000..1d2660b0 --- /dev/null +++ b/deploy/charts/testgen-app/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: testgen +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "v3" diff --git a/deploy/charts/testgen-app/templates/_environment.yaml b/deploy/charts/testgen-app/templates/_environment.yaml new file mode 100644 index 00000000..fa7ae504 --- /dev/null +++ b/deploy/charts/testgen-app/templates/_environment.yaml @@ -0,0 +1,40 @@ +{{- define "testgen.environment" -}} +- name: TG_DECRYPT_SALT + valueFrom: + secretKeyRef: + name: {{ include "testgen.fullname" . }} + key: "decrypt-salt" +- name: TG_DECRYPT_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "testgen.fullname" . }} + key: "decrypt-password" +- name: TG_METADATA_DB_HOST + value: {{ .Values.testgen.databaseHost | quote }} +- name: TG_METADATA_DB_NAME + value: {{ .Values.testgen.databaseName | quote }} +- name: TG_METADATA_DB_SCHEMA + value: {{ .Values.testgen.databaseSchema | quote }} +- name: TG_METADATA_DB_USER + value: {{ .Values.testgen.databaseUser | quote }} +- name: TG_METADATA_DB_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.testgen.databasePasswordSecret.name | quote }} + key: {{ .Values.testgen.databasePasswordSecret.key | quote }} +- name: TG_TARGET_DB_TRUST_SERVER_CERTIFICATE + value: {{ .Values.testgen.trustTargetDatabaseCertificate | ternary "yes" "no" | quote }} +- name: TG_EXPORT_TO_OBSERVABILITY_VERIFY_SSL + value: {{ .Values.testgen.observabilityVerifySsl | ternary "yes" "no" | quote }} +- name: TG_DOCKER_RELEASE_CHECK_ENABLED + value: {{ .Values.testgen.releaseCheck | ternary "yes" "no" | quote }} +{{- end -}} + +{{- define "testgen.hookEnvironment" -}} +{{- if .Values.testgen.uiPassword -}} +- name: TESTGEN_USERNAME + value: {{ .Values.testgen.uiUser | quote }} +- name: TESTGEN_PASSWORD + value: {{ .Values.testgen.uiPassword | quote }} +{{- end -}} +{{- end -}} diff --git a/deploy/charts/testgen-app/templates/_helpers.tpl b/deploy/charts/testgen-app/templates/_helpers.tpl new file mode 100644 index 00000000..844fadb1 --- /dev/null +++ b/deploy/charts/testgen-app/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "testgen.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "testgen.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "testgen.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "testgen.labels" -}} +helm.sh/chart: {{ include "testgen.chart" . }} +{{ include "testgen.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "testgen.selectorLabels" -}} +app.kubernetes.io/name: {{ include "testgen.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "testgen.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "testgen.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/deploy/charts/testgen-app/templates/deployment.yaml b/deploy/charts/testgen-app/templates/deployment.yaml new file mode 100644 index 00000000..489e18bd --- /dev/null +++ b/deploy/charts/testgen-app/templates/deployment.yaml @@ -0,0 +1,58 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "testgen.fullname" . }} + labels: + {{- include "testgen.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "testgen.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + # Used to force Helm to trigger a re-deploy despite this manifest haven't changed + timestamp: {{ now | quote }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "testgen.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "testgen.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + resources: + {{- toYaml .Values.resources | nindent 12 }} + env: + {{- include "testgen.environment" . | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/deploy/charts/testgen-app/templates/job.yaml b/deploy/charts/testgen-app/templates/job.yaml new file mode 100644 index 00000000..f19c44c5 --- /dev/null +++ b/deploy/charts/testgen-app/templates/job.yaml @@ -0,0 +1,57 @@ +{{- range .Values.cliHooks }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "testgen.fullname" $ }}-{{ .name | default "hook" }} + labels: + {{- include "testgen.labels" $ | nindent 4 }} + annotations: + "helm.sh/hook": {{ .triggers | quote }} + "helm.sh/hook-delete-policy": {{ .deletePolicy | default "before-hook-creation,hook-succeeded" | quote }} +spec: + template: + metadata: + {{- with $.Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "testgen.labels" $ | nindent 8 }} + {{- with $.Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + restartPolicy: {{ .restartPolicy | default "Never" | quote }} + {{- with $.Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "testgen.serviceAccountName" $ }} + securityContext: + {{- toYaml $.Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ $.Chart.Name }} + securityContext: + {{- toYaml $.Values.securityContext | nindent 12 }} + image: "{{ $.Values.image.repository }}:{{ $.Values.image.tag }}" + imagePullPolicy: {{ $.Values.image.pullPolicy }} + resources: + {{- toYaml $.Values.resources | nindent 12 }} + env: + {{- include "testgen.environment" $ | nindent 12 }} + {{- include "testgen.hookEnvironment" $ | nindent 12 }} + command: {{ toJson .command }} + {{- with $.Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $.Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $.Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +--- +{{- end }} diff --git a/deploy/charts/testgen-app/templates/secrets.yaml b/deploy/charts/testgen-app/templates/secrets.yaml new file mode 100644 index 00000000..79cc6101 --- /dev/null +++ b/deploy/charts/testgen-app/templates/secrets.yaml @@ -0,0 +1,14 @@ +{{- if .Values.secrets.create -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "testgen.fullname" . }} + labels: + {{- include "testgen.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": "pre-install" +type: Opaque +data: + decrypt-salt: {{ randAlphaNum 32 | b64enc | quote }} + decrypt-password: {{ randAlphaNum 32 | b64enc | quote }} +{{- end }} diff --git a/deploy/charts/testgen-app/templates/service.yaml b/deploy/charts/testgen-app/templates/service.yaml new file mode 100644 index 00000000..9ab30931 --- /dev/null +++ b/deploy/charts/testgen-app/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "testgen.fullname" . }} + labels: + {{- include "testgen.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "testgen.selectorLabels" . | nindent 4 }} diff --git a/deploy/charts/testgen-app/templates/serviceaccount.yaml b/deploy/charts/testgen-app/templates/serviceaccount.yaml new file mode 100644 index 00000000..5ede6579 --- /dev/null +++ b/deploy/charts/testgen-app/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "testgen.serviceAccountName" . }} + labels: + {{- include "testgen.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} +{{- end }} diff --git a/deploy/charts/testgen-app/values.yaml b/deploy/charts/testgen-app/values.yaml new file mode 100644 index 00000000..b412c517 --- /dev/null +++ b/deploy/charts/testgen-app/values.yaml @@ -0,0 +1,81 @@ +# Default values for testgen. + +testgen: + databaseHost: "testgen-services-postgresql" + databaseName: "datakitchen" + databaseSchema: "tgapp" + databaseUser: "postgres" + databasePasswordSecret: + name: "testgen-services-postgresql" + key: "postgres-password" + uiUser: "admin" + uiPassword: + trustTargetDatabaseCertificate: false + observabilityVerifySsl: true + releaseCheck: true + +cliHooks: + - name: "setup-db" + triggers: "pre-install" + command: ["/dk/bin/testgen", "setup-system-db", "--yes"] + - name: "upgrade-db" + triggers: "post-upgrade" + command: ["/dk/bin/testgen", "upgrade-system-version"] + +replicaCount: 1 + +image: + repository: "datakitchen/dataops-testgen" + pullPolicy: IfNotPresent + tag: "v3" + +secrets: + create: true + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: false + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} +podLabels: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 8501 + +resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/deploy/charts/testgen-services/Chart.lock b/deploy/charts/testgen-services/Chart.lock new file mode 100644 index 00000000..72fc8358 --- /dev/null +++ b/deploy/charts/testgen-services/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: postgresql + repository: https://charts.bitnami.com/bitnami + version: 16.3.0 +digest: sha256:92eb2890efc38c617fa56144f4f54c0ac1ee11818f6b00860ec00a87be48f249 +generated: "2025-02-24T09:05:32.15558-05:00" diff --git a/deploy/charts/testgen-services/Chart.yaml b/deploy/charts/testgen-services/Chart.yaml new file mode 100644 index 00000000..79704ab7 --- /dev/null +++ b/deploy/charts/testgen-services/Chart.yaml @@ -0,0 +1,29 @@ +apiVersion: v2 +name: testgen-services +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +dependencies: + - name: postgresql + version: 16.3.0 + repository: https://charts.bitnami.com/bitnami + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/deploy/charts/testgen-services/values.yaml b/deploy/charts/testgen-services/values.yaml new file mode 100644 index 00000000..b7a3f843 --- /dev/null +++ b/deploy/charts/testgen-services/values.yaml @@ -0,0 +1,7 @@ +# Default values for testgen-services. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +postgresql: + auth: + database: "datakitchen" diff --git a/deploy/charts_values/values-app-dev.yaml b/deploy/charts_values/values-app-dev.yaml new file mode 100644 index 00000000..9edb4663 --- /dev/null +++ b/deploy/charts_values/values-app-dev.yaml @@ -0,0 +1,13 @@ +testgen: + # Password that will be assigned to the 'admin' user during the database preparation + uiPassword: "admin" + + # Whether to trust the target database certificates. + trustTargetDatabaseCertificate: true + + # Whether to run the SSL certificate verifications when connecting to DataOps Observability + observabilityVerifySsl: false + +image: + # DataOps TestGen version to be installed / upgraded + tag: v3 From 0b1f40343270a837057cb3d8f512b9207891e20e Mon Sep 17 00:00:00 2001 From: Aarthy Adityan Date: Thu, 27 Feb 2025 15:48:12 -0500 Subject: [PATCH 4/6] feat(helm-charts): support extra volume and mounts, update secret --- deploy/charts/testgen-app/templates/_environment.yaml | 4 ++-- deploy/charts/testgen-app/templates/deployment.yaml | 8 ++++++++ deploy/charts/testgen-app/templates/secrets.yaml | 4 ++-- deploy/charts/testgen-app/templates/service.yaml | 5 +++++ deploy/charts/testgen-app/values.yaml | 11 ++++++----- deploy/charts/testgen-services/values.yaml | 1 + 6 files changed, 24 insertions(+), 9 deletions(-) diff --git a/deploy/charts/testgen-app/templates/_environment.yaml b/deploy/charts/testgen-app/templates/_environment.yaml index fa7ae504..01a08f38 100644 --- a/deploy/charts/testgen-app/templates/_environment.yaml +++ b/deploy/charts/testgen-app/templates/_environment.yaml @@ -2,12 +2,12 @@ - name: TG_DECRYPT_SALT valueFrom: secretKeyRef: - name: {{ include "testgen.fullname" . }} + name: {{ .Values.testgen.databaseDecryptSecret.name | quote }} key: "decrypt-salt" - name: TG_DECRYPT_PASSWORD valueFrom: secretKeyRef: - name: {{ include "testgen.fullname" . }} + name: {{ .Values.testgen.databaseDecryptSecret.name | quote }} key: "decrypt-password" - name: TG_METADATA_DB_HOST value: {{ .Values.testgen.databaseHost | quote }} diff --git a/deploy/charts/testgen-app/templates/deployment.yaml b/deploy/charts/testgen-app/templates/deployment.yaml index 489e18bd..2a98c24f 100644 --- a/deploy/charts/testgen-app/templates/deployment.yaml +++ b/deploy/charts/testgen-app/templates/deployment.yaml @@ -44,6 +44,14 @@ spec: {{- toYaml .Values.resources | nindent 12 }} env: {{- include "testgen.environment" . | nindent 12 }} + {{- with .Values.extraVolumeMounts }} + volumeMounts: + {{ toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.extraVolumes }} + volumes: + {{ toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/deploy/charts/testgen-app/templates/secrets.yaml b/deploy/charts/testgen-app/templates/secrets.yaml index 79cc6101..fdf4fa00 100644 --- a/deploy/charts/testgen-app/templates/secrets.yaml +++ b/deploy/charts/testgen-app/templates/secrets.yaml @@ -1,8 +1,8 @@ -{{- if .Values.secrets.create -}} +{{- if .Values.testgen.databaseDecryptSecret.create -}} apiVersion: v1 kind: Secret metadata: - name: {{ include "testgen.fullname" . }} + name: {{ .Values.testgen.databaseDecryptSecret.name | quote }} labels: {{- include "testgen.labels" . | nindent 4 }} annotations: diff --git a/deploy/charts/testgen-app/templates/service.yaml b/deploy/charts/testgen-app/templates/service.yaml index 9ab30931..fee275a5 100644 --- a/deploy/charts/testgen-app/templates/service.yaml +++ b/deploy/charts/testgen-app/templates/service.yaml @@ -11,5 +11,10 @@ spec: targetPort: http protocol: TCP name: http + {{- if eq .Values.service.type "NodePort" }} + {{- with .Values.service.nodePort }} + nodePort: {{ . }} + {{- end }} + {{- end }} selector: {{- include "testgen.selectorLabels" . | nindent 4 }} diff --git a/deploy/charts/testgen-app/values.yaml b/deploy/charts/testgen-app/values.yaml index b412c517..1a60ccf9 100644 --- a/deploy/charts/testgen-app/values.yaml +++ b/deploy/charts/testgen-app/values.yaml @@ -1,13 +1,16 @@ # Default values for testgen. testgen: - databaseHost: "testgen-services-postgresql" + databaseHost: "postgresql" databaseName: "datakitchen" databaseSchema: "tgapp" databaseUser: "postgres" databasePasswordSecret: name: "testgen-services-postgresql" key: "postgres-password" + databaseDecryptSecret: + create: true + name: testgen-database-decrypt uiUser: "admin" uiPassword: trustTargetDatabaseCertificate: false @@ -29,9 +32,6 @@ image: pullPolicy: IfNotPresent tag: "v3" -secrets: - create: true - imagePullSecrets: [] nameOverride: "" fullnameOverride: "" @@ -62,8 +62,9 @@ securityContext: {} # runAsUser: 1000 service: - type: ClusterIP + type: NodePort port: 8501 + nodePort: 8501 resources: {} # limits: diff --git a/deploy/charts/testgen-services/values.yaml b/deploy/charts/testgen-services/values.yaml index b7a3f843..eae5e615 100644 --- a/deploy/charts/testgen-services/values.yaml +++ b/deploy/charts/testgen-services/values.yaml @@ -3,5 +3,6 @@ # Declare variables to be passed into your templates. postgresql: + fullnameOverride: postgresql auth: database: "datakitchen" From 9b77dd1830dfc1ae2d14044bfea27840001003d0 Mon Sep 17 00:00:00 2001 From: Aarthy Adityan Date: Thu, 27 Feb 2025 17:10:36 -0500 Subject: [PATCH 5/6] ci(github): add chart release workflow --- .github/actions/publish_charts/action.yaml | 31 ++++++++++++++++++++++ .github/workflows/release.yaml | 21 +++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 .github/actions/publish_charts/action.yaml create mode 100644 .github/workflows/release.yaml diff --git a/.github/actions/publish_charts/action.yaml b/.github/actions/publish_charts/action.yaml new file mode 100644 index 00000000..6edbdc48 --- /dev/null +++ b/.github/actions/publish_charts/action.yaml @@ -0,0 +1,31 @@ +name: Publish Charts +description: Push charts to GitHub Pages +inputs: + github-token: + description: 'GitHub token for pushing to gh-pages branch' + required: true +runs: + using: 'composite' + steps: + - name: Configure Git + shell: bash + run: | + git config user.name "$GITHUB_ACTOR" + git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + + - name: Install Helm + uses: azure/setup-helm@v4.2.0 + + - name: Add Helm Repos for Dependencies + shell: bash + run: | + helm repo add bitnami https://charts.bitnami.com/bitnami + + - name: Run chart-releaser + uses: helm/chart-releaser-action@v1.6.0 + with: + charts_dir: deploy/charts + skip_existing: 'true' + packages_with_index: 'true' + env: + CR_TOKEN: '${{ inputs.github-token }}' diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 00000000..14747214 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,21 @@ +name: Release New Version +on: + push: + branches: + - main + workflow_dispatch: + branches: + - main +jobs: + publish: + runs-on: ubuntu-latest + steps: + - name: Checkout Latest Changes + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Package and Publish Charts + uses: ./.github/actions/publish_charts + with: + github-token: '${{ secrets.GITHUB_TOKEN }}' From bb408842da4a860136e35af8ba53e23bedf7edfa Mon Sep 17 00:00:00 2001 From: Aarthy Adityan Date: Fri, 28 Feb 2025 00:12:09 -0500 Subject: [PATCH 6/6] release: 3.0.0 -> 3.1.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ba587de7..8528e09c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta" [project] name = "dataops-testgen" -version = "3.0.0" +version = "3.1.0" description = "DataKitchen's Data Quality DataOps TestGen" authors = [ { "name" = "DataKitchen, Inc.", "email" = "info@datakitchen.io" },