feat: Airflow demo - add DBT job (#342)

* wip

* working dag

* working dag: cleanup

* comment

* working with 3.0.6 and necessary rbac

* wip: tls almost working

* demo setup

* demo setup II

* minor cleanup

* reduce size of demo and prepare for public use

* revert changes

* added back in admin credentials

* renamed stack, remove logging config, correct password

* working dbt test

* working dbt DAG in original airflow stack

* remove project

* use secret for env-var

* demo cleanup

* add workflow for demo image, move dbt folder to demos, complete docs

* correct paths

* add clarifying comments

* corrected link

* re-set demos/stacks links

Author: adwk67

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ exclude: '(stacks/_templates/minio-.*/rendered-chart\.yaml|\.svg)$'
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: 2c9f875913ee60ca25ce70243dc24d5b6415598c # 4.6.0
+    rev: v6.0.0 # 4.6.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -16,22 +16,22 @@ repos:
         exclude: '^stacks/argo-cd-git-ops/secrets/sealed-secrets-key\.yaml$'
 
   - repo: https://github.com/adrienverge/yamllint
-    rev: 81e9f98ffd059efe8aa9c1b1a42e5cce61b640c6 # 1.35.1
+    rev: v1.37.1 # 1.35.1
     hooks:
       - id: yamllint
 
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: f295829140d25717bc79368d3f966fc1f67a824f # 0.41.0
+    rev: v0.46.0 # 0.41.0
     hooks:
       - id: markdownlint
 
   - repo: https://github.com/koalaman/shellcheck-precommit
-    rev: 2491238703a5d3415bb2b7ff11388bf775372f29 # 0.10.0
+    rev: v0.11.0 # 0.10.0
     hooks:
       - id: shellcheck
         args: ["--severity=info"]
 
   - repo: https://github.com/rhysd/actionlint
-    rev: 62dc61a45fc95efe8c800af7a557ab0b9165d63b # 1.7.1
+    rev: v1.7.9 # 1.7.1
     hooks:
       - id: actionlint

demos/airflow-scheduled-job/create-trino-tables.yaml

@@ -0,0 +1,61 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: create-tables-in-trino
+spec:
+  template:
+    spec:
+      serviceAccountName: demo-serviceaccount
+      containers:
+        - name: create-tables-in-trino
+          image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+          command: ["bash", "-c", "python -u /tmp/script/script.py"]
+          volumeMounts:
+            - name: script
+              mountPath: /tmp/script
+            - name: trino-users
+              mountPath: /trino-users
+      volumes:
+        - name: script
+          configMap:
+            name: create-tables-in-trino-script
+        - name: trino-users
+          secret:
+            secretName: trino-users
+      restartPolicy: OnFailure
+  backoffLimit: 50
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: create-tables-in-trino-script
+data:
+  script.py: |
+    import sys
+    import trino
+
+    if not sys.warnoptions:
+        import warnings
+    warnings.simplefilter("ignore")
+
+    def get_connection():
+        connection = trino.dbapi.connect(
+            host="trino-coordinator",
+            port=8443,
+            user="admin",
+            http_scheme='https',
+            auth=trino.auth.BasicAuthentication("admin", open("/trino-users/admin").read()),
+        )
+        connection._http_session.verify = False
+        return connection
+
+    def run_query(connection, query):
+        print(f"[DEBUG] Executing query {query}")
+        cursor = connection.cursor()
+        cursor.execute(query)
+        return cursor.fetchall()
+
+    connection = get_connection()
+
+    run_query(connection, "CREATE SCHEMA iceberg.dbt_schema WITH (location = 's3a://demo/dbt_schema')")

demos/airflow-scheduled-job/dbt/Dockerfile

@@ -11,7 +11,7 @@ RUN python -m venv /opt/venv
 ENV PATH="/opt/venv/bin:$PATH"
 
 # Install Python packages
-COPY requirements.txt .
+COPY demos/airflow-scheduled-job/dbt/requirements.txt .
 RUN pip install --no-cache-dir --upgrade pip && \
     pip install --no-cache-dir -r requirements.txt
 
@@ -30,7 +30,7 @@ ENV PATH="/opt/venv/bin:$PATH"
 
 WORKDIR /dbt
 
-COPY dbt_test ./dbt_test
+COPY demos/airflow-scheduled-job/dbt/dbt_test ./dbt_test
 
 # Security: non-root user
 RUN useradd -m -u 1000 dbt && chown -R dbt:dbt /dbt

demos/airflow-scheduled-job/serviceaccount.yaml

@@ -0,0 +1,49 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: demo-serviceaccount
+  namespace: default
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: demo-clusterrolebinding
+subjects:
+  - kind: ServiceAccount
+    name: demo-serviceaccount
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: demo-clusterrole
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: demo-clusterrole
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - batch
+    resources:
+      - jobs
+    verbs:
+      - get
+      - list
+      - watch

demos/demos-v2.yaml

@@ -52,6 +52,8 @@ demos:
       - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml
       - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/airflow-scheduled-job/05-enable-and-run-kafka-dag.yaml
       - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/airflow-scheduled-job/06-create-opa-users.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/airflow-scheduled-job/serviceaccount.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/airflow-scheduled-job/create-trino-tables.yaml
     supportedNamespaces: []
     resourceRequests:
       cpu: 2401m