10-deploy-spark-app.yaml.j2

---
apiVersion: spark.stackable.tech/v1alpha1
kind: SparkApplication
metadata:
  name: spark-pi-s3-1
spec:
  version: "1.0"
{% if lookup('env', 'VECTOR_AGGREGATOR') %}
  vectorAggregatorConfigMapName: vector-aggregator-discovery
{% endif %}
  sparkImage:
{% if test_scenario['values']['spark'].find(",") > 0 %}
    custom: "{{ test_scenario['values']['spark'].split(',')[1] }}"
    productVersion: "{{ test_scenario['values']['spark'].split(',')[0] }}"
{% else %}
    productVersion: "{{ test_scenario['values']['spark'] }}"
{% endif %}
    pullPolicy: IfNotPresent
  mode: cluster
  mainClass: org.apache.spark.examples.SparkPi
  mainApplicationFile: "s3a://my-bucket/spark-examples_{{ test_scenario['values']['spark'].split(',')[0] }}.jar"
  s3connection:
    reference: spark-data-s3-connection
  logFileDirectory:
    s3:
      prefix: eventlogs/
      bucket:
        reference: spark-history-s3-bucket
  job:
    config:
      logging:
        enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
  driver:
    config:
      logging:
        enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
  executor:
    replicas: 1
    config:
      logging:
        enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}