06-deploy-custom-log-config-spark-app.yaml.j2

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: spark-log-config
data:
  log4j2.properties: |-
    appenders = FILE

    appender.FILE.type = File
    appender.FILE.name = FILE
    appender.FILE.fileName = /stackable/log/spark/spark.log4j2.xml
    appender.FILE.layout.type = XMLLayout

    rootLogger.level = INFO
    rootLogger.appenderRefs = FILE
    rootLogger.appenderRef.FILE.ref = FILE
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: spark-submit-log-config
data:
  log4j2.properties: |-
    appenders = FILE

    appender.FILE.type = File
    appender.FILE.name = FILE
    appender.FILE.fileName = /stackable/log/spark-submit/spark.log4j2.xml
    appender.FILE.layout.type = XMLLayout

    rootLogger.level = INFO
    rootLogger.appenderRefs = FILE
    rootLogger.appenderRef.FILE.ref = FILE
---
apiVersion: spark.stackable.tech/v1alpha1
kind: SparkApplication
metadata:
  name: spark-custom-log-config
spec:
  version: "1.0"
  sparkImage:
{% if test_scenario['values']['spark'].find(",") > 0 %}
    custom: "{{ test_scenario['values']['spark'].split(',')[1] }}"
    productVersion: "{{ test_scenario['values']['spark'].split(',')[0] }}"
{% else %}
    productVersion: "{{ test_scenario['values']['spark'] }}"
{% endif %}
    pullPolicy: IfNotPresent
  image: docker.stackable.tech/stackable/ny-tlc-report:{{ test_scenario['values']['ny-tlc-report'] }}
  vectorAggregatorConfigMapName: spark-vector-aggregator-discovery
  mode: cluster
  mainClass: org.apache.spark.examples.SparkALS
  mainApplicationFile: local:///stackable/spark/examples/jars/spark-examples.jar
  job:
    config:
      logging:
        enableVectorAgent: true
        containers:
          spark-submit:
            custom:
              configMap: spark-submit-log-config
  driver:
    config:
      logging:
        enableVectorAgent: true
        containers:
          spark:
            custom:
              configMap: spark-log-config
  executor:
    replicas: 1
    config:
      logging:
        enableVectorAgent: true
        containers:
          spark:
            custom:
              configMap: spark-log-config