ny-tlc-report-external-dependencies.yaml

---
apiVersion: spark.stackable.tech/v1alpha1
kind: SparkApplication
metadata:
  name: spark-ny-ext
  namespace: default
spec:
  sparkImage:
    productVersion: 3.5.1
    pullPolicy: IfNotPresent
  mode: cluster
  mainApplicationFile: s3a://my-bucket/ny_tlc_report.py
  args:
    - "--input 's3a://my-bucket/yellow_tripdata_2021-07.csv'"
  deps:
    requirements:
      - tabulate==0.8.9
  s3connection:
    inline:
      host: test-minio
      port: 9000
      accessStyle: Path
  sparkConf:
    spark.hadoop.fs.s3a.aws.credentials.provider: "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider"
    spark.driver.extraClassPath: "/dependencies/jars/*"
    spark.executor.extraClassPath: "/dependencies/jars/*"
  volumes:
    - name: job-deps
      persistentVolumeClaim:
        claimName: pvc-ksv
  driver:
    config:
      volumeMounts:
        - name: job-deps
          mountPath: /dependencies
  executor:
    config:
      replicas: 3
      volumeMounts:
        - name: job-deps
          mountPath: /dependencies