Spark Operator example update with best practices

examples/analytics/spark-k8s-operator/README.md

@@ -49,28 +49,19 @@ terraform apply
 
 Enter `yes` to apply.
 
-## Execute Sample Spark Job on EKS Cluster with Spark-k8s-operator:
-
-- Create Spark Namespace, Service Account and ClusterRole and ClusterRole Binding for the jobs
-
-```sh
-   cd examples/analytics/spark-k8s-operator/k8s-schedular
-   kubectl apply -f spark-teams-setup.yaml
-```
-
-- Execute first spark job with simple example
+## Execute Sample Spark Job on EKS Cluster with Spark-k8s-operator
 
 ```sh
-  cd examples/analytics/spark-k8s-operator/k8s-schedular
+  cd examples/analytics/spark-k8s-operator/spark-samples
   kubectl apply -f pyspark-pi-job.yaml
 ```
 
 - Verify the Spark job status
 
 ```sh
-  kubectl get sparkapplications -n spark-ns
+  kubectl get sparkapplications -n spark-team-a
 
-  kubectl describe sparkapplication pyspark-pi -n spark-ns
+  kubectl describe sparkapplication pyspark-pi -n spark-team-a
 ```
 
 ## Cleanup

examples/analytics/spark-k8s-operator/helm-values/aws-for-fluentbit-values.yaml

@@ -0,0 +1,72 @@
+global:
+  ## Override the deployment namespace
+  namespaceOverride: logging
+
+#hostNetwork and dnsPolicy are critical for enabling large clusters to avoid making calls to API server
+# see this link https://docs.fluentbit.io/manual/pipeline/filters/kubernetes#optional-feature-using-kubelet-to-get-metadata
+hostNetwork: true
+dnsPolicy: ClusterFirstWithHostNet
+
+# NOTE: extraFilters config for using Kubelet to get the Metadata instead of talking to API server for large clusters
+filter:
+  name: "kubernetes"
+  match: "kube.*"
+  kubeURL: "https://kubernetes.default.svc.cluster.local:443"
+  mergeLog: "On"
+  mergeLogKey: "log_processed"
+  keepLog: "On"
+  k8sLoggingParser: "On"
+  k8sLoggingExclude: "Off"
+  bufferSize: "0"
+  extraFilters: |
+    Kube_Tag_Prefix     application.var.log.containers.
+    Labels              Off
+    Annotations         Off
+    Use_Kubelet         true
+    Kubelet_Port        10250
+    Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
+
+cloudWatch:
+  enabled: true
+  match: "*"
+  region: ${region}
+  logGroupName: ${aws_for_fluent_bit_cw_log}
+  logStreamName:
+  logStreamPrefix: "fluentbit-"
+  logKey:
+  logFormat:
+  roleArn:
+  autoCreateGroup: true
+  endpoint:
+  credentialsEndpoint:  {}
+
+firehose:
+  enabled: false
+
+kinesis:
+  enabled: false
+
+elasticsearch:
+  enabled: false
+
+serviceAccount:
+  create: true
+
+# Resource config for large clusters
+resources:
+  limits:
+    cpu: 1000m
+    memory: 1500Mi
+  requests:
+    cpu: 500m
+    memory: 500Mi
+
+## Assign a PriorityClassName to pods if set
+priorityClassName: system-node-critical
+
+updateStrategy:
+  type: RollingUpdate
+
+nodeSelector:
+  kubernetes.io/os: linux

examples/analytics/spark-k8s-operator/helm-values/cluster-autoscaler-values.yaml

@@ -0,0 +1,41 @@
+autoDiscovery:
+  clusterName: ${eks_cluster_id}
+
+awsRegion: ${aws_region}
+
+cloudProvider: aws
+
+extraArgs:
+  logtostderr: true
+  stderrthreshold: info
+  v: 4
+  aws-use-static-instance-list: true
+
+nodeSelector:
+  NodeGroupType: ${node_group_type}
+  kubernetes.io/os: ${operating_system}
+
+rbac:
+  create: true
+  pspEnabled: false
+  serviceAccount:
+    create: false
+
+# replicaCount -- Desired number of pods
+replicaCount: 1
+
+# Best practice to update the resource requests and limits for each add-on
+resources:
+   limits:
+     cpu: 1000m
+     memory: 1G
+   requests:
+     cpu: 200m
+     memory: 512Mi
+
+# Best practice to updateStrategy for each add-on
+updateStrategy:
+  type: RollingUpdate
+  rollingUpdate:
+    maxSurge: 0
+    maxUnavailable: 1

examples/analytics/spark-k8s-operator/helm-values/coredns-autoscaler-values.yaml

@@ -0,0 +1,44 @@
+nameOverride: kube-dns-autoscaler
+
+# Formula for controlling the replicas. Adjust according to your needs
+#  replicas = max( ceil( cores * 1/coresPerReplica ) , ceil( nodes * 1/nodesPerReplica ) )
+#  replicas = min(replicas, max)
+#  replicas = max(replicas, min)
+config:
+  linear:
+    coresPerReplica: 256
+    nodesPerReplica: 16
+    min: 1
+    max: 100
+    preventSinglePointFailure: true
+    includeUnschedulableNodes: true
+
+# Target to scale. In format: deployment/*, replicationcontroller/* or replicaset/* (not case sensitive).
+options:
+  target: ${target}
+
+serviceAccount:
+  create: true
+  name: kube-dns-autoscaler
+
+podSecurityContext:
+  seccompProfile:
+    type: RuntimeDefault
+  supplementalGroups: [ 65534 ]
+  fsGroup: 65534
+
+nodeSelector:
+  kubernetes.io/os: ${operating_system}
+  NodeGroupType: ${node_group_type}
+
+resources:
+  limits:
+    cpu: 100m
+    memory: 128Mi
+  requests:
+    cpu: 100m
+    memory: 128Mi
+
+tolerations:
+  - key: "CriticalAddonsOnly"
+    operator: "Exists"

examples/analytics/spark-k8s-operator/helm-values/metrics-server-values.yaml

@@ -0,0 +1,39 @@
+# HA config for metrics-server
+serviceAccount:
+  create: true
+  name: metrics-server
+
+podLabels:
+  k8s-app: metrics-server
+
+# HA enabled by enabling replicas to 2, updateStrategy and podDisruptionBudget to true
+replicas: 2
+
+updateStrategy:
+   type: RollingUpdate
+   rollingUpdate:
+     maxSurge: 0
+     maxUnavailable: 1
+
+podDisruptionBudget:
+  enabled: true
+  minAvailable: 1
+
+resources:
+  requests:
+    cpu: 200m
+    memory: 512Mi
+
+nodeSelector:
+  NodeGroupType: ${node_group_type}
+  kubernetes.io/os: ${operating_system}
+
+affinity:
+  podAntiAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      - labelSelector:
+          matchLabels:
+            k8s-app: metrics-server
+        namespaces:
+          - kube-system
+        topologyKey: kubernetes.io/hostname

examples/analytics/spark-k8s-operator/helm-values/prometheus-values.yaml

@@ -0,0 +1,65 @@
+server:
+  retention: 1h
+  remoteWrite:
+    - queue_config:
+        max_samples_per_send: 1000
+        max_shards: 200
+        capacity: 2500
+  global:
+    evaluation_interval: 30s
+    scrape_interval: 30s
+    scrape_timeout: 10s
+
+  resources:
+    requests:
+      cpu: 500m
+      memory: 1Gi
+
+  verticalAutoscaler:
+    enabled: true
+    updateMode: "Auto"
+    containerPolicies:
+      - containerName: "prometheus-server"
+        minAllowed:
+          cpu: 500m
+          memory: 1Gi
+        maxAllowed:
+          cpu: 1000m
+          memory: 4Gi
+
+  nodeSelector:
+    kubernetes.io/os: ${operating_system}
+    NodeGroupType: ${node_group_type}
+  persistentVolume:
+    accessModes:
+      - ReadWriteOnce
+    enabled: true
+    mountPath: /data
+    size: 20Gi
+    storageClass: gp2
+
+alertmanager:
+  nodeSelector:
+    kubernetes.io/os: ${operating_system}
+    NodeGroupType: ${node_group_type}
+
+kube-state-metrics:
+  nodeSelector:
+    kubernetes.io/os: ${operating_system}
+    NodeGroupType: ${node_group_type}
+
+pushgateway:
+  nodeSelector:
+    kubernetes.io/os: ${operating_system}
+    NodeGroupType: ${node_group_type}
+
+nodeExporter:
+  nodeSelector:
+    kubernetes.io/os: ${operating_system}
+
+  # Additional scrape config for Apache YuniKorn Scheduler metrics
+  #      - job_name: 'yunikorn'
+  #        metrics_path: '/ws/v1/metrics'
+  #        static_configs:
+  #          - targets:
+  #              - yunikorn-service.yunikorn.svc.cluster.local:9080

examples/analytics/spark-k8s-operator/helm-values/spark-k8s-operator-values.yaml

@@ -0,0 +1,28 @@
+replicaCount: 1
+
+# nodeSelector -- Node labels for pod assignment
+nodeSelector:
+  NodeGroupType: ${node_group_type}
+  kubernetes.io/os: ${operating_system}
+
+webhook:
+  # -- Enable webhook server
+  enable: true
+  # -- Webhook service port
+  port: 8080
+
+# resources -- Pod resource requests and limits
+# Note, that each job submission will spawn a JVM within the Spark Operator Pod using "/usr/local/openjdk-11/bin/java -Xmx128m".
+# Kubernetes may kill these Java processes at will to enforce resource limits. When that happens, you will see the following error:
+# 'failed to run spark-submit for SparkApplication [...]: signal: killed' - when this happens, you may want to increase memory limits.
+resources:
+   limits:
+     cpu: 200m
+     memory: 1Gi
+   requests:
+     cpu: 100m
+     memory: 512Mi
+
+batchScheduler:
+  # -- Enable batch scheduler for spark jobs scheduling. If enabled, users can specify batch scheduler name in spark application
+  enable: true

examples/analytics/spark-k8s-operator/helm-values/vpa-values.yaml

@@ -0,0 +1,20 @@
+# Default values for vertical-pod-autoscaler.
+serviceAccount:
+  name: vpa
+
+recommender:
+  enabled: true
+  extraArgs:
+    v: "4"
+    pod-recommendation-min-cpu-millicores: 15
+    pod-recommendation-min-memory-mb: 100
+    # Best practise to use Prometheus as a history provider for the VPA recommender. Not required for this example
+#   storage: prometheus
+#   prometheus-address: http://prometheus-server.prometheus.svc.cluster.local:9090
+  nodeSelector:
+    NodeGroupType: ${node_group_type}
+    kubernetes.io/os: ${operating_system}
+updater:
+  enabled: true
+  nodeSelector:
+    kubernetes.io/os: ${operating_system}