Skip to content

Commit

Permalink
[Addon kubevela#579] Address the review suggestion
Browse files Browse the repository at this point in the history 
Signed-off-by: yanghua <yanghua1127@gmail.com>
  • Loading branch information
@yanghua
yanghua committed Mar 1, 2023
1 parent 5e615b6 commit 00a08aa
Show file tree
Hide file tree
Showing 4 changed files with 116 additions and 41 deletions.
11 changes: 11 additions & 0 deletions examples/spark-kubernetes-operator/sparkapp.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,5 +17,16 @@ spec:
mainClass: org.apache.spark.examples.streaming.JavaQueueStream
mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.12-3.1.1.jar"
sparkVersion: "3.1.1"
volumes:
- name: "test-volume"
hostPath:
path: "/tmp"
type: Directory
driverCores: 1
executorCores: 1
driverVolumeMounts:
- name: "test-volume"
mountPath: "/tmp"
executorVolumeMounts:
- name: "test-volume"
mountPath: "/tmp"
116 changes: 76 additions & 40 deletions experimental/addons/spark-kubernetes-operator/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,39 +34,81 @@ vela ls -A | grep spark
```
vela show spark-workload
# Specification
+---------------------+------------------------------------------------------------------------------------------------------+-------------------+----------+---------+
| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT |
+---------------------+------------------------------------------------------------------------------------------------------+-------------------+----------+---------+
| name | Specify the spark application name. | string | true | |
| namespace | Specify the namespace for spark application to install. | string | true | |
| type | Specify the application language type, e.g. "Scala", "Python", "Java" or "R". | string | true | |
| pythonVersion | Specify the python version. | string | false | |
| mode | Specify the deploy mode, e.go "cluster", "client" or "in-cluster-client". | string | true | |
| image | Specify the container image for the driver, executor, and init-container. | string | true | |
| imagePullPolicy | Specify the image pull policy for the driver, executor, and init-container. | string | true | |
| mainClass | Specify the fully-qualified main class of the Spark application. | string | true | |
| mainApplicationFile | Specify the path to a bundled JAR, Python, or R file of the application. | string | true | |
| sparkVersion | Specify the version of Spark the application uses. | string | true | |
| driverCores | Specify the number of CPU cores to request for the driver pod. | int | true | |
| executorCores | Specify the number of CPU cores to request for the executor pod. | int | true | |
| arguments | Specify a list of arguments to be passed to the application. | []string | false | |
| sparkConf | Specify the config information carries user-specified Spark configuration properties as they would | map[string]string | false | |
| | use the "--conf" option in spark-submit. | | | |
| hadoopConf | Specify the config information carries user-specified Hadoop configuration properties as they would | map[string]string | false | |
| | use the the "--conf" option in spark-submit. The SparkApplication controller automatically adds | | | |
| | prefix "spark.hadoop." to Hadoop configuration properties. | | | |
| sparkConfigMap | Specify the name of the ConfigMap containing Spark configuration files such as log4j.properties. The | string | false | |
| | controller will add environment variable SPARK_CONF_DIR to the path where the ConfigMap is mounted | | | |
| | to. | | | |
| hadoopConfigMap | Specify the name of the ConfigMap containing Hadoop configuration files such as core-site.xml. The | string | false | |
| | controller will add environment variable HADOOP_CONF_DIR to the path where the ConfigMap is mounted | | | |
| | to. | | | |
+---------------------+------------------------------------------------------------------------------------------------------+-------------------+----------+---------+
+----------------------+------------------------------------------------------------------------------------------------------+-------------------------------------------------+----------+---------+
| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT |
+----------------------+------------------------------------------------------------------------------------------------------+-------------------------------------------------+----------+---------+
| name | Specify the spark application name. | string | true | |
| namespace | Specify the namespace for spark application to install. | string | true | |
| type | Specify the application language type, e.g. "Scala", "Python", "Java" or "R". | string | true | |
| pythonVersion | Specify the python version. | string | false | |
| mode | Specify the deploy mode, e.go "cluster", "client" or "in-cluster-client". | string | true | |
| image | Specify the container image for the driver, executor, and init-container. | string | true | |
| imagePullPolicy | Specify the image pull policy for the driver, executor, and init-container. | string | true | |
| mainClass | Specify the fully-qualified main class of the Spark application. | string | true | |
| mainApplicationFile | Specify the path to a bundled JAR, Python, or R file of the application. | string | true | |
| sparkVersion | Specify the version of Spark the application uses. | string | true | |
| driverCores | Specify the number of CPU cores to request for the driver pod. | int | true | |
| executorCores | Specify the number of CPU cores to request for the executor pod. | int | true | |
| arguments | Specify a list of arguments to be passed to the application. | []string | false | |
| sparkConf | Specify the config information carries user-specified Spark configuration properties as they would | map[string]string | false | |
| | use the "--conf" option in spark-submit. | | | |
| hadoopConf | Specify the config information carries user-specified Hadoop configuration properties as they would | map[string]string | false | |
| | use the the "--conf" option in spark-submit. The SparkApplication controller automatically adds | | | |
| | prefix "spark.hadoop." to Hadoop configuration properties. | | | |
| sparkConfigMap | Specify the name of the ConfigMap containing Spark configuration files such as log4j.properties. The | string | false | |
| | controller will add environment variable SPARK_CONF_DIR to the path where the ConfigMap is mounted | | | |
| | to. | | | |
| hadoopConfigMap | Specify the name of the ConfigMap containing Hadoop configuration files such as core-site.xml. The | string | false | |
| | controller will add environment variable HADOOP_CONF_DIR to the path where the ConfigMap is mounted | | | |
| | to. | | | |
| volumes | Specify the list of Kubernetes volumes that can be mounted by the driver and/or executors. | [[]volumes](#volumes) | false | |
| driverVolumeMounts | Specify the volumes listed in "parameter.volumes" to mount into the main container’s filesystem for | [[]driverVolumeMounts](#drivervolumemounts) | false | |
| | driver pod. | | | |
| executorVolumeMounts | Specify the volumes listed in "parameter.volumes" to mount into the main container’s filesystem for | [[]executorVolumeMounts](#executorvolumemounts) | false | |
| | executor pod. | | | |
+----------------------+------------------------------------------------------------------------------------------------------+-------------------------------------------------+----------+---------+
## volumes
+----------+-------------+-----------------------+----------+---------+
| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT |
+----------+-------------+-----------------------+----------+---------+
| name | | string | true | |
| hostPath | | [hostPath](#hostpath) | true | |
+----------+-------------+-----------------------+----------+---------+
### hostPath
+------+-------------+--------+----------+-----------+
| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT |
+------+-------------+--------+----------+-----------+
| path | | string | true | |
| type | | string | false | Directory |
+------+-------------+--------+----------+-----------+
## driverVolumeMounts
+-----------+-------------+--------+----------+---------+
| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT |
+-----------+-------------+--------+----------+---------+
| name | | string | true | |
| mountPath | | string | true | |
+-----------+-------------+--------+----------+---------+
## executorVolumeMounts
+-----------+-------------+--------+----------+---------+
| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT |
+-----------+-------------+--------+----------+---------+
| name | | string | true | |
| mountPath | | string | true | |
+-----------+-------------+--------+----------+---------+
```

# Example for how to run a component typed spark-cluster in application

**Note**: If we want to check and verify the mount volume, we need to specify `parameter.createWebhook` to be `true`

1. Firstly, download or copy `catalog/examples/spark-kubernetes-operator/sparkapp.yaml`

2. Secondly, start the application:
Expand Down Expand Up @@ -109,22 +151,16 @@ Workflow:
Suspend: false
Terminated: false
Steps
- id: r9day16jok
name: deploy-topology-two-clouds
type: deploy
- id: vfgjkrxvih
name: spark-workload-component
type: apply-component
phase: succeeded
Services:
- Name: my-spark-application-component
Cluster: huaweibigdatak8s Namespace: spark-cluster
Type: spark-application
Healthy
No trait applied
- Name: my-spark-application-component
- Name: spark-workload-component
Cluster: local Namespace: spark-cluster
Type: spark-application
Type: spark-workload
Healthy
No trait applied
```
Expand Down
28 changes: 28 additions & 0 deletions experimental/addons/spark-kubernetes-operator/definitions/spark-workload.cue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,24 @@ template: {
sparkConfigMap?: string
// +usage=Specify the name of the ConfigMap containing Hadoop configuration files such as core-site.xml. The controller will add environment variable HADOOP_CONF_DIR to the path where the ConfigMap is mounted to
hadoopConfigMap?: string
// +usage=Specify the list of Kubernetes volumes that can be mounted by the driver and/or executors
volumes?: [...{
name: string
hostPath: {
path: string
type: *"Directory" | string
}
}]
// +usage=Specify the volumes listed in "parameter.volumes" to mount into the main container’s filesystem for driver pod
driverVolumeMounts?: [...{
name: string
mountPath: string
}]
// +usage=Specify the volumes listed in "parameter.volumes" to mount into the main container’s filesystem for executor pod
executorVolumeMounts?: [...{
name: string
mountPath: string
}]
}

output: {
Expand Down Expand Up @@ -79,10 +97,20 @@ template: {
sparkVersion: parameter.sparkVersion
driver: {
cores: parameter.driverCores
if parameter.driverVolumeMounts != _|_ {
volumeMounts: parameter.driverVolumeMounts
}
}
executor: {
cores: parameter.executorCores
if parameter.executorVolumeMounts != _|_ {
volumeMounts: parameter.executorVolumeMounts
}
}
if parameter.volumes != _|_ {
volumes: parameter.volumes
}

}
}
}
Expand Down
2 changes: 1 addition & 1 deletion experimental/addons/spark-kubernetes-operator/parameter.cue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ parameter: {
clusters?: [...string]
// +usage=Namespace to deploy to
namespace: *"spark-operator" | string
// +usage=Specify if create the webhook or not
// +usage=Specify if create the webhook or not
"createWebhook": *false | bool
// +usage=Specify the image repository
"imageRepository": *"ghcr.io/googlecloudplatform/spark-operator" | string
Expand Down

0 comments on commit 00a08aa

Please sign in to comment.