Permalink
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
47 lines (46 sloc) 1.19 KB
jobs:
- pysparkJob:
args:
- storage_bucket_parameter
- data_file_parameter
- results_directory_parameter
mainPythonFileUri: main_python_file_parameter
stepId: ibrd-pyspark
placement:
managedCluster:
clusterName: three-node-cluster
config:
gceClusterConfig:
zoneUri: us-east1-b
masterConfig:
diskConfig:
bootDiskSizeGb: 500
machineTypeUri: n1-standard-4
softwareConfig:
imageVersion: 1.3-deb9
workerConfig:
diskConfig:
bootDiskSizeGb: 500
machineTypeUri: n1-standard-4
numInstances: 2
parameters:
- description: Python script to run
fields:
- jobs['ibrd-pyspark'].pysparkJob.mainPythonFileUri
name: MAIN_PYTHON_FILE
- description: Storage bucket location of data file and results
fields:
- jobs['ibrd-pyspark'].pysparkJob.args[0]
name: STORAGE_BUCKET
validation:
regex:
regexes:
- gs://.*
- description: IBRD data file
fields:
- jobs['ibrd-pyspark'].pysparkJob.args[1]
name: IBRD_DATA_FILE
- description: Result directory
fields:
- jobs['ibrd-pyspark'].pysparkJob.args[2]
name: RESULTS_DIRECTORY