jobJSON.json

{
    "name": "Example_Data_Pipeline",
    "email_notifications": {
        "no_alert_for_skipped_runs": false
    },
    "webhook_notifications": {},
    "timeout_seconds": 0,
    "max_concurrent_runs": 1,
    "tasks": [
        {
            "task_key": "Ingestion_To_Bronze",
            "run_if": "ALL_SUCCESS",
            "notebook_task": {
                "notebook_path": "bronze/ingestionToBronze",
                "source": "GIT"
            },
            "job_cluster_key": "New_job_cluster",
            "timeout_seconds": 0,
            "email_notifications": {},
            "notification_settings": {
                "no_alert_for_skipped_runs": false,
                "no_alert_for_canceled_runs": false,
                "alert_on_last_attempt": false
            }
        },
        {
            "task_key": "Bronze_To_Silver",
            "depends_on": [
                {
                    "task_key": "Ingestion_To_Bronze"
                }
            ],
            "run_if": "ALL_SUCCESS",
            "notebook_task": {
                "notebook_path": "silver/bronzeToSilver",
                "source": "GIT"
            },
            "job_cluster_key": "New_job_cluster",
            "timeout_seconds": 0,
            "email_notifications": {},
            "notification_settings": {
                "no_alert_for_skipped_runs": false,
                "no_alert_for_canceled_runs": false,
                "alert_on_last_attempt": false
            }
        },
        {
            "task_key": "Silver_To_Gold",
            "depends_on": [
                {
                    "task_key": "Bronze_To_Silver"
                }
            ],
            "run_if": "ALL_SUCCESS",
            "notebook_task": {
                "notebook_path": "gold/silverToGold",
                "source": "GIT"
            },
            "job_cluster_key": "New_job_cluster",
            "timeout_seconds": 0,
            "email_notifications": {},
            "notification_settings": {
                "no_alert_for_skipped_runs": false,
                "no_alert_for_canceled_runs": false,
                "alert_on_last_attempt": false
            }
        }
    ],
    "job_clusters": [
        {
            "job_cluster_key": "New_job_cluster",
            "new_cluster": {
                "spark_version": "11.3.x-scala2.12",
                "aws_attributes": {
                    "first_on_demand": 1,
                    "availability": "SPOT_WITH_FALLBACK",
                    "zone_id": "us-west-2a",
                    "spot_bid_price_percent": 100,
                    "ebs_volume_count": 0
                },
                "node_type_id": "i3.xlarge",
                "spark_env_vars": {
                    "PYSPARK_PYTHON": "/databricks/python3/bin/python3"
                },
                "enable_elastic_disk": false,
                "data_security_mode": "SINGLE_USER",
                "runtime_engine": "STANDARD",
                "num_workers": 2
            }
        }
    ],
    "git_source": {
        "git_url": "https://github.com/mpfis/repos-cicd-example",
        "git_provider": "gitHub",
        "git_branch": "dev"
    },
    "format": "MULTI_TASK"
}