-
Notifications
You must be signed in to change notification settings - Fork 0
/
end_to_end_pipeline_from_local_to_bq-deployment.yaml
53 lines (52 loc) · 1.51 KB
/
end_to_end_pipeline_from_local_to_bq-deployment.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
###
### A complete description of a Prefect Deployment for flow 'end_to_end_pipeline_from_local_to_bq'
###
name: Local Parameterized ETL
description: Flow that take the dataset in excel format given a list of links inside
a specific folder, Combine the data, save it to csv, convert to parquet and then
upload to google cloud bucket
version: d9a25ab308947ebc7e205edc7c668377
# The work queue that will handle this deployment's runs
work_queue_name: main-queue
work_pool_name: default-agent-pool
tags: []
parameters: {}
schedule: null
is_schedule_active: null
infra_overrides: {}
infrastructure:
type: process
env: {}
labels: {}
name: null
command: null
stream_output: true
working_dir: null
block_type_slug: process
_block_type_slug: process
###
### DO NOT EDIT BELOW THIS LINE
###
flow_name: end_to_end_pipeline_from_local_to_bq
manifest_path: null
storage: null
path: /Users/lirone/Documents/dev/other_projects/data-engineering-capstone-project
entrypoint: src/flows/parameterized_flow_local_pipeline.py:end_to_end_pipeline_from_local_to_bq
parameter_openapi_schema:
title: Parameters
type: object
properties:
data_folder:
title: data_folder
default: /Users/lirone/Documents/dev/other_projects/data-engineering-capstone-project/data
position: 0
type: string
format: path
name_output_dataset:
title: name_output_dataset
default: global_terrorism_db
position: 1
type: string
required: null
definitions: null
timestamp: '2023-04-20T16:25:30.931847+00:00'