/
recipe.yaml
38 lines (38 loc) · 1.53 KB
/
recipe.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
---
RecipeFormatVersion: "2020-01-25"
ComponentName: com.devopstar.S3Ingestor
ComponentVersion: NEXT_PATCH
ComponentDescription: The Greengrass S3 Ingestor component takes a stream of JSON messages from StreamManager and batches them into a gzip file. It uses a JSON Line (JSONL) format for the messages. The component enables efficient ingestion of data into S3 for further processing or storage.
ComponentPublisher: Nathan Glover
ComponentDependencies:
aws.greengrass.StreamManager:
VersionRequirement: ^2.0.0
DependencyType: HARD
ComponentConfiguration:
DefaultConfiguration:
Path: "/tmp/com.devopstar.S3Ingestor/data"
Interval: "30"
Processor:
StreamName: "BatchMessageStream"
BatchSize: "200"
Uploader:
BucketName: ""
Prefix: ""
LogLevel: "INFO"
Manifests:
- Artifacts:
- URI: "s3://BUCKET_NAME/COMPONENT_NAME/COMPONENT_VERSION/bundle.zip"
Unarchive: ZIP
Lifecycle:
Run:
Setenv:
PYTHONPATH: "{artifacts:decompressedPath}/bundle"
Script: |-
python3 -u {artifacts:decompressedPath}/bundle/main.py \
--path "{configuration:/Path}" \
--interval "{configuration:/Interval}" \
--processor_stream_name "{configuration:/Processor/StreamName}" \
--processor_batch_size "{configuration:/Processor/BatchSize}" \
--uploader_bucket_name "{configuration:/Uploader/BucketName}" \
--uploader_prefix "{configuration:/Uploader/Prefix}" \
--log_level "{configuration:/LogLevel}"