-
Notifications
You must be signed in to change notification settings - Fork 16
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
13 changed files
with
571 additions
and
10 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
13 changes: 13 additions & 0 deletions
13
config/transformer/azure/transformer.kafka.config.minimal.hocon
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| { | ||
| "input": { | ||
| "topicName": "enriched" | ||
| "bootstrapServers": "localhost:9092" | ||
| } | ||
| "output": { | ||
| "path": "https://accountName.blob.core.windows.net/transformed/" | ||
| } | ||
| "queue": { | ||
| "topicName": "loaderTopic" | ||
| "bootstrapServers": "localhost:9092" | ||
| } | ||
| } |
153 changes: 153 additions & 0 deletions
153
config/transformer/azure/transformer.kafka.config.reference.hocon
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,153 @@ | ||
| { | ||
| "input": { | ||
| "type": "kafka" | ||
|
|
||
| # Name of the Kafka topic to read from | ||
| "topicName": "enriched" | ||
|
|
||
| # A list of host:port pairs to use for establishing the initial connection to the Kafka cluster | ||
| # This list should be in the form host1:port1,host2:port2,... | ||
| "bootstrapServers": "localhost:9092" | ||
|
|
||
| # Optional, Kafka Consumer configuration | ||
| # See https://kafka.apache.org/documentation/#consumerconfigs for all properties | ||
| "consumerConf": { | ||
| "enable.auto.commit": "false" | ||
| "auto.offset.reset" : "earliest" | ||
| "group.id": "transformer" | ||
| } | ||
| } | ||
|
|
||
| # Path to transformed archive | ||
| "output": { | ||
| # Path to transformer output | ||
| "path": "https://accountName.blob.core.windows.net/transformed/", | ||
| # Transformer output compression, GZIP or NONE | ||
| # Optional, default value GZIP | ||
| "compression": "GZIP", | ||
|
|
||
| # Optional section specifying details about badrows output. When unspecified, badrows are written as files under 'output.path' URI | ||
| "bad": { | ||
|
|
||
| # Type of output sink. Either 'kafka' or 'file'. Optional, default value 'file'. When 'file', badrows are written as files under 'output.path' URI | ||
| "type": "kafka", | ||
|
|
||
| # Name of the Kafka topic to write to | ||
| "topicName": "bad" | ||
|
|
||
| # A list of host:port pairs to use for establishing the initial connection to the Kafka cluster | ||
| # This list should be in the form host1:port1,host2:port2,... | ||
| "bootstrapServers": "localhost:9092" | ||
|
|
||
| # Optional, Kafka producer configuration | ||
| # See https://kafka.apache.org/documentation/#producerconfigs for all properties | ||
| "producerConf": { | ||
| "acks": "all" | ||
| } | ||
| } | ||
| } | ||
|
|
||
| # Frequency to emit loading finished message - 5,10,15,20,30,60 etc minutes | ||
| # Optional, default value 10 minutes | ||
| "windowing": "10 minutes" | ||
|
|
||
| # Kafka topic used to communicate with Loader | ||
| "queue": { | ||
| "type": "kafka", | ||
|
|
||
| # Name of the Kafka topic to write to | ||
| "topicName": "loaderTopic" | ||
|
|
||
| # A list of host:port pairs to use for establishing the initial connection to the Kafka cluster | ||
| # This list should be in the form host1:port1,host2:port2,... | ||
| "bootstrapServers": "localhost:9092" | ||
|
|
||
| # Optional, Kafka producer configuration | ||
| # See https://kafka.apache.org/documentation/#producerconfigs for all properties | ||
| "producerConf": { | ||
| "acks": "all" | ||
| } | ||
| } | ||
|
|
||
| "formats": { | ||
| # Optional. Denotes output file format. | ||
| # Possible values are 'json' and 'parquet'. Default value 'json'. | ||
| "fileFormat": "json" | ||
| } | ||
|
|
||
| # Events will be validated against given criterias and | ||
| # bad row will be created if validation is not successful | ||
| "validations": { | ||
| "minimumTimestamp": "2021-11-18T11:00:00.00Z" | ||
| } | ||
|
|
||
| # Observability and reporting options | ||
| "monitoring": { | ||
| # Optional, for tracking runtime exceptions | ||
| "sentry": { | ||
| "dsn": "http://sentry.acme.com" | ||
| } | ||
| # Optional. How metrics are reported | ||
| "metrics": { | ||
| # Optional. Send metrics to a StatsD server (e.g. on localhost) | ||
| "statsd": { | ||
| "hostname": "localhost" | ||
| "port": 8125 | ||
| "period": "1 minute" | ||
| # Optional. Any key-value pairs to be tagged on every StatsD metric | ||
| "tags": { | ||
| "app": transformer | ||
| } | ||
| # Optional. Override the default metric prefix | ||
| # "prefix": "snowplow.transformer." | ||
| } | ||
| # Optional. Log to stdout using Slf4j (logger name: transformer.metrics) | ||
| "stdout": { | ||
| "period": "1 minute" | ||
| # Optional. Override the default metric prefix | ||
| # "prefix": "snowplow.transformer." | ||
| } | ||
| } | ||
| } | ||
|
|
||
| # Optional. Configure telemetry | ||
| # All the fields are optional | ||
| "telemetry": { | ||
| # Set to true to disable telemetry | ||
| "disable": false | ||
| # Interval for the heartbeat event | ||
| "interval": 15 minutes | ||
| # HTTP method used to send the heartbeat event | ||
| "method": "POST" | ||
| # URI of the collector receiving the heartbeat event | ||
| "collectorUri": "collector-g.snowplowanalytics.com" | ||
| # Port of the collector receiving the heartbeat event | ||
| "collectorPort": 443 | ||
| # Whether to use https or not | ||
| "secure": true | ||
| # Identifier intended to tie events together across modules, | ||
| # infrastructure and apps when used consistently | ||
| "userProvidedId": "my_pipeline" | ||
| # ID automatically generated upon running a modules deployment script | ||
| # Intended to identify each independent module, and the infrastructure it controls | ||
| "autoGeneratedId": "hfy67e5ydhtrd" | ||
| # Unique identifier for the VM instance | ||
| # Unique for each instance of the app running within a module | ||
| "instanceId": "665bhft5u6udjf" | ||
| # Name of the terraform module that deployed the app | ||
| "moduleName": "transformer-kafka-ce" | ||
| # Version of the terraform module that deployed the app | ||
| "moduleVersion": "1.0.0" | ||
| } | ||
|
|
||
| # Optional. Enable features that are still in beta, or which are here to enable smoother upgrades | ||
| "featureFlags": { | ||
| # Read/write in the legacy version 1 shredding complete message format. | ||
| # This should be enabled during upgrade from older versions of the loader. | ||
| "legacyMessageFormat": false | ||
|
|
||
| # When enabled, event's atomic fields are truncated (based on the length limits from the atomic JSON schema) before transformation. | ||
| # Optional, default "false". | ||
| "truncateAtomicFields": false | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.