From bca2d927f2a757207675c1ca6a6ba9857995df54 Mon Sep 17 00:00:00 2001 From: Maksim Lebedev Date: Sun, 18 Sep 2022 18:21:40 +0400 Subject: [PATCH] Documentation --- .../configuration/error-handling.mdx | 38 +++++++++++++++++++ server/appconfig/config.template.yaml | 13 ++++--- 2 files changed, 45 insertions(+), 6 deletions(-) create mode 100644 documentation/configuration/error-handling.mdx diff --git a/documentation/configuration/error-handling.mdx b/documentation/configuration/error-handling.mdx new file mode 100644 index 000000000..c267597fe --- /dev/null +++ b/documentation/configuration/error-handling.mdx @@ -0,0 +1,38 @@ +# Error Handling and Retries + +**Jitsu** supports sending events and re-sending them at failures. + +## Configuration + +```yaml +server: + ... + streaming_retry_delay_minutes: 1 # Optional. Default value is 1 + error_retry_period_hours: 24 # Optional. Default value is 24 +log: + ... + rotation_min: 5 # Optional. Default value is 5 +... +destinations: + ... + destination_id: + ... + data_layout: + ... + # Positive value overrides server option for destination + error_retry_period_hours: 0 # Optional. Default value is 0 +``` + +## Batch mode + +**Jitsu** receives a lot of events. All of them are saved into a file system. +Every `log.rotation_min` minutes **Jitsu** parses saved files with events and tries to send them to destinations. +If an event was sent correctly it will be archived into a special folder with successful events. Otherwise, **Jitsu** tries to resend it at the next uploading iteration. +If an event cannot be sent for any reason during `server.error_retry_period_hours` hours **Jitsu** stops resending an event and archives it into a special folder with unsuccessful events. + +## Streaming mode + +**Jitsu** receives an event and tries to send it to destinations. +If an event was sent correctly it will be archived into a special folder with successful events. Otherwise, **Jitsu** tries to resend it at the next uploading iteration that happens every `server.streaming_retry_delay_minutes` minutes. +If an event cannot be sent for any reason during `server.error_retry_period_hours` hours **Jitsu** stops resending an event and archives it into a special folder with unsuccessful events. +Any destination supports local `destinations.destination_id.data_layout.error_retry_period_hours` option that overrides global `server.error_retry_period_hours` option. diff --git a/server/appconfig/config.template.yaml b/server/appconfig/config.template.yaml index d972d3a08..f572c4589 100644 --- a/server/appconfig/config.template.yaml +++ b/server/appconfig/config.template.yaml @@ -32,9 +32,10 @@ server: ### Sources reloading. If 'sources' key is http or file:/// URL than it will be reloaded every sources_reload_sec #sources_reload_sec: 1 #Optional. Default value is 1. - # Interval to retry failed event. Optional. Default value is 1 + # Error handling and retries https://jitsu.com/docs/configuration/error-handling + # Interval to retry failed events. Optional. Default value is 1 streaming_retry_delay_minutes: 1 - # Clear failed event after retry period. Optional. Default value is 24 + # Stop resending failed events after retry period. Optional. Default value is 24 error_retry_period_hours: 24 ### Application metrics @@ -131,7 +132,7 @@ destinations: # type: varchar(256) #SQL type # column_type: varchar(256) encode zstd # max_columns: 100 # Optional. The limit of the count of columns. -# error_retry_period_hours: 24 # Optional. Clear failed event after retry period. +# error_retry_period_hours: 0 # Optional. Default value is 0. Positive value overrides server option # table_name_template: '{{.event_type}}_{{._timestamp.Format "2006_01"}}' #Optional. Default value constant is 'events'. Template for extracting table name # ### BigQuery https://jitsu.com/docs/destinations-configuration/bigquery @@ -144,7 +145,7 @@ destinations: # key_file: /home/eventnative/data/config/bqkey.json # or json string of key e.g. "{"service_account":...}" # data_layout: # max_columns: 100 # Optional. The limit of the count of columns. -# error_retry_period_hours: 24 # Optional. Clear failed event after retry period. +# error_retry_period_hours: 0 # Optional. Default value is 0. Positive value overrides server option # table_name_template: 'my_events' #Optional. Default value constant is 'events'. Template for extracting table name ### Postgres https://jitsu.com/docs/destinations-configuration/postgres @@ -163,7 +164,7 @@ destinations: # connect_timeout: 300 # data_layout: # max_columns: 100 # Optional. The limit of the count of columns. -# error_retry_period_hours: 24 # Optional. Clear failed event after retry period. +# error_retry_period_hours: 0 # Optional. Default value is 0. Positive value overrides server option # table_name_template: 'my_events' #Optional. Default value constant is 'events'. Template for extracting table name # #Required for Postgres users recognition feature. # primary_key_fields: @@ -215,7 +216,7 @@ destinations: # endpoint: #Optional. Default value is AWS s3 endpoint. If you use DigitalOcean spaces or others - specify your endpoint # data_layout: # max_columns: 100 # Optional. The limit of the count of columns. -# error_retry_period_hours: 24 # Optional. Clear failed event after retry period. +# error_retry_period_hours: 0 # Optional. Default value is 0. Positive value overrides server option # table_name_template: '{{.event_type}}_{{._timestamp.Format "2006_01"}}' #Template will be used for file naming ### Snowflake https://jitsu.com/docs/destinations-configuration/snowflake