Documentation

jitsucom · Sep 18, 2022 · bca2d92 · bca2d92
1 parent 2dd2654
commit bca2d92
Show file tree

Hide file tree

Showing 2 changed files with 45 additions and 6 deletions.
diff --git a/documentation/configuration/error-handling.mdx b/documentation/configuration/error-handling.mdx
@@ -0,0 +1,38 @@
+# Error Handling and Retries
+
+**Jitsu** supports sending events and re-sending them at failures.
+
+##  Configuration
+
+```yaml
+server:
+  ...
+  streaming_retry_delay_minutes: 1 # Optional. Default value is 1
+  error_retry_period_hours: 24 # Optional. Default value is 24
+log:
+  ...
+  rotation_min: 5 # Optional. Default value is 5
+...
+destinations:
+  ...
+  destination_id:
+    ...
+    data_layout:
+      ...
+      # Positive value overrides server option for destination
+      error_retry_period_hours: 0 # Optional. Default value is 0
+```
+
+## Batch mode
+
+**Jitsu** receives a lot of events. All of them are saved into a file system.
+Every `log.rotation_min` minutes **Jitsu** parses saved files with events and tries to send them to destinations.
+If an event was sent correctly it will be archived into a special folder with successful events. Otherwise, **Jitsu** tries to resend it at the next uploading iteration.
+If an event cannot be sent for any reason during `server.error_retry_period_hours` hours **Jitsu** stops resending an event and archives it into a special folder with unsuccessful events.
+
+## Streaming mode
+
+**Jitsu** receives an event and tries to send it to destinations.
+If an event was sent correctly it will be archived into a special folder with successful events. Otherwise, **Jitsu** tries to resend it at the next uploading iteration that happens every `server.streaming_retry_delay_minutes` minutes.
+If an event cannot be sent for any reason during `server.error_retry_period_hours` hours **Jitsu** stops resending an event and archives it into a special folder with unsuccessful events.
+Any destination supports local `destinations.destination_id.data_layout.error_retry_period_hours` option that overrides global `server.error_retry_period_hours` option.
diff --git a/server/appconfig/config.template.yaml b/server/appconfig/config.template.yaml
@@ -32,9 +32,10 @@ server:
   ### Sources reloading. If 'sources' key is http or file:/// URL than it will be reloaded every sources_reload_sec
   #sources_reload_sec: 1 #Optional. Default value is 1.
 
-  # Interval to retry failed event. Optional. Default value is 1
+  # Error handling and retries https://jitsu.com/docs/configuration/error-handling
+  # Interval to retry failed events. Optional. Default value is 1
   streaming_retry_delay_minutes: 1
-  # Clear failed event after retry period. Optional. Default value is 24
+  # Stop resending failed events after retry period. Optional. Default value is 24
   error_retry_period_hours: 24
 
   ### Application metrics
@@ -131,7 +132,7 @@ destinations:
 #          type: varchar(256) #SQL type
 #          column_type: varchar(256) encode zstd
 #      max_columns: 100 # Optional. The limit of the count of columns.
-#      error_retry_period_hours: 24 # Optional. Clear failed event after retry period.
+#      error_retry_period_hours: 0 # Optional. Default value is 0. Positive value overrides server option
 #      table_name_template: '{{.event_type}}_{{._timestamp.Format "2006_01"}}' #Optional. Default value constant is 'events'. Template for extracting table name
 #
    ### BigQuery https://jitsu.com/docs/destinations-configuration/bigquery
@@ -144,7 +145,7 @@ destinations:
 #      key_file: /home/eventnative/data/config/bqkey.json # or json string of key e.g. "{"service_account":...}"
 #    data_layout:
 #      max_columns: 100 # Optional. The limit of the count of columns.
-#      error_retry_period_hours: 24 # Optional. Clear failed event after retry period.
+#      error_retry_period_hours: 0 # Optional. Default value is 0. Positive value overrides server option
 #      table_name_template: 'my_events' #Optional. Default value constant is 'events'. Template for extracting table name
 
   ### Postgres https://jitsu.com/docs/destinations-configuration/postgres
@@ -163,7 +164,7 @@ destinations:
 #        connect_timeout: 300
 #    data_layout:
 #      max_columns: 100 # Optional. The limit of the count of columns.
-#      error_retry_period_hours: 24 # Optional. Clear failed event after retry period.
+#      error_retry_period_hours: 0 # Optional. Default value is 0. Positive value overrides server option
 #      table_name_template: 'my_events' #Optional. Default value constant is 'events'. Template for extracting table name
 #      #Required for Postgres users recognition feature.
 #      primary_key_fields:
@@ -215,7 +216,7 @@ destinations:
 #      endpoint: #Optional. Default value is AWS s3 endpoint. If you use DigitalOcean spaces or others - specify your endpoint
 #    data_layout:
 #      max_columns: 100 # Optional. The limit of the count of columns.
-#      error_retry_period_hours: 24 # Optional. Clear failed event after retry period.
+#      error_retry_period_hours: 0 # Optional. Default value is 0. Positive value overrides server option
 #      table_name_template: '{{.event_type}}_{{._timestamp.Format "2006_01"}}' #Template will be used for file naming
 
   ### Snowflake https://jitsu.com/docs/destinations-configuration/snowflake