From ea8f1ae8b36d793fec7cfced887bae38650c0ba6 Mon Sep 17 00:00:00 2001 From: zentol Date: Tue, 23 Jan 2018 14:57:20 +0100 Subject: [PATCH] [FLINK-8475][config][docs] Integrate Checkpointing options --- .../checkpointing_configuration.html | 51 +++++++++++++++++++ docs/ops/config.md | 4 ++ .../configuration/CheckpointingOptions.java | 32 +++++++++--- 3 files changed, 79 insertions(+), 8 deletions(-) create mode 100644 docs/_includes/generated/checkpointing_configuration.html diff --git a/docs/_includes/generated/checkpointing_configuration.html b/docs/_includes/generated/checkpointing_configuration.html new file mode 100644 index 0000000000000..a0cbef560381d --- /dev/null +++ b/docs/_includes/generated/checkpointing_configuration.html @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
KeyDefaultDescription
state.backend
(none)The state backend to be used to store and checkpoint state.
state.backend.async
trueOption whether the state backend should use an asynchronous snapshot method where possible and configurable. Some state backends may not support asynchronous snapshots, or only support asynchronous snapshots, and ignore this option.
state.backend.fs.memory-threshold
1024The minimum size of state data files. All state chunks smaller than that are stored inline in the root checkpoint metadata file.
state.backend.incremental
falseOption whether the state backend should create incremental checkpoints, if possible. For an incremental checkpoint, only a diff from the previous checkpoint is stored, rather than the complete checkpoint state. Some state backends may not support incremental checkpoints and ignore this option.
state.backend.rocksdb.localdir
(none)The local directory (on the TaskManager) where RocksDB puts its files.
state.checkpoints.dir
(none)The default directory used for checkpoints. Used by the state backends that write checkpoints to file systems (MemoryStateBackend, FsStateBackend, RocksDBStateBackend).
state.checkpoints.num-retained
1The maximum number of completed checkpoints to retain.
state.savepoints.dir
(none)The default directory for savepoints. Used by the state backends that write savepoints to file systems (MemoryStateBackend, FsStateBackend, RocksDBStateBackend).
diff --git a/docs/ops/config.md b/docs/ops/config.md index 41f0349572b4d..0a822bbdc3a80 100644 --- a/docs/ops/config.md +++ b/docs/ops/config.md @@ -540,6 +540,10 @@ Previously this key was named `recovery.mode` and the default value was `standal - `env.ssh.opts`: Additional command line options passed to SSH clients when starting or stopping JobManager, TaskManager, and Zookeeper services (start-cluster.sh, stop-cluster.sh, start-zookeeper-quorum.sh, stop-zookeeper-quorum.sh). +### Checkpointing + +{% include generated/checkpointing_configuration.html %} + ### Queryable State {% include generated/queryable_state_configuration.html %} diff --git a/flink-core/src/main/java/org/apache/flink/configuration/CheckpointingOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/CheckpointingOptions.java index 1825ba30eccbd..1175edbd6c494 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/CheckpointingOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/CheckpointingOptions.java @@ -31,12 +31,14 @@ public class CheckpointingOptions { /** The state backend to be used to store and checkpoint state. */ public static final ConfigOption STATE_BACKEND = ConfigOptions .key("state.backend") - .noDefaultValue(); + .noDefaultValue() + .withDescription("The state backend to be used to store and checkpoint state."); /** The maximum number of completed checkpoints to retain.*/ public static final ConfigOption MAX_RETAINED_CHECKPOINTS = ConfigOptions .key("state.checkpoints.num-retained") - .defaultValue(1); + .defaultValue(1) + .withDescription("The maximum number of completed checkpoints to retain."); /** Option whether the state backend should use an asynchronous snapshot method where * possible and configurable. @@ -45,7 +47,10 @@ public class CheckpointingOptions { * asynchronous snapshots, and ignore this option. */ public static final ConfigOption ASYNC_SNAPSHOTS = ConfigOptions .key("state.backend.async") - .defaultValue(true); + .defaultValue(true) + .withDescription("Option whether the state backend should use an asynchronous snapshot method where" + + " possible and configurable. Some state backends may not support asynchronous snapshots, or only support" + + " asynchronous snapshots, and ignore this option."); /** Option whether the state backend should create incremental checkpoints, * if possible. For an incremental checkpoint, only a diff from the previous @@ -55,7 +60,11 @@ public class CheckpointingOptions { * this option.*/ public static final ConfigOption INCREMENTAL_CHECKPOINTS = ConfigOptions .key("state.backend.incremental") - .defaultValue(false); + .defaultValue(false) + .withDescription("Option whether the state backend should create incremental checkpoints, if possible. For" + + " an incremental checkpoint, only a diff from the previous checkpoint is stored, rather than the" + + " complete checkpoint state. Some state backends may not support incremental checkpoints and ignore" + + " this option."); // ------------------------------------------------------------------------ // Options specific to the file-system-based state backends @@ -66,19 +75,25 @@ public class CheckpointingOptions { public static final ConfigOption SAVEPOINT_DIRECTORY = ConfigOptions .key("state.savepoints.dir") .noDefaultValue() - .withDeprecatedKeys("savepoints.state.backend.fs.dir"); + .withDeprecatedKeys("savepoints.state.backend.fs.dir") + .withDescription("The default directory for savepoints. Used by the state backends that write savepoints to" + + " file systems (MemoryStateBackend, FsStateBackend, RocksDBStateBackend)."); /** The default directory used for checkpoints. Used by the state backends that write * checkpoints to file systems (MemoryStateBackend, FsStateBackend, RocksDBStateBackend). */ public static final ConfigOption CHECKPOINTS_DIRECTORY = ConfigOptions .key("state.checkpoints.dir") - .noDefaultValue(); + .noDefaultValue() + .withDescription("The default directory used for checkpoints. Used by the state backends that write" + + " checkpoints to file systems (MemoryStateBackend, FsStateBackend, RocksDBStateBackend)."); /** The minimum size of state data files. All state chunks smaller than that * are stored inline in the root checkpoint metadata file. */ public static final ConfigOption FS_SMALL_FILE_THRESHOLD = ConfigOptions .key("state.backend.fs.memory-threshold") - .defaultValue(1024); + .defaultValue(1024) + .withDescription("The minimum size of state data files. All state chunks smaller than that are stored" + + " inline in the root checkpoint metadata file."); // ------------------------------------------------------------------------ // Options specific to the RocksDB state backend @@ -88,5 +103,6 @@ public class CheckpointingOptions { public static final ConfigOption ROCKSDB_LOCAL_DIRECTORIES = ConfigOptions .key("state.backend.rocksdb.localdir") .noDefaultValue() - .withDeprecatedKeys("state.backend.rocksdb.checkpointdir"); + .withDeprecatedKeys("state.backend.rocksdb.checkpointdir") + .withDescription("The local directory (on the TaskManager) where RocksDB puts its files."); }