From 3eab918ee7645239c9b8fe53c8a350239c2909f4 Mon Sep 17 00:00:00 2001 From: Shaun Struwig <41984034+Blargian@users.noreply.github.com> Date: Thu, 20 Mar 2025 16:01:12 +0100 Subject: [PATCH 1/4] autogenerate global server settings --- package.json | 2 +- scripts/settings/autogenerate-settings.sh | 76 +++++++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 0cf0d5bb870..c92cfb07eb6 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "private": true, "config": { "clickhouse_repo_folders": "docs/en/development docs/en/engines docs/en/getting-started docs/en/interfaces docs/en/operations docs/en/sql-reference", - "autogen_needed_files": "src/Core/FormatFactorySettings.h src/Core/Settings.cpp CHANGELOG.md" + "autogen_needed_files": "src/Core/FormatFactorySettings.h src/Core/Settings.cpp CHANGELOG.md, docs/en/operations/server-configuration-parameters/_server_settings_outside_source.md" }, "scripts": { "build": "yarn copy-clickhouse-repo-docs && yarn run-markdown-linter && yarn generate-changelog && yarn autogenerate-settings && yarn autogenerate-table-of-contents && yarn build-api-doc && yarn build-swagger && yarn build-docs", diff --git a/scripts/settings/autogenerate-settings.sh b/scripts/settings/autogenerate-settings.sh index bc4e97a9c17..ebf47c54cfc 100755 --- a/scripts/settings/autogenerate-settings.sh +++ b/scripts/settings/autogenerate-settings.sh @@ -119,8 +119,84 @@ SELECT prefix || (SELECT groupConcat(*) FROM main_content) INTO OUTFILE 'settings.md' TRUNCATE FORMAT LineAsString " > /dev/null || { echo "Failed to Autogenerate Core settings"; exit 1; } +./clickhouse -q " +WITH + settings_outside_source AS + ( + SELECT + arrayJoin(extractAllGroups(raw_blob, '## (\\w+)(?:\\s[^\n]+)?\n\\s+((?:[^#]|#[^#]|##[^ ])+)')) AS g, + g[1] AS name, + replaceRegexpAll(replaceRegexpAll(g[2], '\n(Type|Default( value)?): [^\n]+\n', ''), '^\n+|\n+$', '') AS doc + FROM file('_server_settings_outside_source.md', RawBLOB) + ), + settings_in_source AS + ( + SELECT + name, + replaceRegexpAll(description, '(?m)^[ \t]+', '') AS description + FROM system.server_settings + ), + combined_settings AS + ( + SELECT + name, + description + FROM settings_in_source + UNION ALL + SELECT + name, + doc AS description + FROM settings_outside_source + ), + formatted_settings AS + ( + SELECT + format('## {} {}\n\n{}\n\n', name, '{#'||name||'}', description) AS formatted_text + FROM combined_settings + ORDER BY name ASC + ), + prefix_text AS + ( + SELECT + '--- +description: ''This section contains descriptions of server settings that cannot be + changed at the session or query level.'' +keywords: [''global server settings''] +sidebar_label: ''Global Server Settings'' +sidebar_position: 57 +slug: /operations/server-configuration-parameters/settings +title: Global Server Settings +--- + +import Tabs from ''@theme/Tabs''; +import TabItem from ''@theme/TabItem''; +import SystemLogParameters from ''@site/docs/operations/server-configuration-parameters/_snippets/_system-log-parameters.md'' + +# Global Server Settings + +This section contains descriptions of server settings that cannot be changed at +the session or query level. These settings are stored in the `config.xml` file +on the ClickHouse server. For more information on configuration files in +ClickHouse see [""Configuration Files""](/operations/configuration-files). + +Other settings are described in the ""[Settings](/operations/settings/overview)"" section. +Before studying the settings, we recommend reading the [Configuration files](/operations/configuration-files) +section and note the use of substitutions (the `incl` and `optional` attributes). + +' AS prefix_content + ) +SELECT + arrayStringConcat([ + (SELECT prefix_content FROM prefix_text), + arrayStringConcat(groupArray(formatted_text), '') + ], '') +FROM formatted_settings +INTO OUTFILE 'server_settings.md' +TRUNCATE FORMAT LineAsString" > /dev/null || { echo "Failed to Autogenerate Format settings"; exit 1; } + mv settings-formats.md "$root/docs/operations/settings" || { echo "Failed to move generated settings-format.md"; exit 1; } mv settings.md "$root/docs/operations/settings" || { echo "Failed to move generated settings.md"; exit 1; } +mv server_settings.md "$root/docs/operations/server-configuration-parameters/settings.md" || { echo "Failed to move generated server_settings.md"; exit 1; } echo "[$SCRIPT_NAME] Auto-generation of settings markdown pages completed successfully" From 819abeadd7fabe10a93f42ece1284805e94b82ad Mon Sep 17 00:00:00 2001 From: Shaun Struwig <41984034+Blargian@users.noreply.github.com> Date: Mon, 24 Mar 2025 10:29:30 +0100 Subject: [PATCH 2/4] incorporate review feedback --- README.md | 7 ++- ...autogenerated-documentation-from-source.md | 48 +++++++++++++++++++ scripts/settings/autogenerate-settings.sh | 30 ++++++------ 3 files changed, 69 insertions(+), 16 deletions(-) create mode 100644 contribute/autogenerated-documentation-from-source.md diff --git a/README.md b/README.md index 6eacb3ab97a..5342725bc4a 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,12 @@ Check out the GitHub docs for a refresher on [how to create a pull request](http ### Style guidelines -For style guidelines, see ["Style guide"](/contribute/style-guide.md). +For documentation style guidelines, see ["Style guide"](/contribute/style-guide.md). + +### Generating documentation from source code + +For an overview of how reference documentation such as settings, system tables +and functions are generated from the source code, see ["Generating documentation from source code"](/contribute/autogenerated-documentation-from-source.md) ### Tests and CI/CD {#tests-and-cicd} diff --git a/contribute/autogenerated-documentation-from-source.md b/contribute/autogenerated-documentation-from-source.md new file mode 100644 index 00000000000..e8fb55b4b49 --- /dev/null +++ b/contribute/autogenerated-documentation-from-source.md @@ -0,0 +1,48 @@ +# Generating documentation from source code + +A [script in our docs repo](https://github.com/ClickHouse/clickhouse-docs/blob/main/scripts/settings/autogenerate-settings.sh) +extracts setting names, descriptions, default values, etc. from ClickHouse's source code. + +## Session settings + +Documentation for session settings are extracted from [`src/Core/Settings.cpp`](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/Settings.cpp). + +## Format settings + +Documentation for session settings are extracted from [`src/Core/FormatFactorySettings.h`](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/FormatFactorySettings.h). + +## Server settings + +Documentation for session settings are extracted from [`src/Core/ServerSettings.cpp`](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/ServerSettings.cpp). + +Note that this file contains only a fraction of all server settings. +The reason is that server settings can be nested and `system.server_settings` +(which is built from `src/Core/ServerSettings.cpp`) and cannot represent nested +settings. + +Example: + +```yaml + + 1073741824 + 1024 + 1048576 + 30000000 + +``` +As a result of the above, you will find the server settings which are not found +in `system.server_settings` documented in file `_server_settings_outside_source.md` + +The auto-generation script reads these in, combines them with the ones from +`system.settings` and appends the formatted settings to `settings.md`. + +**As such, if you need to make a change to the settings you see on the +[server settings](clickhouse.com/docs/operations/server-configuration-parameters/) +page, you will need to check if the setting is in `system.server_settings`. +If it is, then please edit the setting description in the source code documentation +in [`ServerSettings.cpp`](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/ServerSettings.cpp) +or else edit `_server_settings_outside_source.md`.** + +## System tables + +## Functions diff --git a/scripts/settings/autogenerate-settings.sh b/scripts/settings/autogenerate-settings.sh index ebf47c54cfc..4aaf98fe4c4 100755 --- a/scripts/settings/autogenerate-settings.sh +++ b/scripts/settings/autogenerate-settings.sh @@ -121,7 +121,7 @@ INTO OUTFILE 'settings.md' TRUNCATE FORMAT LineAsString ./clickhouse -q " WITH - settings_outside_source AS + server_settings_outside_source AS ( SELECT arrayJoin(extractAllGroups(raw_blob, '## (\\w+)(?:\\s[^\n]+)?\n\\s+((?:[^#]|#[^#]|##[^ ])+)')) AS g, @@ -129,55 +129,55 @@ WITH replaceRegexpAll(replaceRegexpAll(g[2], '\n(Type|Default( value)?): [^\n]+\n', ''), '^\n+|\n+$', '') AS doc FROM file('_server_settings_outside_source.md', RawBLOB) ), - settings_in_source AS + server_settings_in_source AS ( SELECT name, replaceRegexpAll(description, '(?m)^[ \t]+', '') AS description FROM system.server_settings ), - combined_settings AS + combined_server_settings AS ( SELECT name, description - FROM settings_in_source + FROM server_settings_in_source UNION ALL SELECT name, doc AS description - FROM settings_outside_source + FROM server_settings_outside_source ), formatted_settings AS ( SELECT format('## {} {}\n\n{}\n\n', name, '{#'||name||'}', description) AS formatted_text - FROM combined_settings + FROM combined_server_settings ORDER BY name ASC ), prefix_text AS ( SELECT '--- -description: ''This section contains descriptions of server settings that cannot be - changed at the session or query level.'' +description: ''This section contains descriptions of server settings i.e settings + which cannot be changed at the session or query level.'' keywords: [''global server settings''] -sidebar_label: ''Global Server Settings'' +sidebar_label: ''Server Settings'' sidebar_position: 57 slug: /operations/server-configuration-parameters/settings -title: Global Server Settings +title: Server Settings --- import Tabs from ''@theme/Tabs''; import TabItem from ''@theme/TabItem''; import SystemLogParameters from ''@site/docs/operations/server-configuration-parameters/_snippets/_system-log-parameters.md'' -# Global Server Settings +# Server Settings -This section contains descriptions of server settings that cannot be changed at -the session or query level. These settings are stored in the `config.xml` file -on the ClickHouse server. For more information on configuration files in -ClickHouse see [""Configuration Files""](/operations/configuration-files). +This section contains descriptions of server settings. These are settings which +cannot be changed at the session or query level. + +For more information on configuration files in ClickHouse see [""Configuration Files""](/operations/configuration-files). Other settings are described in the ""[Settings](/operations/settings/overview)"" section. Before studying the settings, we recommend reading the [Configuration files](/operations/configuration-files) From c435e91cdbba696604e9180a7b271c08a62927f7 Mon Sep 17 00:00:00 2001 From: Shaun Struwig <41984034+Blargian@users.noreply.github.com> Date: Mon, 24 Mar 2025 11:32:43 +0100 Subject: [PATCH 3/4] fix style --- scripts/settings/autogenerate-settings.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/settings/autogenerate-settings.sh b/scripts/settings/autogenerate-settings.sh index 4aaf98fe4c4..b956566291f 100755 --- a/scripts/settings/autogenerate-settings.sh +++ b/scripts/settings/autogenerate-settings.sh @@ -165,7 +165,7 @@ keywords: [''global server settings''] sidebar_label: ''Server Settings'' sidebar_position: 57 slug: /operations/server-configuration-parameters/settings -title: Server Settings +title: ''Server Settings'' --- import Tabs from ''@theme/Tabs''; From 4b8b07954f7dff7bf6422605d854b088bc411af4 Mon Sep 17 00:00:00 2001 From: Shaun Struwig <41984034+Blargian@users.noreply.github.com> Date: Wed, 26 Mar 2025 12:25:13 +0100 Subject: [PATCH 4/4] fix typo in package.json and lowercase explicit anchor tags in server config sutogen settings --- package.json | 2 +- scripts/settings/autogenerate-settings.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index c92cfb07eb6..82cd5f00012 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "private": true, "config": { "clickhouse_repo_folders": "docs/en/development docs/en/engines docs/en/getting-started docs/en/interfaces docs/en/operations docs/en/sql-reference", - "autogen_needed_files": "src/Core/FormatFactorySettings.h src/Core/Settings.cpp CHANGELOG.md, docs/en/operations/server-configuration-parameters/_server_settings_outside_source.md" + "autogen_needed_files": "src/Core/FormatFactorySettings.h src/Core/Settings.cpp CHANGELOG.md docs/en/operations/server-configuration-parameters/_server_settings_outside_source.md" }, "scripts": { "build": "yarn copy-clickhouse-repo-docs && yarn run-markdown-linter && yarn generate-changelog && yarn autogenerate-settings && yarn autogenerate-table-of-contents && yarn build-api-doc && yarn build-swagger && yarn build-docs", diff --git a/scripts/settings/autogenerate-settings.sh b/scripts/settings/autogenerate-settings.sh index b956566291f..f568cda7942 100755 --- a/scripts/settings/autogenerate-settings.sh +++ b/scripts/settings/autogenerate-settings.sh @@ -119,6 +119,7 @@ SELECT prefix || (SELECT groupConcat(*) FROM main_content) INTO OUTFILE 'settings.md' TRUNCATE FORMAT LineAsString " > /dev/null || { echo "Failed to Autogenerate Core settings"; exit 1; } +# Auto generate global server settings ./clickhouse -q " WITH server_settings_outside_source AS @@ -151,7 +152,7 @@ WITH formatted_settings AS ( SELECT - format('## {} {}\n\n{}\n\n', name, '{#'||name||'}', description) AS formatted_text + format('## {} {}\n\n{}\n\n', name, lcase('{#'||name||'}'), description) AS formatted_text FROM combined_server_settings ORDER BY name ASC ),