diff --git a/README.md b/README.md index 6eacb3ab97a..5342725bc4a 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,12 @@ Check out the GitHub docs for a refresher on [how to create a pull request](http ### Style guidelines -For style guidelines, see ["Style guide"](/contribute/style-guide.md). +For documentation style guidelines, see ["Style guide"](/contribute/style-guide.md). + +### Generating documentation from source code + +For an overview of how reference documentation such as settings, system tables +and functions are generated from the source code, see ["Generating documentation from source code"](/contribute/autogenerated-documentation-from-source.md) ### Tests and CI/CD {#tests-and-cicd} diff --git a/contribute/autogenerated-documentation-from-source.md b/contribute/autogenerated-documentation-from-source.md new file mode 100644 index 00000000000..e8fb55b4b49 --- /dev/null +++ b/contribute/autogenerated-documentation-from-source.md @@ -0,0 +1,48 @@ +# Generating documentation from source code + +A [script in our docs repo](https://github.com/ClickHouse/clickhouse-docs/blob/main/scripts/settings/autogenerate-settings.sh) +extracts setting names, descriptions, default values, etc. from ClickHouse's source code. + +## Session settings + +Documentation for session settings are extracted from [`src/Core/Settings.cpp`](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/Settings.cpp). + +## Format settings + +Documentation for session settings are extracted from [`src/Core/FormatFactorySettings.h`](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/FormatFactorySettings.h). + +## Server settings + +Documentation for session settings are extracted from [`src/Core/ServerSettings.cpp`](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/ServerSettings.cpp). + +Note that this file contains only a fraction of all server settings. +The reason is that server settings can be nested and `system.server_settings` +(which is built from `src/Core/ServerSettings.cpp`) and cannot represent nested +settings. + +Example: + +```yaml + + 1073741824 + 1024 + 1048576 + 30000000 + +``` +As a result of the above, you will find the server settings which are not found +in `system.server_settings` documented in file `_server_settings_outside_source.md` + +The auto-generation script reads these in, combines them with the ones from +`system.settings` and appends the formatted settings to `settings.md`. + +**As such, if you need to make a change to the settings you see on the +[server settings](clickhouse.com/docs/operations/server-configuration-parameters/) +page, you will need to check if the setting is in `system.server_settings`. +If it is, then please edit the setting description in the source code documentation +in [`ServerSettings.cpp`](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/ServerSettings.cpp) +or else edit `_server_settings_outside_source.md`.** + +## System tables + +## Functions diff --git a/package.json b/package.json index 0cf0d5bb870..82cd5f00012 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "private": true, "config": { "clickhouse_repo_folders": "docs/en/development docs/en/engines docs/en/getting-started docs/en/interfaces docs/en/operations docs/en/sql-reference", - "autogen_needed_files": "src/Core/FormatFactorySettings.h src/Core/Settings.cpp CHANGELOG.md" + "autogen_needed_files": "src/Core/FormatFactorySettings.h src/Core/Settings.cpp CHANGELOG.md docs/en/operations/server-configuration-parameters/_server_settings_outside_source.md" }, "scripts": { "build": "yarn copy-clickhouse-repo-docs && yarn run-markdown-linter && yarn generate-changelog && yarn autogenerate-settings && yarn autogenerate-table-of-contents && yarn build-api-doc && yarn build-swagger && yarn build-docs", diff --git a/scripts/settings/autogenerate-settings.sh b/scripts/settings/autogenerate-settings.sh index bc4e97a9c17..f568cda7942 100755 --- a/scripts/settings/autogenerate-settings.sh +++ b/scripts/settings/autogenerate-settings.sh @@ -119,8 +119,85 @@ SELECT prefix || (SELECT groupConcat(*) FROM main_content) INTO OUTFILE 'settings.md' TRUNCATE FORMAT LineAsString " > /dev/null || { echo "Failed to Autogenerate Core settings"; exit 1; } +# Auto generate global server settings +./clickhouse -q " +WITH + server_settings_outside_source AS + ( + SELECT + arrayJoin(extractAllGroups(raw_blob, '## (\\w+)(?:\\s[^\n]+)?\n\\s+((?:[^#]|#[^#]|##[^ ])+)')) AS g, + g[1] AS name, + replaceRegexpAll(replaceRegexpAll(g[2], '\n(Type|Default( value)?): [^\n]+\n', ''), '^\n+|\n+$', '') AS doc + FROM file('_server_settings_outside_source.md', RawBLOB) + ), + server_settings_in_source AS + ( + SELECT + name, + replaceRegexpAll(description, '(?m)^[ \t]+', '') AS description + FROM system.server_settings + ), + combined_server_settings AS + ( + SELECT + name, + description + FROM server_settings_in_source + UNION ALL + SELECT + name, + doc AS description + FROM server_settings_outside_source + ), + formatted_settings AS + ( + SELECT + format('## {} {}\n\n{}\n\n', name, lcase('{#'||name||'}'), description) AS formatted_text + FROM combined_server_settings + ORDER BY name ASC + ), + prefix_text AS + ( + SELECT + '--- +description: ''This section contains descriptions of server settings i.e settings + which cannot be changed at the session or query level.'' +keywords: [''global server settings''] +sidebar_label: ''Server Settings'' +sidebar_position: 57 +slug: /operations/server-configuration-parameters/settings +title: ''Server Settings'' +--- + +import Tabs from ''@theme/Tabs''; +import TabItem from ''@theme/TabItem''; +import SystemLogParameters from ''@site/docs/operations/server-configuration-parameters/_snippets/_system-log-parameters.md'' + +# Server Settings + +This section contains descriptions of server settings. These are settings which +cannot be changed at the session or query level. + +For more information on configuration files in ClickHouse see [""Configuration Files""](/operations/configuration-files). + +Other settings are described in the ""[Settings](/operations/settings/overview)"" section. +Before studying the settings, we recommend reading the [Configuration files](/operations/configuration-files) +section and note the use of substitutions (the `incl` and `optional` attributes). + +' AS prefix_content + ) +SELECT + arrayStringConcat([ + (SELECT prefix_content FROM prefix_text), + arrayStringConcat(groupArray(formatted_text), '') + ], '') +FROM formatted_settings +INTO OUTFILE 'server_settings.md' +TRUNCATE FORMAT LineAsString" > /dev/null || { echo "Failed to Autogenerate Format settings"; exit 1; } + mv settings-formats.md "$root/docs/operations/settings" || { echo "Failed to move generated settings-format.md"; exit 1; } mv settings.md "$root/docs/operations/settings" || { echo "Failed to move generated settings.md"; exit 1; } +mv server_settings.md "$root/docs/operations/server-configuration-parameters/settings.md" || { echo "Failed to move generated server_settings.md"; exit 1; } echo "[$SCRIPT_NAME] Auto-generation of settings markdown pages completed successfully"