diff --git a/.circleci/config.yml b/.circleci/config.yml index c473d1b24eb25..f8dd032727d69 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -55,7 +55,7 @@ jobs: command: | sudo bash -c 'echo "222.222.95.49 uc.qbox.me" >> /etc/hosts'; if [ "${CIRCLE_BRANCH}" == "master" ]; then - python3 scripts/upload.py output.pdf tidb-manual-en-dev.pdf; + python3 scripts/upload.py output.pdf tidb-dev-en-manual.pdf; fi - run: diff --git a/.github/assign-by-files.yml b/.github/assign-by-files.yml new file mode 100644 index 0000000000000..3314d000052ac --- /dev/null +++ b/.github/assign-by-files.yml @@ -0,0 +1,111 @@ +--- +# Changing all md files should request a language review in case of format failures. +# TomShawn is the repo administrator. + +# you can use glob pattern + +# You can set multiple reviewers +'.github/**/*.yml': + - yikeke + - ran-huang + +# 'benchmark/**/*.md': +# - jackysp +# - kissmydb +# - TomShawn + +'best-practices/**/*.md': + - kissmydb + - lilin90 + - TomShawn + +'br/**/*.md': + - kissmydb + - WangXiangUSTC + - TomShawn + +'config-templates/**/*.yaml': + - kissmydb + - superlzs0476 + - TomShawn + +'dashboard/**/*.md': + - breeswish + - hundundm + - kissmydb + - TomShawn + +'faq/**/*.md': + - kissmydb + - TomShawn + +# 'functions-and-operators/**/*.md': +# - kissmydb +# - TomShawn + +# 'media/**/*': +# - TomShawn +# - lilin90 + +'releases/**/*.md': + - scsldb + - uglyengineer + - TomShawn + +'resources/**/*': + - yikeke + - lilin90 + +'scripts/**/*': + - yikeke + - lilin90 + +'sql-statements/**/*.md': + - bb7133 + - zimulala + - TomShawn + +# 'storage-engine/**/*.md': +# - TomShawn +# - + +# 'sync-diff-inspector/**/*.md': +# - TomShawn +# - + +# 'system-tables/**/*.md': +# - TomShawn +# - + +'ticdc/**/*.md': + - WangXiangUSTC + - 3pointer + - kissmydb + - TomShawn + +'tidb-binlog/**/*.md': + - july2993 + - kissmydb + - TomShawn + +'tidb-lightning/**/*.md': + - WangXiangUSTC + - kissmydb + - TomShawn + +'tiflash/**/*.md': + - ilovesoup + - zanmato1984 + - kissmydb + - yikeke + +'tiup/**/*.md': + - lonng + - lucklove + - kissmydb + - yikeke + +# All other file pattern will match the following: + +'**/*': + - TomShawn diff --git a/.github/auto-label.json b/.github/auto-label.json new file mode 100644 index 0000000000000..d00f6973330ae --- /dev/null +++ b/.github/auto-label.json @@ -0,0 +1,6 @@ +{ + "rules": { + "translation/welcome": ["*.md", "*.yml", "*.yaml", "*.json", "media/", "scripts/"], + "status/PTAL": ["*.md", "*.yml", "*.yaml", "*.json", "media/", "scripts/"] + } +} \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 2048bba0e68b2..b29d00efb63e3 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,12 +1,12 @@ - + ### What is changed, added or deleted? (Required) - + ### Which TiDB version(s) do your changes apply to? (Required) - + - [ ] master (the latest development version) - [ ] v4.0 (TiDB 4.0 versions) @@ -14,11 +14,12 @@ - [ ] v3.0 (TiDB 3.0 versions) - [ ] v2.1 (TiDB 2.1 versions) -**If you select two or more versions from above**, to trigger the bot to cherry-pick this PR to your desired release version branch(es), you **must** add corresponding labels such as **needs-cherry-pick-4.0**, **needs-cherry-pick-3.1**, **needs-cherry-pick-3.0**, and **needs-cherry-pick-2.1**. + ### What is the related PR or file link(s)? - + -- This PR is translated from: -- Other reference link(s): +- This PR is translated from: +- Other reference link(s): diff --git a/.github/workflows/assign-to-project.yml b/.github/workflows/assign-to-project.yml new file mode 100644 index 0000000000000..4c31e1b09d5b8 --- /dev/null +++ b/.github/workflows/assign-to-project.yml @@ -0,0 +1,39 @@ +name: Assign to Project + +on: + issues: + types: [opened, labeled] + pull_request: + types: [opened, labeled] +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +jobs: + assign_one_project: + runs-on: ubuntu-latest + name: Assign to Project + steps: + - name: Assign NEW issues to Docs SIG project + uses: srggrs/assign-one-project-github-action@1.2.0 + if: | + github.event_name == 'issues' && + github.event.action == 'opened' + with: + project: 'https://github.com/pingcap/docs/projects/1' + column_name: 'Issue: Backlog' + - name: Assign NEW pull requests to Docs SIG project + uses: srggrs/assign-one-project-github-action@1.2.0 + if: | + github.event_name == 'pull_request' && + github.event.action == 'opened' + with: + project: 'https://github.com/pingcap/docs/projects/1' + column_name: 'PR: In Progress' + - name: Assign issues or PRs with translation-welcome label + uses: srggrs/assign-one-project-github-action@1.2.0 + if: | + contains(github.event.issue.labels.*.name, 'translation/welcome') || + contains(github.event.pull_request.labels.*.name, 'translation/welcome') + with: + project: 'https://github.com/pingcap/docs/projects/2' + column_name: 'To do' diff --git a/.github/workflows/auto-assign.yml b/.github/workflows/auto-assign.yml new file mode 100644 index 0000000000000..17c24bbf30c21 --- /dev/null +++ b/.github/workflows/auto-assign.yml @@ -0,0 +1,15 @@ +name: "Auto Assign" +on: + pull_request: + branches: + - master + types: [opened] + +jobs: + assign_reviewer: + runs-on: ubuntu-latest + steps: + - uses: shufo/auto-assign-reviewer-by-files@v1.1.1 + with: + config: '.github/assign-by-files.yml' + token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/auto-label.yml b/.github/workflows/auto-label.yml new file mode 100644 index 0000000000000..1a498379833ea --- /dev/null +++ b/.github/workflows/auto-label.yml @@ -0,0 +1,16 @@ +name: Auto Label +on: + pull_request: + branches: + - master + types: [opened] + +jobs: + auto-label: + name: Auto Label + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: banyan/auto-label@1.2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index fd9ffe0c2b502..35a883df07476 100644 --- a/.gitignore +++ b/.gitignore @@ -8,5 +8,6 @@ out gen .DS_Store +*.swp /node_modules/ diff --git a/TOC.md b/TOC.md index 9498a94de02ab..c9e469081bd06 100644 --- a/TOC.md +++ b/TOC.md @@ -5,507 +5,521 @@ ## Documentation List -+ Introduction - - [TiDB Introduction](/overview.md) ++ About TiDB + + [TiDB Introduction](/overview.md) + + [What's New in TiDB 4.0](/whats-new-in-tidb-4.0.md) + + [Key Features](/key-features.md) + + Compatibility + + [MySQL Compatibility](/mysql-compatibility.md) + + [TiDB Limitations](/tidb-limitations.md) ++ Quick Start + + [Try Out TiDB](/quick-start-with-tidb.md) + + [Learn TiDB SQL](/basic-sql-operations.md) ++ Deploy + + [Software and Hardware Requirements](/hardware-and-software-requirements.md) + + [Environment Configuration Checklist](/check-before-deployment.md) + + Topology Patterns + + [Minimal Topology](/minimal-deployment-topology.md) + + [TiFlash Topology](/tiflash-deployment-topology.md) + + [TiCDC Topology](/ticdc-deployment-topology.md) + + [TiDB Binlog Topology](/tidb-binlog-deployment-topology.md) + + [Cross-DC Topology](/geo-distributed-deployment-topology.md) + + [Hybrid Topology](/hybrid-deployment-topology.md) + + Install and Start + + Linux OS + + [Use TiUP (Recommended)](/production-deployment-using-tiup.md) + + [Use TiUP Offline (Recommended)](/production-offline-deployment-using-tiup.md) + + [Use TiDB Ansible](/online-deployment-using-ansible.md) + + [Use TiDB Ansible Offline](/offline-deployment-using-ansible.md) + + [Verify Cluster Status](/post-installation-check.md) + Benchmarks - - [How to Test TiDB Using Sysbench](/benchmark/benchmark-tidb-using-sysbench.md) - - [How to Run TPC-C Test on TiDB](/benchmark/benchmark-tidb-using-tpcc.md) - - [Sysbench Performance Test - v3.0 vs. v2.1](/benchmark/v3.0-performance-benchmarking-with-sysbench.md) - - [TPC-C Performance Test - v4.0 vs. v3.0](/benchmark/v4.0-performance-benchmarking-with-tpcc.md) - - [TPC-C Performance Test - v3.0 vs. v2.1](/benchmark/v3.0-performance-benchmarking-with-tpcc.md) - - [Interaction Test on Online Workloads and `ADD INDEX` Operations](/benchmark/online-workloads-and-add-index-operations.md) -- [Quick Start with TiDB](/quick-start-with-tidb.md) -+ Concepts - - [Architecture](/architecture.md) - + Key Features - - [Horizontal Scalability](/key-features.md#horizontal-scalability) - - [MySQL Compatible Syntax](/key-features.md#mysql-compatible-syntax) - - [Replicate from and to MySQL](/key-features.md#replicate-from-and-to-mysql) - - [Distributed Transactions with Strong Consistency](/key-features.md#distributed-transactions-with-strong-consistency) - - [Cloud Native Architecture](/key-features.md#cloud-native-architecture) - - [Minimize ETL with HTAP](/key-features.md#minimize-etl-with-htap) - - [Fault Tolerance & Recovery with Raft](/key-features.md#fault-tolerance--recovery-with-raft) - - [Automatic Rebalancing](/key-features.md#automatic-rebalancing) - - [Deployment and Orchestration with Ansible, Kubernetes, Docker](/key-features.md#deployment-and-orchestration-with-ansible-kubernetes-docker) - - [JSON Support](/key-features.md#json-support) - - [Spark Integration](/key-features.md#spark-integration) - - [Read Historical Data Without Restoring from Backup](/key-features.md#read-historical-data-without-restoring-from-backup) - - [Fast Import and Restore of Data](/key-features.md#fast-import-and-restore-of-data) - - [Hybrid of Column and Row Storage](/key-features.md#hybrid-of-column-and-row-storage) - - [SQL Plan Management](/key-features.md#sql-plan-management) - - [Open Source](/key-features.md#open-source) - - [Online Schema Changes](/key-features.md#online-schema-changes) -+ How-to - + Get Started - + Start a Cluster - - [From Binary](/deploy-tidb-from-binary.md) - - [From Homebrew](/deploy-tidb-from-homebrew.md) - - [From DBdeployer](/deploy-tidb-from-dbdeployer.md) - - [From Docker Compose](/deploy-test-cluster-using-docker-compose.md) - - [Explore SQL with TiDB](/basic-sql-operations.md) - - [Import Example Database](/import-example-data.md) - - [Read Historical Data](/read-historical-data.md) - - [TiDB Binlog Tutorial](/get-started-with-tidb-binlog.md) - - [TiDB Data Migration Tutorial](https://pingcap.com/docs/tidb-data-migration/stable/get-started/) - - [TiDB Lightning Tutorial](/get-started-with-tidb-lightning.md) - - [TiSpark Quick Start Guide](/get-started-with-tispark.md) - + Deploy - - [Hardware Recommendations](/hardware-and-software-requirements.md) - + From Binary Tarball - - [For Testing Environments](/test-deployment-from-binary-tarball.md) - - [For Production Environments](/production-deployment-from-binary-tarball.md) - + Orchestrated Deployment - - [Use TiUP (Recommended)](/production-deployment-using-tiup.md) - - [Use TiDB Ansible](/online-deployment-using-ansible.md) - - [Use TiDB Ansible Offline](/offline-deployment-using-ansible.md) - - [Use Docker](/test-deployment-using-docker.md) - + Geographic Redundancy - - [Overview](/geo-redundancy-deployment.md) - - [Configure Location Awareness](/location-awareness.md) - - [Data Migration with Ansible](https://pingcap.com/docs/tidb-data-migration/stable/deploy-a-dm-cluster-using-ansible/) - + Configure - - [Time Zone](/configure-time-zone.md) - - [Memory Control](/configure-memory-usage.md) - - [Placement Rules](/configure-placement-rules.md) - + Secure - + Transport Layer Security (TLS) - - [Enable TLS For MySQL Clients](/encrypted-connections-with-tls-protocols.md) - - [Enable TLS Between TiDB Components](/enable-tls-between-components.md) - - [Generate Self-signed Certificates](/generate-self-signed-certificates.md) - + Monitor - - [Overview](/tidb-monitoring-framework.md) - - [Monitor a TiDB Cluster](/monitor-a-tidb-cluster.md) - + Migrate - - [Migration Tools User Guide](/ecosystem-tool-user-guide.md) - + Migrate from MySQL - - [Migration Case of Amazon Aurora](/migrate-from-aurora-mysql-database.md) - - [Migrate from CSV](/tidb-lightning/migrate-from-csv-using-tidb-lightning.md) - + Maintain - - [Common TiUP Operations](/maintain-tidb-using-tiup.md) - - [Common Ansible Operations](/maintain-tidb-using-ansible.md) - + Backup and Restore - - [Use Mydumper and TiDB Lightning](/backup-and-restore-using-mydumper-lightning.md) - - [Use BR](/br/backup-and-restore-tool.md) - - [BR Usage Scenarios](/br/backup-and-restore-use-cases.md) - + Identify Abnormal Queries - - [Identify Slow Queries](/identify-slow-queries.md) - - [Identify Expensive Queries](/identify-expensive-queries.md) - + Scale - - [Scale using Ansible](/scale-tidb-using-ansible.md) - - [Scale using TiUP](/scale-tidb-using-tiup.md) + + [Test TiDB Using Sysbench](/benchmark/benchmark-tidb-using-sysbench.md) + + [Run TPC-C Test on TiDB](/benchmark/benchmark-tidb-using-tpcc.md) + + [v4.0 Sysbench Performance Test](/benchmark/benchmark-sysbench-v4-vs-v3.md) + + [v4.0 TPC-H Performance Test](/benchmark/v4.0-performance-benchmarking-with-tpch.md) + + [v4.0 TPC-C Performance Test](/benchmark/v4.0-performance-benchmarking-with-tpcc.md) + + [v3.0 Sysbench Performance Test](/benchmark/v3.0-performance-benchmarking-with-sysbench.md) + + [v3.0 TPC-C Performance Test](/benchmark/v3.0-performance-benchmarking-with-tpcc.md) + + [Interaction Test on Online Workloads and `ADD INDEX`](/benchmark/online-workloads-and-add-index-operations.md) ++ Migrate + + [Overview](/migration-overview.md) + + Migrate from MySQL + + [Migrate from Mydumper Files](/migrate-from-mysql-mydumper-files.md) + + [Migrate from Amazon Aurora MySQL Using DM](/migrate-from-aurora-mysql-database.md) + + Migrate from CSV Files + + [Use TiDB Lightning](/tidb-lightning/migrate-from-csv-using-tidb-lightning.md) + + [Use `LOAD DATA` Statement](/sql-statements/sql-statement-load-data.md) + + [Migrate from SQL Files](/migrate-from-mysql-mydumper-files.md) ++ Maintain + Upgrade - - [Upgrade Using TiUP (Recommended)](/upgrade-tidb-using-tiup.md) - - [Upgrade Using TiDB Ansible](/upgrade-tidb-using-ansible.md) - - Troubleshoot - - [TiDB Troubleshooting Map](/tidb-troubleshooting-map.md) - - [Troubleshoot Cluster Setup](/troubleshoot-tidb-cluster.md) - - [Troubleshoot TiDB Lightning](/troubleshoot-tidb-lightning.md) -+ Reference - + SQL - - [MySQL Compatibility](/mysql-compatibility.md) - + SQL Language Structure - - [Literal Values](/literal-values.md) - - [Schema Object Names](/schema-object-names.md) - - [Keywords and Reserved Words](/keywords-and-reserved-words.md) - - [User-Defined Variables](/user-defined-variables.md) - - [Expression Syntax](/expression-syntax.md) - - [Comment Syntax](/comment-syntax.md) - + Attributes - - [`AUTO_RANDOM`](/auto-random.md) - + Data Types - - [Overview](/data-type-overview.md) - - [Default Values](/data-type-default-values.md) - + Numeric Types - - [`BIT`](/data-type-numeric.md#bit-type) - - [`BOOL|BOOLEAN`](/data-type-numeric.md#boolean-type) - - [`TINYINT`](/data-type-numeric.md#tinyint-type) - - [`SMALLINT`](/data-type-numeric.md#smallint-type) - - [`MEDIUMINT`](/data-type-numeric.md#mediumint-type) - - [`INT|INTEGER`](/data-type-numeric.md#integer-type) - - [`BIGINT`](/data-type-numeric.md#bigint-type) - - [`FLOAT`](/data-type-numeric.md#float-type) - - [`DOUBLE`](/data-type-numeric.md#double-type) - - [`DECIMAL`](/data-type-numeric.md#decimal-type) - + Date and Time Types - - [`DATE`](/data-type-date-and-time.md#date-type) - - [`TIME`](/data-type-date-and-time.md#time-type) - - [`DATETIME`](/data-type-date-and-time.md#datetime-type) - - [`TIMESTAMP`](/data-type-date-and-time.md#timestamp-type) - - [`YEAR`](/data-type-date-and-time.md#year-type) - + String Types - - [`CHAR`](/data-type-string.md#char-type) - - [`VARCHAR`](/data-type-string.md#varchar-type) - - [`TEXT`](/data-type-string.md#text-type) - - [`TINYTEXT`](/data-type-string.md#tinytext-type) - - [`MEDIUMTEXT`](/data-type-string.md#text-type) - - [`LONGTEXT`](/data-type-string.md#longtext-type) - - [`BINARY`](/data-type-string.md#binary-type) - - [`VARBINARY`](/data-type-string.md#varbinary-type) - - [`BLOB`](/data-type-string.md#blob-type) - - [`TINYBLOB`](/data-type-string.md#tinyblob-type) - - [`MEDIUMBLOB`](/data-type-string.md#mediumblob-type) - - [`LONGBLOB`](/data-type-string.md#longblob-type) - - [`ENUM`](/data-type-string.md#enum-type) - - [`SET`](/data-type-string.md#set-type) - - [JSON Type](/data-type-json.md) - + Functions and Operators - - [Function and Operator Reference](/functions-and-operators/functions-and-operators-overview.md) - - [Type Conversion in Expression Evaluation](/functions-and-operators/type-conversion-in-expression-evaluation.md) - - [Operators](/functions-and-operators/operators.md) - - [Control Flow Functions](/functions-and-operators/control-flow-functions.md) - - [String Functions](/functions-and-operators/string-functions.md) - - [Numeric Functions and Operators](/functions-and-operators/numeric-functions-and-operators.md) - - [Date and Time Functions](/functions-and-operators/date-and-time-functions.md) - - [Bit Functions and Operators](/functions-and-operators/bit-functions-and-operators.md) - - [Cast Functions and Operators](/functions-and-operators/cast-functions-and-operators.md) - - [Encryption and Compression Functions](/functions-and-operators/encryption-and-compression-functions.md) - - [Information Functions](/functions-and-operators/information-functions.md) - - [JSON Functions](/functions-and-operators/json-functions.md) - - [Aggregate (GROUP BY) Functions](/functions-and-operators/aggregate-group-by-functions.md) - - [Window Functions](/functions-and-operators/window-functions.md) - - [Miscellaneous Functions](/functions-and-operators/miscellaneous-functions.md) - - [Precision Math](/functions-and-operators/precision-math.md) - - [List of Expressions for Pushdown](/functions-and-operators/expressions-pushed-down.md) - + SQL Statements - - [`ADD COLUMN`](/sql-statements/sql-statement-add-column.md) - - [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) - - [`ADMIN`](/sql-statements/sql-statement-admin.md) - - [`ALTER DATABASE`](/sql-statements/sql-statement-alter-database.md) - - [`ALTER INSTANCE`](/sql-statements/sql-statement-alter-instance.md) - - [`ALTER TABLE`](/sql-statements/sql-statement-alter-table.md) - - [`ALTER USER`](/sql-statements/sql-statement-alter-user.md) - - [`ANALYZE TABLE`](/sql-statements/sql-statement-analyze-table.md) - - [`BEGIN`](/sql-statements/sql-statement-begin.md) - - [`COMMIT`](/sql-statements/sql-statement-commit.md) - - [`CREATE DATABASE`](/sql-statements/sql-statement-create-database.md) - - [`CREATE INDEX`](/sql-statements/sql-statement-create-index.md) - - [`CREATE SEQUENCE`](/sql-statements/sql-statement-create-sequence.md) - - [`CREATE TABLE LIKE`](/sql-statements/sql-statement-create-table-like.md) - - [`CREATE TABLE`](/sql-statements/sql-statement-create-table.md) - - [`CREATE USER`](/sql-statements/sql-statement-create-user.md) - - [`CREATE VIEW`](/sql-statements/sql-statement-create-view.md) - - [`DEALLOCATE`](/sql-statements/sql-statement-deallocate.md) - - [`DELETE`](/sql-statements/sql-statement-delete.md) - - [`DESC`](/sql-statements/sql-statement-desc.md) - - [`DESCRIBE`](/sql-statements/sql-statement-describe.md) - - [`DO`](/sql-statements/sql-statement-do.md) - - [`DROP COLUMN`](/sql-statements/sql-statement-drop-column.md) - - [`DROP DATABASE`](/sql-statements/sql-statement-drop-database.md) - - [`DROP INDEX`](/sql-statements/sql-statement-drop-index.md) - - [`DROP SEQUENCE`](/sql-statements/sql-statement-drop-sequence.md) - - [`DROP TABLE`](/sql-statements/sql-statement-drop-table.md) - - [`DROP USER`](/sql-statements/sql-statement-drop-user.md) - - [`DROP VIEW`](/sql-statements/sql-statement-drop-view.md) - - [`EXECUTE`](/sql-statements/sql-statement-execute.md) - - [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md) - - [`EXPLAIN`](/sql-statements/sql-statement-explain.md) - - [`FLASHBACK TABLE`](/sql-statements/sql-statement-flashback-table.md) - - [`FLUSH PRIVILEGES`](/sql-statements/sql-statement-flush-privileges.md) - - [`FLUSH STATUS`](/sql-statements/sql-statement-flush-status.md) - - [`FLUSH TABLES`](/sql-statements/sql-statement-flush-tables.md) - - [`GRANT `](/sql-statements/sql-statement-grant-privileges.md) - - [`INSERT`](/sql-statements/sql-statement-insert.md) - - [`KILL [TIDB]`](/sql-statements/sql-statement-kill.md) - - [`LOAD DATA`](/sql-statements/sql-statement-load-data.md) - - [`MODIFY COLUMN`](/sql-statements/sql-statement-modify-column.md) - - [`PREPARE`](/sql-statements/sql-statement-prepare.md) - - [`RECOVER TABLE`](/sql-statements/sql-statement-recover-table.md) - - [`RENAME INDEX`](/sql-statements/sql-statement-rename-index.md) - - [`RENAME TABLE`](/sql-statements/sql-statement-rename-table.md) - - [`REPLACE`](/sql-statements/sql-statement-replace.md) - - [`REVOKE `](/sql-statements/sql-statement-revoke-privileges.md) - - [`ROLLBACK`](/sql-statements/sql-statement-rollback.md) - - [`SELECT`](/sql-statements/sql-statement-select.md) - - [`SET [NAMES|CHARACTER SET]`](/sql-statements/sql-statement-set-names.md) - - [`SET PASSWORD`](/sql-statements/sql-statement-set-password.md) - - [`SET TRANSACTION`](/sql-statements/sql-statement-set-transaction.md) - - [`SET [GLOBAL|SESSION] `](/sql-statements/sql-statement-set-variable.md) - - [`SHOW CHARACTER SET`](/sql-statements/sql-statement-show-character-set.md) - - [`SHOW COLLATION`](/sql-statements/sql-statement-show-collation.md) - - [`SHOW [FULL] COLUMNS FROM`](/sql-statements/sql-statement-show-columns-from.md) - - [`SHOW CREATE SEQUENCE`](/sql-statements/sql-statement-show-create-sequence.md) - - [`SHOW CREATE TABLE`](/sql-statements/sql-statement-show-create-table.md) - - [`SHOW CREATE USER`](/sql-statements/sql-statement-show-create-user.md) - - [`SHOW DATABASES`](/sql-statements/sql-statement-show-databases.md) - - [`SHOW ENGINES`](/sql-statements/sql-statement-show-engines.md) - - [`SHOW ERRORS`](/sql-statements/sql-statement-show-errors.md) - - [`SHOW [FULL] FIELDS FROM`](/sql-statements/sql-statement-show-fields-from.md) - - [`SHOW GRANTS`](/sql-statements/sql-statement-show-grants.md) - - [`SHOW INDEXES [FROM|IN]`](/sql-statements/sql-statement-show-indexes.md) - - [`SHOW INDEX [FROM|IN]`](/sql-statements/sql-statement-show-index.md) - - [`SHOW KEYS [FROM|IN]`](/sql-statements/sql-statement-show-keys.md) - - [`SHOW PRIVILEGES`](/sql-statements/sql-statement-show-privileges.md) - - [`SHOW [FULL] PROCESSSLIST`](/sql-statements/sql-statement-show-processlist.md) - - [`SHOW SCHEMAS`](/sql-statements/sql-statement-show-schemas.md) - - [`SHOW [FULL] TABLES`](/sql-statements/sql-statement-show-tables.md) - - [`SHOW TABLE REGIONS`](/sql-statements/sql-statement-show-table-regions.md) - - [`SHOW TABLE STATUS`](/sql-statements/sql-statement-show-table-status.md) - - [`SHOW [GLOBAL|SESSION] VARIABLES`](/sql-statements/sql-statement-show-variables.md) - - [`SHOW WARNINGS`](/sql-statements/sql-statement-show-warnings.md) - - [`SPLIT REGION`](/sql-statements/sql-statement-split-region.md) - - [`START TRANSACTION`](/sql-statements/sql-statement-start-transaction.md) - - [`TRACE`](/sql-statements/sql-statement-trace.md) - - [`TRUNCATE`](/sql-statements/sql-statement-truncate.md) - - [`UPDATE`](/sql-statements/sql-statement-update.md) - - [`USE`](/sql-statements/sql-statement-use.md) - - [Constraints](/constraints.md) - - [Generated Columns](/generated-columns.md) - - [Partitioning](/partitioned-table.md) - - [Character Set and Collation](/character-set-and-collation.md) - - [SQL Mode](/sql-mode.md) - - [SQL Diagnosis](/system-tables/system-table-sql-diagnosis.md) - - [Views](/views.md) - + Configuration - + tidb-server - - [MySQL System Variables](/system-variables.md) - - [TiDB Specific System Variables](/tidb-specific-system-variables.md) - - [Configuration Flags](/command-line-flags-for-tidb-configuration.md) - - [Configuration File](/tidb-configuration-file.md) - + pd-server - - [Configuration Flags](/command-line-flags-for-pd-configuration.md) - - [Configuration File](/pd-configuration-file.md) - + tikv-server - - [Configuration Flags](/command-line-flags-for-tikv-configuration.md) - - [Configuration File](/tikv-configuration-file.md) - + Security - - [Security Compatibility with MySQL](/security-compatibility-with-mysql.md) - - [The TiDB Access Privilege System](/privilege-management.md) - - [TiDB User Account Management](/user-account-management.md) - - [Role-Based Access Control](/role-based-access-control.md) - - [Certificate-Based Authentication](/certificate-authentication.md) - + Transactions - - [Overview](/transaction-overview.md) - - [Isolation Levels](/transaction-isolation-levels.md) - - [Optimistic Transactions](/optimistic-transaction.md) - - [Pessimistic Transactions](/pessimistic-transaction.md) - + System Databases - - [`mysql`](/system-tables/system-table-overview.md) - - [`information_schema`](/system-tables/system-table-information-schema.md) - + `sql-diagnosis` - - [`cluster_info`](/system-tables/system-table-cluster-info.md) - - [`cluster_hardware`](/system-tables/system-table-cluster-hardware.md) - - [`cluster_config`](/system-tables/system-table-cluster-config.md) - - [`cluster_load`](/system-tables/system-table-cluster-load.md) - - [`cluster_systeminfo`](/system-tables/system-table-cluster-systeminfo.md) - - [`cluster_log`](/system-tables/system-table-cluster-log.md) - - [`metrics_schema`](/system-tables/system-table-metrics-schema.md) - - [`metrics_tables`](/system-tables/system-table-metrics-tables.md) - - [`metrics_summary`](/system-tables/system-table-metrics-summary.md) - - [`inspection_result`](/system-tables/system-table-inspection-result.md) - - [`inspection_summary`](/system-tables/system-table-inspection-summary.md) - - [Errors Codes](/error-codes.md) - - [Supported Client Drivers](/connectors-and-apis.md) - + Garbage Collection (GC) - - [GC Overview](/garbage-collection-overview.md) - - [GC Configuration](/garbage-collection-configuration.md) - + Performance - - [Overview](/sql-optimization-concepts.md) - - [Understanding the Query Execution Plan](/query-execution-plan.md) - - [Introduction to Join Reorder](/join-reorder.md) - - [Introduction to Statistics](/statistics.md) - - [Optimizer Hints](/optimizer-hints.md) - - [Follower Read](/follower-read.md) - - [Check the TiDB Cluster Status Using SQL Statements](/check-cluster-status-using-sql-statements.md) - - [Execution Plan Binding](/execution-plan-binding.md) - - [Access Tables Using `IndexMerge`](/index-merge.md) - - [Statement Summary Table](/statement-summary-tables.md) - - [Tune TiKV](/tune-tikv-performance.md) - + Key Monitoring Metrics - - [Overview](/grafana-overview-dashboard.md) - - [TiDB](/grafana-tidb-dashboard.md) - - [PD](/grafana-pd-dashboard.md) - - [TiKV](/grafana-tikv-dashboard.md) - - [Alert Rules](/alert-rules.md) + + [Use TiUP (Recommended)](/upgrade-tidb-using-tiup.md) + + [Use TiUP Offline (Recommended)](/upgrade-tidb-using-tiup-offline.md) + + [Use TiDB Operator](https://pingcap.com/docs/tidb-in-kubernetes/stable/upgrade-a-tidb-cluster/) + + [Use TiDB Ansible](/upgrade-tidb-using-ansible.md) + + Scale + + [Use TiUP (Recommended)](/scale-tidb-using-tiup.md) + + [Use TiDB Ansible](/scale-tidb-using-ansible.md) + + [Use TiDB Operator](https://pingcap.com/docs/tidb-in-kubernetes/stable/scale-a-tidb-cluster/) + + Backup and Restore + + [Use Mydumper and TiDB Lightning](/backup-and-restore-using-mydumper-lightning.md) + + [Use Dumpling for Export or Backup](/export-or-backup-using-dumpling.md) + + Use BR Tool + + [Use BR Tool](/br/backup-and-restore-tool.md) + + [BR Use Cases](/br/backup-and-restore-use-cases.md) + + [BR storages](/br/backup-and-restore-storages.md) + + [Daily Checklist](/daily-check.md) + + [Manage TiCDC Cluster and Replication Tasks](/ticdc/manage-ticdc.md) + + [Maintain TiFlash](/tiflash/maintain-tiflash.md) + + [Maintain TiDB Using TiUP](/maintain-tidb-using-tiup.md) + + [Maintain TiDB Using Ansible](/maintain-tidb-using-ansible.md) ++ Monitor and Alert + + [Monitoring Framework Overview](/tidb-monitoring-framework.md) + + [Monitoring API](/tidb-monitoring-api.md) + + [Deploy Monitoring Services](/deploy-monitoring-services.md) + + [TiDB Cluster Alert Rules](/alert-rules.md) + + [TiFlash Alert Rules](/tiflash/tiflash-alert-rules.md) ++ Troubleshoot + + [Identify Slow Queries](/identify-slow-queries.md) + + [SQL Diagnostics](/system-tables/system-table-sql-diagnostics.md) + + [Identify Expensive Queries](/identify-expensive-queries.md) + + [Statement Summary Tables](/statement-summary-tables.md) + + [Troubleshoot Cluster Setup](/troubleshoot-tidb-cluster.md) + + [TiDB Troubleshooting Map](/tidb-troubleshooting-map.md) + + [Troubleshoot TiCDC](/ticdc/troubleshoot-ticdc.md) + + [Troubleshoot TiFlash](/tiflash/troubleshoot-tiflash.md) ++ Performance Tuning + + Software Tuning + + Configuration + + [TiKV Tuning](/tune-tikv-performance.md) + + [TiKV Follower Read](/follower-read.md) + + [TiFlash Tuning](/tiflash/tune-tiflash-performance.md) + + [Coprocessor Cache](/coprocessor-cache.md) + + SQL Tuning + + [SQL Tuning with `EXPLAIN`](/query-execution-plan.md) + + SQL Optimization + + [SQL Optimization Process](/sql-optimization-concepts.md) + + Logic Optimization + + [Join Reorder](/join-reorder.md) + + Physical Optimization + + [Statistics](/statistics.md) + + Control Execution Plan + + [Optimizer Hints](/optimizer-hints.md) + + [SQL Plan Management](/sql-plan-management.md) + + [Access Tables Using `IndexMerge`](/index-merge.md) ++ Tutorials + + [Geo-Redundant Deployment](/geo-redundancy-deployment.md) + Best Practices - - [Highly Concurrent Write Best Practices](/best-practices/high-concurrency-best-practices.md) - - [HAProxy Best Practices](/best-practices/haproxy-best-practices.md) - - [Java Application Development Best Practices](/best-practices/java-app-best-practices.md) - - [PD Scheduling Best Practices](/best-practices/pd-scheduling-best-practices.md) - - [Grafana Best Practices](/best-practices/grafana-monitor-best-practices.md) - - [TiKV Performance Tuning with Massive Regions](/best-practices/massive-regions-best-practices.md) - - [TiSpark](/tispark-overview.md) - + TiFlash - - [Overview](/tiflash/tiflash-overview.md) - - [Deploy a TiFlash Cluster](/tiflash/deploy-tiflash.md) - - [Use TiFlash](/tiflash/use-tiflash.md) - - [Maintain a TiFlash Cluster](/tiflash/maintain-tiflash.md) - - [Monitor TiFlash](/tiflash/monitor-tiflash.md) - - [Scale TiFlash](/scale-tidb-using-tiup.md#scale-out-a-tiflash-node) - - [Upgrade TiFlash Nodes](/tiflash/upgrade-tiflash.md) - - [Configure TiFlash](/tiflash/tiflash-configuration.md) - - [TiFlash Alert Rules](/tiflash/tiflash-alert-rules.md) - - [Tune TiFlash Performance](/tiflash/tune-tiflash-performance.md) - - [FAQ](/tiflash/tiflash-faq.md) + + [Java Application Development](/best-practices/java-app-best-practices.md) + + [Use HAProxy](/best-practices/haproxy-best-practices.md) + + [Highly Concurrent Write](/best-practices/high-concurrency-best-practices.md) + + [Grafana Monitoring](/best-practices/grafana-monitor-best-practices.md) + + [PD Scheduling](/best-practices/pd-scheduling-best-practices.md) + + [TiKV Performance Tuning with Massive Regions](/best-practices/massive-regions-best-practices.md) + + [Use Placement Rules](/configure-placement-rules.md) ++ TiDB Ecosystem Tools + + [Overview](/ecosystem-tool-user-guide.md) + + [Use Cases](/ecosystem-tool-user-case.md) + + [Download](/download-ecosystem-tools.md) + + Backup & Restore (BR) + + [BR FAQ](/br/backup-and-restore-faq.md) + + [Use BR Tool](/br/backup-and-restore-tool.md) + + [BR Use Cases](/br/backup-and-restore-use-cases.md) + TiDB Binlog - - [Overview](/tidb-binlog/tidb-binlog-overview.md) - - [Deploy](/tidb-binlog/deploy-tidb-binlog.md) - - [Maintain](/tidb-binlog/maintain-tidb-binlog-cluster.md) - - [Monitor](/tidb-binlog/monitor-tidb-binlog-cluster.md) + + [Overview](/tidb-binlog/tidb-binlog-overview.md) + + [Deploy](/tidb-binlog/deploy-tidb-binlog.md) + + [Maintain](/tidb-binlog/maintain-tidb-binlog-cluster.md) + [Configure](/tidb-binlog/tidb-binlog-configuration-file.md) - - [Pump](/tidb-binlog/tidb-binlog-configuration-file.md#pump) - - [Drainer](/tidb-binlog/tidb-binlog-configuration-file.md#drainer) - - [Upgrade](/tidb-binlog/upgrade-tidb-binlog.md) - - [Reparo](/tidb-binlog/tidb-binlog-reparo.md) - - [Binlog Slave Client](/tidb-binlog/binlog-slave-client.md) - - [TiDB Binlog Relay Log](/tidb-binlog/tidb-binlog-relay-log.md) - - [Bidirectional Replication Between TiDB Clusters](/tidb-binlog/bidirectional-replication-between-tidb-clusters.md) - - [Glossary](/tidb-binlog/tidb-binlog-glossary.md) + + [Pump](/tidb-binlog/tidb-binlog-configuration-file.md#pump) + + [Drainer](/tidb-binlog/tidb-binlog-configuration-file.md#drainer) + + [Upgrade](/tidb-binlog/upgrade-tidb-binlog.md) + + [Monitor](/tidb-binlog/monitor-tidb-binlog-cluster.md) + + [Reparo](/tidb-binlog/tidb-binlog-reparo.md) + + [binlogctl](/tidb-binlog/binlog-control.md) + + [Binlog Slave Client](/tidb-binlog/binlog-slave-client.md) + + [TiDB Binlog Relay Log](/tidb-binlog/tidb-binlog-relay-log.md) + + [Bidirectional Replication Between TiDB Clusters](/tidb-binlog/bidirectional-replication-between-tidb-clusters.md) + + [Glossary](/tidb-binlog/tidb-binlog-glossary.md) + Troubleshoot - - [Troubleshooting](/tidb-binlog/troubleshoot-tidb-binlog.md) - - [Error Handling](/tidb-binlog/handle-tidb-binlog-errors.md) - - [FAQ](/tidb-binlog/tidb-binlog-faq.md) - + Tools - - [Tools User Guide](/ecosystem-tool-user-guide.md) - - [Mydumper](/mydumper-overview.md) - - [Syncer](/syncer-overview.md) - - [Loader](/loader-overview.md) - - [Data Migration](https://pingcap.com/docs/tidb-data-migration/stable/) - + TiDB Lightning - - [Overview](/tidb-lightning/tidb-lightning-overview.md) - - [Deployment](/tidb-lightning/deploy-tidb-lightning.md) - - [Configuration](/tidb-lightning/tidb-lightning-configuration.md) - - [Checkpoints](/tidb-lightning/tidb-lightning-checkpoints.md) - - [Table Filter](/tidb-lightning/tidb-lightning-table-filter.md) - - [CSV Support](/tidb-lightning/migrate-from-csv-using-tidb-lightning.md) - - [TiDB-backend](/tidb-lightning/tidb-lightning-tidb-backend.md) - - [Web Interface](/tidb-lightning/tidb-lightning-web-interface.md) - - [Monitor](/tidb-lightning/monitor-tidb-lightning.md) - - [Troubleshoot](/troubleshoot-tidb-lightning.md) - - [FAQ](/tidb-lightning/tidb-lightning-faq.md) - - [Glossary](/tidb-lightning/tidb-lightning-glossary.md) - + TiCDC - - [Overview](/ticdc/ticdc-overview.md) - - [Deploy and Use TiCDC](/ticdc/deploy-ticdc.md) - - [Manage TiCDC Cluster and Replication Tasks](/ticdc/manage-ticdc.md) - - [Configure Sink URI](/ticdc/sink-url.md) - - [TiCDC Open Protocol](/ticdc/ticdc-open-protocol.md) - - [Column and DDL Type Codes](/ticdc/column-ddl-type-codes.md) - + sync-diff-inspector - - [Overview](/sync-diff-inspector/sync-diff-inspector-overview.md) - - [Data Check for Tables with Different Schema or Table Names](/sync-diff-inspector/route-diff.md) - - [Data Check in the Sharding Scenario](/sync-diff-inspector/shard-diff.md) - - [Data Check for TiDB Upstream and Downstream Clusters](/sync-diff-inspector/upstream-downstream-diff.md) - + TiUP - - [TiUP Overview](/tiup/tiup-overview.md) - - [Manage Components with Commands](/tiup/manage-tiup-component.md) - - [Quickly Deploy a Local TiDB Cluster](/tiup/tiup-playground.md) - - [Deploy and Maintain an Online TiDB Cluster](/tiup/tiup-cluster.md) - - [Create a Private Mirror](/tiup/tiup-mirrors.md) - - [Package a Component](/tiup/package-tiup-component.md) - - [Stress Test TiDB Using TiUP](/tiup/tiup-bench.md) - - [PD Control](/pd-control.md) - - [PD Recover](/pd-recover.md) - - [TiKV Control](/tikv-control.md) - - [TiDB Control](/tidb-control.md) - - [Download](/download-ecosystem-tools.md) -- [TiDB in Kubernetes](https://pingcap.com/docs/tidb-in-kubernetes/stable/) + + [Troubleshoot](/tidb-binlog/troubleshoot-tidb-binlog.md) + + [Handle Errors](/tidb-binlog/handle-tidb-binlog-errors.md) + + [FAQ](/tidb-binlog/tidb-binlog-faq.md) + + TiDB Lightning + + [Overview](/tidb-lightning/tidb-lightning-overview.md) + + [Tutorial](/get-started-with-tidb-lightning.md) + + [Deploy](/tidb-lightning/deploy-tidb-lightning.md) + + [Configure](/tidb-lightning/tidb-lightning-configuration.md) + + Key Features + + [Checkpoints](/tidb-lightning/tidb-lightning-checkpoints.md) + + [Table Filter](/tidb-lightning/tidb-lightning-table-filter.md) + + [CSV Support](/tidb-lightning/migrate-from-csv-using-tidb-lightning.md) + + [TiDB-backend](/tidb-lightning/tidb-lightning-tidb-backend.md) + + [Web Interface](/tidb-lightning/tidb-lightning-web-interface.md) + + [Monitor](/tidb-lightning/monitor-tidb-lightning.md) + + [Troubleshoot](/troubleshoot-tidb-lightning.md) + + [FAQ](/tidb-lightning/tidb-lightning-faq.md) + + [Glossary](/tidb-lightning/tidb-lightning-glossary.md) + + [TiCDC](/ticdc/ticdc-overview.md) + + sync-diff-inspector + + [Overview](/sync-diff-inspector/sync-diff-inspector-overview.md) + + [Data Check for Tables with Different Schema/Table Names](/sync-diff-inspector/route-diff.md) + + [Data Check in Sharding Scenarios](/sync-diff-inspector/shard-diff.md) + + [Data Check for TiDB Upstream/Downstream Clusters](/sync-diff-inspector/upstream-downstream-diff.md) + + [Loader](/loader-overview.md) + + [Mydumper](/mydumper-overview.md) + + [Syncer](/syncer-overview.md) + + TiSpark + + [Quick Start](/get-started-with-tispark.md) + + [User Guide](/tispark-overview.md) ++ Reference + + [Architecture](/architecture.md) + + Key Monitoring Metrics + + [Overview](/grafana-overview-dashboard.md) + + [TiDB](/grafana-tidb-dashboard.md) + + [PD](/grafana-pd-dashboard.md) + + [TiKV](/grafana-tikv-dashboard.md) + + [TiFlash](/tiflash/monitor-tiflash.md) + + Secure + + [Enable TLS Between TiDB Clients and Servers](/enable-tls-between-clients-and-servers.md) + + [Enable TLS Between TiDB Components](/enable-tls-between-components.md) + + [Generate Self-signed Certificates](/generate-self-signed-certificates.md) + + [Encryption-At-Rest](/encryption-at-rest.md) + + Privileges + + [Security Compatibility with MySQL](/security-compatibility-with-mysql.md) + + [Privilege Management](/privilege-management.md) + + [User Account Management](/user-account-management.md) + + [Role-Based Access Control](/role-based-access-control.md) + + [Certificate-Based Authentication](/certificate-authentication.md) + + SQL + + SQL Language Structure and Syntax + + Attributes + + [AUTO_INCREMENT](/auto-increment.md) + + [AUTO_RANDOM](/auto-random.md) + + [Literal Values](/literal-values.md) + + [Schema Object Names](/schema-object-names.md) + + [Keywords and Reserved Words](/keywords-and-reserved-words.md) + + [User-Defined Variables](/user-defined-variables.md) + + [Expression Syntax](/expression-syntax.md) + + [Comment Syntax](/comment-syntax.md) + + SQL Statements + + [`ADD COLUMN`](/sql-statements/sql-statement-add-column.md) + + [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) + + [`ADMIN`](/sql-statements/sql-statement-admin.md) + + [`ALTER DATABASE`](/sql-statements/sql-statement-alter-database.md) + + [`ALTER INSTANCE`](/sql-statements/sql-statement-alter-instance.md) + + [`ALTER TABLE`](/sql-statements/sql-statement-alter-table.md) + + [`ALTER USER`](/sql-statements/sql-statement-alter-user.md) + + [`ANALYZE TABLE`](/sql-statements/sql-statement-analyze-table.md) + + [`BACKUP`](/sql-statements/sql-statement-backup.md) + + [`BEGIN`](/sql-statements/sql-statement-begin.md) + + [`CHANGE COLUMN`](/sql-statements/sql-statement-change-column.md) + + [`COMMIT`](/sql-statements/sql-statement-commit.md) + + [`CHANGE DRAINER`](/sql-statements/sql-statement-change-drainer.md) + + [`CHANGE PUMP`](/sql-statements/sql-statement-change-pump.md) + + [`CREATE BINDING`](/sql-statements/sql-statement-create-binding.md) + + [`CREATE DATABASE`](/sql-statements/sql-statement-create-database.md) + + [`CREATE INDEX`](/sql-statements/sql-statement-create-index.md) + + [`CREATE SEQUENCE`](/sql-statements/sql-statement-create-sequence.md) + + [`CREATE TABLE LIKE`](/sql-statements/sql-statement-create-table-like.md) + + [`CREATE TABLE`](/sql-statements/sql-statement-create-table.md) + + [`CREATE USER`](/sql-statements/sql-statement-create-user.md) + + [`CREATE VIEW`](/sql-statements/sql-statement-create-view.md) + + [`DEALLOCATE`](/sql-statements/sql-statement-deallocate.md) + + [`DELETE`](/sql-statements/sql-statement-delete.md) + + [`DESC`](/sql-statements/sql-statement-desc.md) + + [`DESCRIBE`](/sql-statements/sql-statement-describe.md) + + [`DO`](/sql-statements/sql-statement-do.md) + + [`DROP BINDING`](/sql-statements/sql-statement-drop-binding.md) + + [`DROP COLUMN`](/sql-statements/sql-statement-drop-column.md) + + [`DROP DATABASE`](/sql-statements/sql-statement-drop-database.md) + + [`DROP INDEX`](/sql-statements/sql-statement-drop-index.md) + + [`DROP SEQUENCE`](/sql-statements/sql-statement-drop-sequence.md) + - [`DROP STATS`](/sql-statements/sql-statement-drop-stats.md) + + [`DROP TABLE`](/sql-statements/sql-statement-drop-table.md) + + [`DROP USER`](/sql-statements/sql-statement-drop-user.md) + + [`DROP VIEW`](/sql-statements/sql-statement-drop-view.md) + + [`EXECUTE`](/sql-statements/sql-statement-execute.md) + + [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md) + + [`EXPLAIN`](/sql-statements/sql-statement-explain.md) + + [`FLASHBACK TABLE`](/sql-statements/sql-statement-flashback-table.md) + + [`FLUSH PRIVILEGES`](/sql-statements/sql-statement-flush-privileges.md) + + [`FLUSH STATUS`](/sql-statements/sql-statement-flush-status.md) + + [`FLUSH TABLES`](/sql-statements/sql-statement-flush-tables.md) + + [`GRANT `](/sql-statements/sql-statement-grant-privileges.md) + + [`INSERT`](/sql-statements/sql-statement-insert.md) + + [`KILL [TIDB]`](/sql-statements/sql-statement-kill.md) + + [`LOAD DATA`](/sql-statements/sql-statement-load-data.md) + + [`LOAD STATS`](/sql-statements/sql-statement-load-stats.md) + + [`MODIFY COLUMN`](/sql-statements/sql-statement-modify-column.md) + + [`PREPARE`](/sql-statements/sql-statement-prepare.md) + + [`RECOVER TABLE`](/sql-statements/sql-statement-recover-table.md) + + [`RENAME INDEX`](/sql-statements/sql-statement-rename-index.md) + + [`RENAME TABLE`](/sql-statements/sql-statement-rename-table.md) + + [`REPLACE`](/sql-statements/sql-statement-replace.md) + + [`RESTORE`](/sql-statements/sql-statement-restore.md) + + [`REVOKE `](/sql-statements/sql-statement-revoke-privileges.md) + + [`ROLLBACK`](/sql-statements/sql-statement-rollback.md) + + [`SELECT`](/sql-statements/sql-statement-select.md) + + [`SET [NAMES|CHARACTER SET]`](/sql-statements/sql-statement-set-names.md) + + [`SET PASSWORD`](/sql-statements/sql-statement-set-password.md) + + [`SET ROLE`](/sql-statements/sql-statement-set-role.md) + + [`SET TRANSACTION`](/sql-statements/sql-statement-set-transaction.md) + + [`SET [GLOBAL|SESSION] `](/sql-statements/sql-statement-set-variable.md) + + [`SHOW ANALYZE STATUS`](/sql-statements/sql-statement-show-analyze-status.md) + + [`SHOW [BACKUPS|RESTORES]`](/sql-statements/sql-statement-show-backups.md) + + [`SHOW BINDINGS`](/sql-statements/sql-statement-show-bindings.md) + + [`SHOW BUILTINS`](/sql-statements/sql-statement-show-builtins.md) + + [`SHOW CHARACTER SET`](/sql-statements/sql-statement-show-character-set.md) + + [`SHOW COLLATION`](/sql-statements/sql-statement-show-collation.md) + + [`SHOW [FULL] COLUMNS FROM`](/sql-statements/sql-statement-show-columns-from.md) + + [`SHOW CONFIG`](/sql-statements/sql-statement-show-config.md) + + [`SHOW CREATE SEQUENCE`](/sql-statements/sql-statement-show-create-sequence.md) + + [`SHOW CREATE TABLE`](/sql-statements/sql-statement-show-create-table.md) + + [`SHOW CREATE USER`](/sql-statements/sql-statement-show-create-user.md) + + [`SHOW DATABASES`](/sql-statements/sql-statement-show-databases.md) + + [`SHOW DRAINER STATUS`](/sql-statements/sql-statement-show-drainer-status.md) + + [`SHOW ENGINES`](/sql-statements/sql-statement-show-engines.md) + + [`SHOW ERRORS`](/sql-statements/sql-statement-show-errors.md) + + [`SHOW [FULL] FIELDS FROM`](/sql-statements/sql-statement-show-fields-from.md) + + [`SHOW GRANTS`](/sql-statements/sql-statement-show-grants.md) + + [`SHOW INDEX [FROM|IN]`](/sql-statements/sql-statement-show-index.md) + + [`SHOW INDEXES [FROM|IN]`](/sql-statements/sql-statement-show-indexes.md) + + [`SHOW KEYS [FROM|IN]`](/sql-statements/sql-statement-show-keys.md) + + [`SHOW MASTER STATUS`](/sql-statements/sql-statement-show-master-status.md) + + [`SHOW PLUGINS`](/sql-statements/sql-statement-show-plugins.md) + + [`SHOW PRIVILEGES`](/sql-statements/sql-statement-show-privileges.md) + + [`SHOW [FULL] PROCESSSLIST`](/sql-statements/sql-statement-show-processlist.md) + + [`SHOW PROFILES`](/sql-statements/sql-statement-show-profiles.md) + + [`SHOW PUMP STATUS`](/sql-statements/sql-statement-show-pump-status.md) + + [`SHOW SCHEMAS`](/sql-statements/sql-statement-show-schemas.md) + + [`SHOW STATS_HISTOGRAMS`](/sql-statements/sql-statement-show-histograms.md) + + [`SHOW STATES_META`](/sql-statements/sql-statement-show-stats-meta.md) + + [`SHOW STATUS`](/sql-statements/sql-statement-show-status.md) + + [`SHOW TABLE NEXT_ROW_ID`](/sql-statements/sql-statement-show-table-next-rowid.md) + + [`SHOW TABLE REGIONS`](/sql-statements/sql-statement-show-table-regions.md) + + [`SHOW TABLE STATUS`](/sql-statements/sql-statement-show-table-status.md) + + [`SHOW [FULL] TABLES`](/sql-statements/sql-statement-show-tables.md) + + [`SHOW [GLOBAL|SESSION] VARIABLES`](/sql-statements/sql-statement-show-variables.md) + + [`SHOW WARNINGS`](/sql-statements/sql-statement-show-warnings.md) + + [`SHUTDOWN`](/sql-statements/sql-statement-shutdown.md) + + [`SPLIT REGION`](/sql-statements/sql-statement-split-region.md) + + [`START TRANSACTION`](/sql-statements/sql-statement-start-transaction.md) + + [`TRACE`](/sql-statements/sql-statement-trace.md) + + [`TRUNCATE`](/sql-statements/sql-statement-truncate.md) + + [`UPDATE`](/sql-statements/sql-statement-update.md) + + [`USE`](/sql-statements/sql-statement-use.md) + + Data Types + + [Overview](/data-type-overview.md) + + [Default Values](/data-type-default-values.md) + + [Numeric Types](/data-type-numeric.md) + + [Date and Time Types](/data-type-date-and-time.md) + + [String Types](/data-type-string.md) + + [JSON Type](/data-type-json.md) + + Functions and Operators + + [Overview](/functions-and-operators/functions-and-operators-overview.md) + + [Type Conversion in Expression Evaluation](/functions-and-operators/type-conversion-in-expression-evaluation.md) + + [Operators](/functions-and-operators/operators.md) + + [Control Flow Functions](/functions-and-operators/control-flow-functions.md) + + [String Functions](/functions-and-operators/string-functions.md) + + [Numeric Functions and Operators](/functions-and-operators/numeric-functions-and-operators.md) + + [Date and Time Functions](/functions-and-operators/date-and-time-functions.md) + + [Bit Functions and Operators](/functions-and-operators/bit-functions-and-operators.md) + + [Cast Functions and Operators](/functions-and-operators/cast-functions-and-operators.md) + + [Encryption and Compression Functions](/functions-and-operators/encryption-and-compression-functions.md) + + [Information Functions](/functions-and-operators/information-functions.md) + + [JSON Functions](/functions-and-operators/json-functions.md) + + [Aggregate (GROUP BY) Functions](/functions-and-operators/aggregate-group-by-functions.md) + + [Window Functions](/functions-and-operators/window-functions.md) + + [Miscellaneous Functions](/functions-and-operators/miscellaneous-functions.md) + + [Precision Math](/functions-and-operators/precision-math.md) + + [List of Expressions for Pushdown](/functions-and-operators/expressions-pushed-down.md) + + [Constraints](/constraints.md) + + [Generated Columns](/generated-columns.md) + + [SQL Mode](/sql-mode.md) + + Transactions + + [Overview](/transaction-overview.md) + + [Isolation Levels](/transaction-isolation-levels.md) + + [Optimistic Transactions](/optimistic-transaction.md) + + [Pessimistic Transactions](/pessimistic-transaction.md) + + Garbage Collection (GC) + + [Overview](/garbage-collection-overview.md) + + [Configuration](/garbage-collection-configuration.md) + + [Views](/views.md) + + [Partitioning](/partitioned-table.md) + + [Character Set and Collation](/character-set-and-collation.md) + + System Tables + + [`mysql`](/system-tables/system-table-overview.md) + + [`information_schema`](/system-tables/system-table-information-schema.md) + + sql-diagnosis + + [`cluster_info`](/system-tables/system-table-cluster-info.md) + + [`cluster_hardware`](/system-tables/system-table-cluster-hardware.md) + + [`cluster_config`](/system-tables/system-table-cluster-config.md) + + [`cluster_load`](/system-tables/system-table-cluster-load.md) + + [`cluster_systeminfo`](/system-tables/system-table-cluster-systeminfo.md) + + [`cluster_log`](/system-tables/system-table-cluster-log.md) + + [`metrics_schema`](/system-tables/system-table-metrics-schema.md) + + [`metrics_tables`](/system-tables/system-table-metrics-tables.md) + + [`metrics_summary`](/system-tables/system-table-metrics-summary.md) + + [`inspection_result`](/system-tables/system-table-inspection-result.md) + + [`inspection_summary`](/system-tables/system-table-inspection-summary.md) + + UI + + TiDB Dashboard + + [Overview](/dashboard/dashboard-intro.md) + + Maintain + + [Deploy](/dashboard/dashboard-ops-deploy.md) + + [Reverse Proxy](/dashboard/dashboard-ops-reverse-proxy.md) + + [Secure](/dashboard/dashboard-ops-security.md) + + [Access](/dashboard/dashboard-access.md) + + [Overview Page](/dashboard/dashboard-overview.md) + + [Cluster Info Page](/dashboard/dashboard-cluster-info.md) + + [Key Visualizer Page](/dashboard/dashboard-key-visualizer.md) + + SQL Statements Analysis + + [SQL Statements Page](/dashboard/dashboard-statement-list.md) + + [SQL Details Page](/dashboard/dashboard-statement-details.md) + + [Slow Queries Page](/dashboard/dashboard-slow-query.md) + + Cluster Diagnostics + + [Access Cluster Diagnostics Page](/dashboard/dashboard-diagnostics-access.md) + + [View Diagnostics Report](/dashboard/dashboard-diagnostics-report.md) + + [Use Diagnostics](/dashboard/dashboard-diagnostics-usage.md) + + [Search Logs Page](/dashboard/dashboard-log-search.md) + + [Profile Instances Page](/dashboard/dashboard-profiling.md) + + [FAQ](/dashboard/dashboard-faq.md) + + CLI + + [tikv-ctl](/tikv-control.md) + + [pd-ctl](/pd-control.md) + + [tidb-ctl](/tidb-control.md) + + [pd-recover](/pd-recover.md) + + Command Line Flags + + [tidb-server](/command-line-flags-for-tidb-configuration.md) + + [tikv-server](/command-line-flags-for-tikv-configuration.md) + + [pd-server](/command-line-flags-for-pd-configuration.md) + + Configuration File Parameters + + [tidb-server](/tidb-configuration-file.md) + + [tikv-server](/tikv-configuration-file.md) + + [tiflash-server](/tiflash/tiflash-configuration.md) + + [pd-server](/pd-configuration-file.md) + + System Variables + + [MySQL System Variables](/system-variables.md) + + [TiDB Specific System Variables](/tidb-specific-system-variables.md) + + Storage Engines + + TiFlash + + [Overview](/tiflash/tiflash-overview.md) + + [Use TiFlash](/tiflash/use-tiflash.md) + + [FAQ](/tiflash/tiflash-faq.md) + + TiUP + + [Overview](/tiup/tiup-overview.md) + + [Manage TiUP Components](/tiup/manage-tiup-component.md) + + TiUP Components + + [tiup-playground](/tiup/tiup-playground.md) + + [tiup-cluster](/tiup/tiup-cluster.md) + + [tiup-mirror](/tiup/tiup-mirrors.md) + + [tiup-bench](/tiup/tiup-bench.md) + + [Errors Codes](/error-codes.md) + + [TiCDC Overview](/ticdc/ticdc-overview.md) + + [TiCDC Open Protocol](/ticdc/ticdc-open-protocol.md) + FAQs - - [TiDB FAQs](/faq/tidb-faq.md) - - [TiDB Lightning FAQs](/tidb-lightning/tidb-lightning-faq.md) - - [Upgrade FAQs](/faq/upgrade-faq.md) -+ Support - - [Support Resources](/support.md) - - [Report an Issue](/report-issue.md) -+ Contribute - - [Contribute to TiDB](/contribute.md#contribute-to-tidb) - - [Improve the Docs](/contribute.md#improve-the-docs) -- [Adopters](/adopters.md) -- [Roadmap](/roadmap.md) -+ [Releases](/releases/release-notes.md) + + [TiDB FAQs](/faq/tidb-faq.md) + + [TiDB Lightning FAQs](/tidb-lightning/tidb-lightning-faq.md) + + [Upgrade FAQs](/faq/upgrade-faq.md) ++ [Glossary](/glossary.md) ++ Release Notes + + [All Releases](/releases/release-notes.md) + v4.0 - - [4.0.0-rc.2](/releases/release-4.0.0-rc.2.md) - - [4.0.0-rc.1](/releases/release-4.0.0-rc.1.md) - - [4.0.0-rc](/releases/release-4.0.0-rc.md) - - [4.0.0-beta.2](/releases/release-4.0.0-beta.2.md) - - [4.0.0-beta.1](/releases/release-4.0.0-beta.1.md) - - [4.0.0-beta](/releases/release-4.0.0-beta.md) + + [4.0.1](/releases/release-4.0.1.md) + + [4.0 GA](/releases/release-4.0-ga.md) + + [4.0.0-rc.2](/releases/release-4.0.0-rc.2.md) + + [4.0.0-rc.1](/releases/release-4.0.0-rc.1.md) + + [4.0.0-rc](/releases/release-4.0.0-rc.md) + + [4.0.0-beta.2](/releases/release-4.0.0-beta.2.md) + + [4.0.0-beta.1](/releases/release-4.0.0-beta.1.md) + + [4.0.0-beta](/releases/release-4.0.0-beta.md) + v3.1 - - [3.1.1](/releases/release-3.1.1.md) - - [3.1.0 GA](/releases/release-3.1.0-ga.md) - - [3.1.0-rc](/releases/release-3.1.0-rc.md) - - [3.1.0-beta.2](/releases/release-3.1.0-beta.2.md) - - [3.1.0-beta.1](/releases/release-3.1.0-beta.1.md) - - [3.1.0-beta](/releases/release-3.1.0-beta.md) + + [3.1.2](/releases/release-3.1.2.md) + + [3.1.1](/releases/release-3.1.1.md) + + [3.1.0 GA](/releases/release-3.1.0-ga.md) + + [3.1.0-rc](/releases/release-3.1.0-rc.md) + + [3.1.0-beta.2](/releases/release-3.1.0-beta.2.md) + + [3.1.0-beta.1](/releases/release-3.1.0-beta.1.md) + + [3.1.0-beta](/releases/release-3.1.0-beta.md) + v3.0 - - [3.0.14](/releases/release-3.0.14.md) - - [3.0.13](/releases/release-3.0.13.md) - - [3.0.12](/releases/release-3.0.12.md) - - [3.0.11](/releases/release-3.0.11.md) - - [3.0.10](/releases/release-3.0.10.md) - - [3.0.9](/releases/release-3.0.9.md) - - [3.0.8](/releases/release-3.0.8.md) - - [3.0.7](/releases/release-3.0.7.md) - - [3.0.6](/releases/release-3.0.6.md) - - [3.0.5](/releases/release-3.0.5.md) - - [3.0.4](/releases/release-3.0.4.md) - - [3.0.3](/releases/release-3.0.3.md) - - [3.0.2](/releases/release-3.0.2.md) - - [3.0.1](/releases/release-3.0.1.md) - - [3.0 GA](/releases/release-3.0-ga.md) - - [3.0.0-rc.3](/releases/release-3.0.0-rc.3.md) - - [3.0.0-rc.2](/releases/release-3.0.0-rc.2.md) - - [3.0.0-rc.1](/releases/release-3.0.0-rc.1.md) - - [3.0.0-beta.1](/releases/release-3.0.0-beta.1.md) - - [3.0.0-beta](/releases/release-3.0-beta.md) + + [3.0.15](/releases/release-3.0.15.md) + + [3.0.14](/releases/release-3.0.14.md) + + [3.0.13](/releases/release-3.0.13.md) + + [3.0.12](/releases/release-3.0.12.md) + + [3.0.11](/releases/release-3.0.11.md) + + [3.0.10](/releases/release-3.0.10.md) + + [3.0.9](/releases/release-3.0.9.md) + + [3.0.8](/releases/release-3.0.8.md) + + [3.0.7](/releases/release-3.0.7.md) + + [3.0.6](/releases/release-3.0.6.md) + + [3.0.5](/releases/release-3.0.5.md) + + [3.0.4](/releases/release-3.0.4.md) + + [3.0.3](/releases/release-3.0.3.md) + + [3.0.2](/releases/release-3.0.2.md) + + [3.0.1](/releases/release-3.0.1.md) + + [3.0 GA](/releases/release-3.0-ga.md) + + [3.0.0-rc.3](/releases/release-3.0.0-rc.3.md) + + [3.0.0-rc.2](/releases/release-3.0.0-rc.2.md) + + [3.0.0-rc.1](/releases/release-3.0.0-rc.1.md) + + [3.0.0-beta.1](/releases/release-3.0.0-beta.1.md) + + [3.0.0-beta](/releases/release-3.0-beta.md) + v2.1 - - [2.1.19](/releases/release-2.1.19.md) - - [2.1.18](/releases/release-2.1.18.md) - - [2.1.17](/releases/release-2.1.17.md) - - [2.1.16](/releases/release-2.1.16.md) - - [2.1.15](/releases/release-2.1.15.md) - - [2.1.14](/releases/release-2.1.14.md) - - [2.1.13](/releases/release-2.1.13.md) - - [2.1.12](/releases/release-2.1.12.md) - - [2.1.11](/releases/release-2.1.11.md) - - [2.1.10](/releases/release-2.1.10.md) - - [2.1.9](/releases/release-2.1.9.md) - - [2.1.8](/releases/release-2.1.8.md) - - [2.1.7](/releases/release-2.1.7.md) - - [2.1.6](/releases/release-2.1.6.md) - - [2.1.5](/releases/release-2.1.5.md) - - [2.1.4](/releases/release-2.1.4.md) - - [2.1.3](/releases/release-2.1.3.md) - - [2.1.2](/releases/release-2.1.2.md) - - [2.1.1](/releases/release-2.1.1.md) - - [2.1 GA](/releases/release-2.1-ga.md) - - [2.1 RC5](/releases/release-2.1-rc.5.md) - - [2.1 RC4](/releases/release-2.1-rc.4.md) - - [2.1 RC3](/releases/release-2.1-rc.3.md) - - [2.1 RC2](/releases/release-2.1-rc.2.md) - - [2.1 RC1](/releases/release-2.1-rc.1.md) - - [2.1 Beta](/releases/release-2.1-beta.md) + + [2.1.19](/releases/release-2.1.19.md) + + [2.1.18](/releases/release-2.1.18.md) + + [2.1.17](/releases/release-2.1.17.md) + + [2.1.16](/releases/release-2.1.16.md) + + [2.1.15](/releases/release-2.1.15.md) + + [2.1.14](/releases/release-2.1.14.md) + + [2.1.13](/releases/release-2.1.13.md) + + [2.1.12](/releases/release-2.1.12.md) + + [2.1.11](/releases/release-2.1.11.md) + + [2.1.10](/releases/release-2.1.10.md) + + [2.1.9](/releases/release-2.1.9.md) + + [2.1.8](/releases/release-2.1.8.md) + + [2.1.7](/releases/release-2.1.7.md) + + [2.1.6](/releases/release-2.1.6.md) + + [2.1.5](/releases/release-2.1.5.md) + + [2.1.4](/releases/release-2.1.4.md) + + [2.1.3](/releases/release-2.1.3.md) + + [2.1.2](/releases/release-2.1.2.md) + + [2.1.1](/releases/release-2.1.1.md) + + [2.1 GA](/releases/release-2.1-ga.md) + + [2.1 RC5](/releases/release-2.1-rc.5.md) + + [2.1 RC4](/releases/release-2.1-rc.4.md) + + [2.1 RC3](/releases/release-2.1-rc.3.md) + + [2.1 RC2](/releases/release-2.1-rc.2.md) + + [2.1 RC1](/releases/release-2.1-rc.1.md) + + [2.1 Beta](/releases/release-2.1-beta.md) + v2.0 - - [2.0.11](/releases/release-2.0.11.md) - - [2.0.10](/releases/release-2.0.10.md) - - [2.0.9](/releases/release-2.0.9.md) - - [2.0.8](/releases/release-2.0.8.md) - - [2.0.7](/releases/release-2.0.7.md) - - [2.0.6](/releases/release-2.0.6.md) - - [2.0.5](/releases/release-2.0.5.md) - - [2.0.4](/releases/release-2.0.4.md) - - [2.0.3](/releases/release-2.0.3.md) - - [2.0.2](/releases/release-2.0.2.md) - - [2.0.1](/releases/release-2.0.1.md) - - [2.0](/releases/release-2.0-ga.md) - - [2.0 RC5](/releases/release-2.0-rc.5.md) - - [2.0 RC4](/releases/release-2.0-rc.4.md) - - [2.0 RC3](/releases/release-2.0-rc.3.md) - - [2.0 RC1](/releases/release-2.0-rc.1.md) - - [1.1 Beta](/releases/release-1.1-beta.md) - - [1.1 Alpha](/releases/release-1.1-alpha.md) + + [2.0.11](/releases/release-2.0.11.md) + + [2.0.10](/releases/release-2.0.10.md) + + [2.0.9](/releases/release-2.0.9.md) + + [2.0.8](/releases/release-2.0.8.md) + + [2.0.7](/releases/release-2.0.7.md) + + [2.0.6](/releases/release-2.0.6.md) + + [2.0.5](/releases/release-2.0.5.md) + + [2.0.4](/releases/release-2.0.4.md) + + [2.0.3](/releases/release-2.0.3.md) + + [2.0.2](/releases/release-2.0.2.md) + + [2.0.1](/releases/release-2.0.1.md) + + [2.0](/releases/release-2.0-ga.md) + + [2.0 RC5](/releases/release-2.0-rc.5.md) + + [2.0 RC4](/releases/release-2.0-rc.4.md) + + [2.0 RC3](/releases/release-2.0-rc.3.md) + + [2.0 RC1](/releases/release-2.0-rc.1.md) + + [1.1 Beta](/releases/release-1.1-beta.md) + + [1.1 Alpha](/releases/release-1.1-alpha.md) + v1.0 - - [1.0.8](/releases/release-1.0.8.md) - - [1.0.7](/releases/release-1.0.7.md) - - [1.0.6](/releases/release-1.0.6.md) - - [1.0.5](/releases/release-1.0.5.md) - - [1.0.4](/releases/release-1.0.4.md) - - [1.0.3](/releases/release-1.0.3.md) - - [1.0.2](/releases/release-1.0.2.md) - - [1.0.1](/releases/release-1.0.1.md) - - [1.0](/releases/release-1.0-ga.md) - - [Pre-GA](/releases/release-pre-ga.md) - - [RC4](/releases/release-rc.4.md) - - [RC3](/releases/release-rc.3.md) - - [RC2](/releases/release-rc.2.md) - - [RC1](/releases/release-rc.1.md) -+ [Glossary](/glossary.md) + + [1.0.8](/releases/release-1.0.8.md) + + [1.0.7](/releases/release-1.0.7.md) + + [1.0.6](/releases/release-1.0.6.md) + + [1.0.5](/releases/release-1.0.5.md) + + [1.0.4](/releases/release-1.0.4.md) + + [1.0.3](/releases/release-1.0.3.md) + + [1.0.2](/releases/release-1.0.2.md) + + [1.0.1](/releases/release-1.0.1.md) + + [1.0](/releases/release-1.0-ga.md) + + [Pre-GA](/releases/release-pre-ga.md) + + [RC4](/releases/release-rc.4.md) + + [RC3](/releases/release-rc.3.md) + + [RC2](/releases/release-rc.2.md) + + [RC1](/releases/release-rc.1.md) diff --git a/_index.md b/_index.md index 2708bd93cf052..d74a19ecfc884 100644 --- a/_index.md +++ b/_index.md @@ -6,45 +6,101 @@ category: introduction # TiDB Introduction -[TiDB](https://github.com/pingcap/tidb) ("Ti" stands for Titanium) is an open-source NewSQL database that supports Hybrid Transactional and Analytical Processing (HTAP) workloads. It is MySQL compatible and features horizontal scalability, strong consistency, and high availability. +[TiDB](https://github.com/pingcap/tidb) ("Ti" stands for Titanium) is an open-source NewSQL database that supports Hybrid Transactional and Analytical Processing (HTAP) workloads. It is MySQL compatible and features horizontal scalability, strong consistency, and high availability. TiDB can be deployed on-premise or in-cloud. -TiDB can be deployed on-premise or in-cloud. The following deployment options are officially supported by PingCAP: + + +About TiDB -- [TiUP Deployment](/production-deployment-using-tiup.md): This guide describes how to deploy a TiDB cluster using [TiUP](https://github.com/pingcap-incubator/tiup). It is strongly recommended for production deployment. -- [Docker Deployment](/test-deployment-using-docker.md): This guide describes how to deploy TiDB using Docker. -- [Docker Compose Deployment](/deploy-test-cluster-using-docker-compose.md): This guide describes how to deploy TiDB using Docker compose. You can follow this guide to quickly deploy a TiDB cluster for testing and development on your local drive. -- Kubernetes Deployment: +- [TiDB Introduction](/overview.md) +- [Key Features](/key-features.md) +- [Compatibility with MySQL](/mysql-compatibility.md) +- [Usage Limitations](/tidb-limitations.md) - You can use [TiDB Operator](https://github.com/pingcap/tidb-operator) to deploy TiDB for production environments on: + - - [AWS EKS (Elastic Kubernetes Service)](https://pingcap.com/docs/tidb-in-kubernetes/stable/deploy-on-aws-eks/) - - [GKE (Google Kubernetes Engine)](https://pingcap.com/docs/tidb-in-kubernetes/stable/deploy-on-gcp-gke/) - - [Google Cloud Shell](https://pingcap.com/docs/tidb-in-kubernetes/stable/deploy-tidb-from-kubernetes-gke/) - - [Alibaba Cloud ACK (Container Service for Kubernetes)](https://pingcap.com/docs/tidb-in-kubernetes/stable/deploy-on-alibaba-cloud/) + +Quick Start - Or deploy TiDB locally using: +- [Quick Start Guide](/quick-start-with-tidb.md) +- [Explore SQL with TiDB](/basic-sql-operations.md) - - [kind](https://pingcap.com/docs/tidb-in-kubernetes/stable/deploy-tidb-from-kubernetes-kind/) - - [Minikube](https://pingcap.com/docs/tidb-in-kubernetes/stable/deploy-tidb-from-kubernetes-minikube/) + -- [Binary Tarball Deployment](/production-deployment-from-binary-tarball.md): This guide describes how to deploy TiDB from a binary tarball in production. Guides for [development](/deploy-tidb-from-binary.md) and [testing](/test-deployment-from-binary-tarball.md) environments are also available. + +Deploy and Use -## Community Provided Blog Posts & Tutorials +- [Hardware and Software Requirements](/hardware-and-software-requirements.md) +- [Check Environment and Configuration](/check-before-deployment.md) +- [Deploy a TiDB Cluster Using TiUP](/production-deployment-using-tiup.md) +- [Use TiFlash for Analytical Processing](/tiflash/use-tiflash.md) +- [Deploy TiDB in Kubernetes](https://pingcap.com/docs/tidb-in-kubernetes/stable/) -The following list collects deployment guides and tutorials from the community. The content is subject to change by the contributors. + -- [How To Spin Up an HTAP Database in 5 Minutes with TiDB + TiSpark](https://pingcap.com/blog/how_to_spin_up_an_htap_database_in_5_minutes_with_tidb_tispark/) -- [Developer install guide (single machine)](http://www.tocker.ca/this-blog-now-powered-by-wordpress-tidb.html) -- [TiDB Best Practices](https://pingcap.com/blog/2017-07-24-tidbbestpractice/) + +Migrate Data -_Your contribution is also welcome! Feel free to open a [pull request](https://github.com/pingcap/docs/blob/master/overview.md) to add additional links._ +- [Migration Overview](/migration-overview.md) +- [Migrate from MySQL SQL Files](/migrate-from-mysql-mydumper-files.md) +- [Migrate from Aurora MySQL Database](/migrate-from-aurora-mysql-database.md) +- [Migrate from CSV Files](/tidb-lightning/migrate-from-csv-using-tidb-lightning.md) -## Source Code + -Source code for [all components of the TiDB platform](https://github.com/pingcap) is available on GitHub. + +Maintain -- [TiDB](https://github.com/pingcap/tidb) -- [TiKV](https://github.com/tikv/tikv) -- [PD](https://github.com/pingcap/pd) -- [TiSpark](https://github.com/pingcap/tispark) -- [TiDB Operator](https://github.com/pingcap/tidb-operator) +- [Upgrade TiDB Using TiUP](/upgrade-tidb-using-tiup.md) +- [Scale TiDB Using TiUP](/scale-tidb-using-tiup.md) +- [Back up and Restore Data](/br/backup-and-restore-tool.md) +- [Deploy and Manage TiCDC](/ticdc/manage-ticdc.md) +- [Maintain TiDB Using TiUP](/maintain-tidb-using-tiup.md) +- [Maintain TiFlash](/tiflash/maintain-tiflash.md) + + + + +Monitor and Alert + +- [Monitoring Framework](/tidb-monitoring-framework.md) +- [Monitoring API](/tidb-monitoring-api.md) +- [Deploy Monitoring Services](/deploy-monitoring-services.md) +- [Alert Rules and Solutions](/alert-rules.md) +- [TiFlash Alert Rules and Solutions](/tiflash/tiflash-alert-rules.md) + + + + +Troubleshoot + +- [TiDB Troubleshooting Map](/tidb-troubleshooting-map.md) +- [Identify Slow Queries](/identify-slow-queries.md) +- [SQL Diagnostics](/system-tables/system-table-sql-diagnostics.md) +- [Troubleshoot the TiDB Cluster](/troubleshoot-tidb-cluster.md) +- [Troubleshoot TiCDC](/ticdc/troubleshoot-ticdc.md) +- [Troubleshoot TiFlash](/tiflash/troubleshoot-tiflash.md) + + + + +Reference + +- [TiDB Architecture](/architecture.md) +- [Key Monitoring Metrics](/grafana-overview-dashboard.md) +- [Enable TLS](/enable-tls-between-clients-and-servers.md) +- [Privilege Management](/privilege-management.md) +- [Role-Based Access Control](/role-based-access-control.md) +- [Certificate-Based Authentication](/certificate-authentication.md) + + + + +FAQs + +- [TiDB FAQs](/faq/tidb-faq.md) +- [FAQs After Upgrade](/faq/upgrade-faq.md) +- [TiDB Lightning FAQs](/tidb-lightning/tidb-lightning-faq.md) + + + \ No newline at end of file diff --git a/auto-increment.md b/auto-increment.md new file mode 100644 index 0000000000000..8d1a66d8528e5 --- /dev/null +++ b/auto-increment.md @@ -0,0 +1,211 @@ +--- +title: AUTO_INCREMENT +summary: Learn the `AUTO_INCREMENT` column attribute of TiDB. +category: reference +--- + +# AUTO_INCREMENT + +This document introduces the `AUTO_INCREMENT` column attribute, including its concept, implementation principles, auto-increment related features, and restrictions. + +## Concept + +`AUTO_INCREMENT` is a column attribute that is used to automatically fill in default column values. When the `INSERT` statement does not specify values for the `AUTO_INCREMENT` column, the system automatically assigns values to this column. Each value is unique, as well as incremental and continuous in **special cases**. + +You can use it as in the following example: + +{{< copyable "sql" >}} + +```sql +create table t(id int primary key AUTO_INCREMENT, c int); +``` + +{{< copyable "sql" >}} + +```sql +insert into t(c) values (1); +insert into t(c) values (2); +insert into t(c) values (3), (4), (5); +``` + +```sql +mysql> select * from t; ++----+---+ +| id | c | ++----+---+ +| 1 | 1 | +| 2 | 2 | +| 3 | 3 | +| 4 | 4 | +| 5 | 5 | ++----+---+ +5 rows in set (0.01 sec) +``` + +In addition, `AUTO_INCREMENT` also supports the `INSERT` statements that explicitly specify column values. In such cases, TiDB stores the explicitly specified values: + +{{< copyable "sql" >}} + +```sql +insert into t(id, c) values (6, 6); +``` + +```sql +mysql> select * from t; ++----+---+ +| id | c | ++----+---+ +| 1 | 1 | +| 2 | 2 | +| 3 | 3 | +| 4 | 4 | +| 5 | 5 | +| 6 | 6 | ++----+---+ +6 rows in set (0.01 sec) +``` + +The usage above is the same as that of `AUTO_INCREMENT` in MySQL. However, in terms of the specific value that is implicitly assigned, TiDB differs from MySQL significantly. + +## Implementation principles + +TiDB implements the `AUTO_INCREMENT` implicit assignment in the following way: + +For each auto-increment column, a globally visible key-value pair is used to record the maximum ID that has been assigned. In a distributed environment, communication between nodes has some overhead. Therefore, to avoid the issue of write amplification, each TiDB node applies for a batch of consecutive IDs as caches when assigning IDs, and then applies for the next batch of IDs after the first batch is assigned. Therefore, TiDB nodes do not apply to the storage node for IDs when assigning IDs each time. For example: + +```sql +create table t(id int unique key AUTO_INCREMENT, c int); +``` + +Assume two TiDB instances, `A` and `B`, in the cluster. If you execute an `INSERT` statement on the `t` table on `A` and `B` respectively: + +```sql +insert into t (c) values (1) +``` + +Instance `A` might cache the auto-increment IDs of `[1,30000]`, and instance `B` might cache the auto-increment IDs of `[30001,60000]`. In `INSERT` statements to be executed, these cached IDs of each instance will be assigned to the `AUTO_INCREMENT` column as the default values. + +## Basic Features + +### Uniqueness + +> **Warning:** +> +> When the cluster has multiple TiDB instances, if the table schema contains the auto-increment IDs, it is recommended not to use explicit insert and implicit assignment at the same time, which means using the default values of the auto-increment column and the custom values. Otherwise, it might break the uniqueness of implicitly assigned values. + +In the example above, perform the following operations in order: + +1. The client inserts a statement `insert into t values (2, 1)` to instance `B`, which sets `id` to `2`. The statement is successfully executed. + +2. The client sends a statement `insert into t (c) (1)` to instance `A`. This statement does not specify the value of `id`, so the ID is assigned by `A`. At present, because `A` caches the IDs of `[1, 30000]`, it might assign `2` as the value of the auto-increment ID, and increases the local counter by `1`. At this time, the data whose ID is `2` already exists in the database, so the `Duplicated Error` error is returned. + +### Increment + +TiDB guarantees that implicitly assigned values of the `AUTO_INCREMENT` column are incremental only in the cluster with a single TiDB instance. That is, for the same auto-increment column, the value assigned earlier is smaller than the value assigned later. However, in a cluster with multiple instances, TiDB cannot guarantee that the auto-increment column is incremental. + +In the example above, if you first execute an `INSERT` statement on instance `B`, and then execute an `INSERT` statement on instance `A`. Because of the behavior of caching auto-increment ID, the auto-increment column might be implicitly assigned `30002` and `2` respectively. The assigned values are not incremental in the time order. + +### Continuity + +In a cluster with multiple TiDB instances, the `AUTO_INCREMENT` assigned values are **only** continuous in the batch insert statement. + +Take the following table as an example: + +```sql +create table t (a int primary key AUTO_INCREMENT) +``` + +Execute the following statement on the table: + +```sql +insert into t values (), (), (), () +``` + +Even if other TiDB instances are performing concurrent write operations, or if the current instance does not have enough cached IDs left, the assigned values are still continuous. + +### Relation with `_tidb_rowid` + +If the primary key of integer type does not exist, TiDB uses `_tidb_rowid` to identify rows. `_tidb_rowid` and the auto-increment column (if any) share an allocator. In such cases, the cache size might be consumed by both `_tidb_rowid` and the auto-increment column. Therefore, you might encounter the following situation: + +```sql +mysql> create table t(id int unique key AUTO_INCREMENT); +Query OK, 0 rows affected (0.05 sec) + +mysql> insert into t values (),(),(); +Query OK, 3 rows affected (0.00 sec) +Records: 3 Duplicates: 0 Warnings: 0 + +mysql> select _tidb_rowid, id from t; ++-------------+------+ +| _tidb_rowid | id | ++-------------+------+ +| 4 | 1 | +| 5 | 2 | +| 6 | 3 | ++-------------+------+ +3 rows in set (0.01 sec) +``` + +### Cache size control + +In earlier versions of TiDB, the cache size of the auto-increment ID was transparent to users. Starting from v3.0.14, v3.1.2, and v4.0.rc-2, TiDB has introduced the `AUTO_ID_CACHE` table option to allow users to set the cache size for allocating the auto-increment ID. + +```sql +mysql> create table t(a int auto_increment key) AUTO_ID_CACHE 100; +Query OK, 0 rows affected (0.02 sec) + +mysql> insert into t values(); +Query OK, 1 row affected (0.00 sec) +Records: 1 Duplicates: 0 Warnings: 0 + +mysql> select * from t; ++---+ +| a | ++---+ +| 1 | ++---+ +1 row in set (0.01 sec) +``` + +At this time, if you invalidate the auto-increment cache of this column and redo the implicit insertion, the result is as follows: + +```sql +mysql> delete from t; +Query OK, 1 row affected (0.01 sec) + +mysql> rename table t to t1; +Query OK, 0 rows affected (0.01 sec) + +mysql> insert into t1 values() +Query OK, 1 row affected (0.00 sec) + +mysql> select * from t; ++-----+ +| a | ++-----+ +| 101 | ++-----+ +1 row in set (0.00 sec) +``` + +The re-assigned value is `101`. This shows that the size of cache for allocating the auto-increment ID is `100`. + +In addition, when the length of consecutive IDs in a batch `insert` statement exceeds the length of `AUTO_ID_CACHE`, TiDB increases the cache size accordingly to ensure that the statement can be inserted properly. + +### Auto-increment step size and offset + +Starting from v3.0.9 and v4.0.0-rc.1, similar to the behavior of MySQL, the value implicitly assigned to the auto-increment column is controlled by the `@@auto_increment_increment` and `@@auto_increment_offset` session variables. + +The value (ID) implicitly assigned to auto-increment columns satisfies the following equation: + +`(ID - auto_increment_offset) % auto_increment_increment == 0` + +## Restrictions + +Currently, `AUTO_INCREMENT` has the following restrictions when used in TiDB: + +- It must be defined on the column of the primary key or unique index. +- It must be defined on the column of `INTEGER`, `FLOAT`, or `DOUBLE` type. +- It cannot be specified on the same column with the `DEFAULT` column value. +- `ALTER TABLE` cannot be used to add the `AUTO_INCREMENT` attribute. +- `ALTER TABLE` can be used to remove the `AUTO_INCREMENT` attribute. However, starting from v2.1.18 and v3.0.4, TiDB uses the session variable `@@tidb_allow_remove_auto_inc` to control whether `ALTER TABLE MODIFY` or `ALTER TABLE CHANGE` can be used to remove the `AUTO_INCREMENT` attribute of a column. By default, you cannot use `ALTER TABLE MODIFY` or `ALTER TABLE CHANGE` to remove the `AUTO_INCREMENT` attribute. diff --git a/auto-random.md b/auto-random.md index 8b995f810d036..862bc32729e5b 100644 --- a/auto-random.md +++ b/auto-random.md @@ -122,7 +122,7 @@ TiDB supports parsing the version comment syntax. See the following example: {{< copyable "sql" >}} ```sql -create table t (a bigint primary key /*T!30100 auto_random */) +create table t (a bigint primary key /*T![auto_rand] auto_random */) ``` {{< copyable "sql" >}} @@ -133,9 +133,9 @@ create table t (a bigint primary key auto_random) The above two statements have the same meaning. -In the result of `show create table`, the `AUTO_RANDOM` attribute is commented out. This comment includes a version number (for example, `/*T!30100 auto_random */`). Here `30100` indicates that the `AUTO_RANDOM` attribute is introduced in v3.1.0. TiDB of a lower version ignores the `AUTO_RANDOM` attribute in the above comment. +In the result of `show create table`, the `AUTO_RANDOM` attribute is commented out. This comment includes an attribute identifier (for example, `/*T![auto_rand] auto_random */`). Here `auto_rand` represents the `AUTO_RANDOM` attribute. Only the version of TiDB that implements the feature corresponding to this identifier can parse the SQL statement fragment properly. -This attribute supports forward compatibility, namely, downgrade compatibility. TiDB earlier than v3.1.0 ignores the `AUTO_RANDOM` attribute of a table (with the above comment), so TiDB of earlier versions can also use the table with the attribute. +This attribute supports forward compatibility, namely, downgrade compatibility. TiDB of earlier versions that do not implement this feature ignore the `AUTO_RANDOM` attribute of a table (with the above comment) and can also use the table with the attribute. ## Restrictions diff --git a/benchmark/benchmark-sysbench-v4-vs-v3.md b/benchmark/benchmark-sysbench-v4-vs-v3.md new file mode 100644 index 0000000000000..6545927a89e29 --- /dev/null +++ b/benchmark/benchmark-sysbench-v4-vs-v3.md @@ -0,0 +1,174 @@ +--- +title: TiDB Sysbench Performance Test Report -- v4.0 vs. v3.0 +summary: Compare the Sysbench performance of TiDB 4.0 and TiDB 3.0. +category: benchmark +--- + +# TiDB Sysbench Performance Test Report -- v4.0 vs. v3.0 + +## Test purpose + +This test aims to compare the Sysbench performance of TiDB 4.0 and TiDB 3.0 in the Online Transactional Processing (OLTP) scenario. + +## Test environment (AWS EC2） + +### Hardware configuration + +| Service type | EC2 type | Instance count | +|:----------|:----------|:----------| +| PD | m5.xlarge | 3 | +| TiKV | i3.4xlarge| 3 | +| TiDB | c5.4xlarge| 3 | +| Sysbench | m5.4xlarge| 1 | + +### Software version + +| Service type | Software version | +|:----------|:-----------| +| PD | 3.0 and 4.0 | +| TiDB | 3.0 and 4.0 | +| TiKV | 3.0 and 4.0 | +| Sysbench | 1.0.20 | + +### Parameter configuration + +#### TiDB v3.0 configuration + +{{< copyable "" >}} + +```yaml +log.level: "error" +performance.max-procs: 20 +prepared-plan-cache.enabled: true +tikv-client.max-batch-wait-time: 2000000 +``` + +#### TiKV v3.0 configuration + +{{< copyable "" >}} + +```yaml +storage.scheduler-worker-pool-size: 5 +raftstore.store-pool-size: 3 +raftstore.apply-pool-size: 3 +rocksdb.max-background-jobs: 3 +raftdb.max-background-jobs: 3 +raftdb.allow-concurrent-memtable-write: true +server.grpc-concurrency: 6 +readpool.storage.normal-concurrency: 10 +readpool.coprocessor.normal-concurrency: 5 +``` + +#### TiDB v4.0 configuration + +{{< copyable "" >}} + +```yaml +log.level: "error" +performance.max-procs: 20 +prepared-plan-cache.enabled: true +tikv-client.max-batch-wait-time: 2000000 +``` + +#### TiKV v4.0 configuration + +{{< copyable "" >}} + +```yaml +storage.scheduler-worker-pool-size: 5 +raftstore.store-pool-size: 3 +raftstore.apply-pool-size: 3 +rocksdb.max-background-jobs: 3 +raftdb.max-background-jobs: 3 +raftdb.allow-concurrent-memtable-write: true +server.request-batch-enable-cross-command: false +server.grpc-concurrency: 6 +readpool.unifiy-read-pool: true +readpool.unified.min-thread-count: 5 +readpool.unified.max-thread-count: 20 +readpool.storage.normal-concurrency: 10 +pessimistic-txn.pipelined: true +``` + +#### Global variable configuration + +{{< copyable "sql" >}} + +```sql +set global tidb_hashagg_final_concurrency=1; +set global tidb_hashagg_partial_concurrency=1; +set global tidb_disable_txn_auto_retry=0; +``` + +## Test plan + +1. Deploy TiDB v4.0 and v3.0 using TiUP. +2. Use Sysbench to import 16 tables, each table with 10 million rows of data. +3. Execute the `analyze table` statement on each table. +4. Back up the data used for restore before different concurrency tests, which ensures data consistency for each test. +5. Start the Sysbench client to perform the `point_select`, `read_write`, `update_index`, and `update_non_index` tests. Perform stress tests on TiDB via AWS NLB. In each type of test, the warm-up takes 1 minute and the test takes 5 minutes. +6. After each type of test is completed, stop the cluster, overwrite the cluster with the backup data in step 4, and restart the cluster. + +### Prepare test data + +Execute the following command to prepare the test data: + +{{< copyable "shell-regular" >}} + +```bash +sysbench oltp_common \ + --threads=16 \ + --rand-type=uniform \ + --db-driver=mysql \ + --mysql-db=sbtest \ + --mysql-host=$aws_nlb_host \ + --mysql-port=$aws_nlb_port \ + --mysql-user=root \ + --mysql-password=password \ + prepare --tables=16 --table-size=10000000 +``` + +### Perform the test + +Execute the following command to perform the test. + +{{< copyable "shell-regular" >}} + +```bash +sysbench $testname \ + --threads=$threads \ + --time=300 \ + --report-interval=1 \ + --rand-type=uniform \ + --db-driver=mysql \ + --mysql-db=sbtest \ + --mysql-host=$aws_nlb_host \ + --mysql-port=$aws_nlb_port \ + run --tables=16 --table-size=10000000 +``` + +## Test results + +### Point Select performance + +Compared with v3.0, the Point Select performance of TiDB v4.0 has increased by 14%. + +![Point Select](/media/sysbench-v4vsv3-point-select.png) + +### Update Non-index performance + +Compared with v3.0, the Update Non-index performance of TiDB v4.0 has increased by 15%. + +![Update Non-index](/media/sysbench-v4vsv3-update-non-index.png) + +### Update Index performance + +Compared with v3.0, the Update Index performance of TiDB v4.0 has increased by 17%. + +![Update Index](/media/sysbench-v4vsv3-update-index.png) + +### Read-write performance + +Compared with v3.0, the read-write performance of TiDB v4.0 has increased by 31%. + +![Read Write](/media/sysbench-v4vsv3-read-write.png) diff --git a/benchmark/benchmark-tidb-using-sysbench.md b/benchmark/benchmark-tidb-using-sysbench.md index 692e57f697fd4..3fdd5eb725659 100644 --- a/benchmark/benchmark-tidb-using-sysbench.md +++ b/benchmark/benchmark-tidb-using-sysbench.md @@ -12,7 +12,7 @@ In this test, Sysbench 1.0.14 and TiDB 3.0 Beta are used. It is recommended to u - [Hardware recommendations](/hardware-and-software-requirements.md) -- The TiDB cluster is deployed according to the [TiDB Deployment Guide](/online-deployment-using-ansible.md). Suppose there are 3 servers in total. It is recommended to deploy 1 TiDB instance, 1 PD instance and 1 TiKV instance on each server. As for disk space, supposing that there are 32 tables and 10M rows of data on each table, it is recommended that the disk space where TiKV's data directory resides is larger than 512 GB. +- The TiDB cluster is deployed according to the [TiDB Deployment Guide](https://pingcap.com/docs/v3.0/online-deployment-using-ansible/). Suppose there are 3 servers in total. It is recommended to deploy 1 TiDB instance, 1 PD instance and 1 TiKV instance on each server. As for disk space, supposing that there are 32 tables and 10M rows of data on each table, it is recommended that the disk space where TiKV's data directory resides is larger than 512 GB. The number of concurrent connections to a single TiDB cluster is recommended to be under 500. If you need to increase the concurrency pressure on the entire system, you can add TiDB instances to the cluster whose number depends on the pressure of the test. diff --git a/benchmark/v4.0-performance-benchmarking-with-tpcc.md b/benchmark/v4.0-performance-benchmarking-with-tpcc.md index afe4019feabe1..b535a14ff8c96 100644 --- a/benchmark/v4.0-performance-benchmarking-with-tpcc.md +++ b/benchmark/v4.0-performance-benchmarking-with-tpcc.md @@ -1,113 +1,133 @@ --- title: TiDB TPC-C Performance Test Report -- v4.0 vs. v3.0 -summary: Compares the TPC-C performance of TiDB 4.0 and TiDB 3.0 using BenchmarkSQL. +summary: Compare the TPC-C performance of TiDB 4.0 and TiDB 3.0 using BenchmarkSQL. category: benchmark -aliases: ['/docs/dev/benchmark/tpcc-v2/'] --- # TiDB TPC-C Performance Test Report -- v4.0 vs. v3.0 ## Test purpose -This test aims to compare the TPC-C performance of TiDB 4.0 and TiDB 3.0. +This test aims to compare the TPC-C performance of TiDB 4.0 and TiDB 3.0 in the Online Transactional Processing (OLTP) scenario. -## Test version, time, and place +## Test environment (AWS EC2) -TiDB versions: v4.0.0-rc vs. v3.0.8 +### Hardware configuration -Time: April, 2020 +| Service type | EC2 type | Instance count | +|:----------|:----------|:----------| +| PD | m5.xlarge | 3 | +| TiKV | i3.4xlarge| 3 | +| TiDB | c5.4xlarge| 3 | +| Sysbench | m5.4xlarge| 1 | -Place: Beijing +### Software version -## Test environment +| Service type | Software version | +|:----------|:-----------| +| PD | 3.0 and 4.0 | +| TiDB | 3.0 and 4.0 | +| TiKV | 3.0 and 4.0 | +| BenchmarkSQL | None | -AWS machine: +### Parameter configuration -| Type | Machine | Number | vCPUs | -| :- | :- | :- | :- | -| TiKV | i3.4xlarge | 3 | 16 | -| TiDB | c5.4xlarge | 2 | 16 | -| PD | m5.xlarge | 3 | 4 | +#### TiDB v3.0 configuration -This test uses the open-source BenchmarkSQL 5.0 as the TPC-C testing tool and adds the support for the MySQL protocol. You can download the testing program by using the following command: - -{{< copyable "shell-regular" >}} +{{< copyable "" >}} -```shell -git clone -b 5.0-mysql-support-opt https://github.com/pingcap/benchmarksql.git +```yaml +log.level: "error" +performance.max-procs: 20 +prepared-plan-cache.enabled: true +tikv-client.max-batch-wait-time: 2000000 ``` -## Test Plan - -Use BenchmarkSQL to load the data of **5000 warehouses** into the TiDB cluster. By using HAProxy, send concurrent requests to the cluster at an incremental number. A single concurrent test lasts 10 minutes. +#### TiKV v3.0 configuration -### TiDB version information +{{< copyable "" >}} -#### v4.0.0-rc +```yaml +storage.scheduler-worker-pool-size: 5 +raftstore.store-pool-size: 3 +raftstore.apply-pool-size: 3 +rocksdb.max-background-jobs: 3 +raftdb.max-background-jobs: 3 +raftdb.allow-concurrent-memtable-write: true +server.grpc-concurrency: 6 +readpool.storage.normal-concurrency: 10 +readpool.coprocessor.normal-concurrency: 5 +``` -| Component | GitHash | -| :- | :- | -| TiDB | 79db9e30ab8f98ac07c8ae55c66dfecc24b43d56 | -| TiKV | f45d0c963df3ee4b1011caf5eb146cacd1fbbad8 | -| PD | 6f06805f3b0070107fcb4af68b2fc224dee0714d | +#### TiDB v4.0 configuration -#### v3.0.8 +{{< copyable "" >}} -| Component | GitHash | -| :- | :- | -| TiDB | 8f13cf1449bd8903ff465a4f12ed89ecbac858a4 | -| TiKV | 0d3168cfd8224fbc48a07796df83ddac0fbcbf46 | -| PD | 456c42b8b0955b33426b58054e43b771801a74d0 | +```yaml +log.level: "error" +performance.max-procs: 20 +prepared-plan-cache.enabled: true +tikv-client.max-batch-wait-time: 2000000 +``` -### TiDB parameter configuration +#### TiKV v4.0 configuration {{< copyable "" >}} -```toml -[log] -level = "error" -[performance] -max-procs = 20 -[prepared_plan_cache] -enabled = true +```yaml +storage.scheduler-worker-pool-size: 5 +raftstore.store-pool-size: 3 +raftstore.apply-pool-size: 3 +rocksdb.max-background-jobs: 3 +raftdb.max-background-jobs: 3 +raftdb.allow-concurrent-memtable-write: true +server.request-batch-enable-cross-command: false +server.grpc-concurrency: 6 +readpool.unifiy-read-pool: true +readpool.unified.min-thread-count: 5 +readpool.unified.max-thread-count: 20 +readpool.storage.normal-concurrency: 10 +pessimistic-txn.pipelined: true ``` -### TiKV parameter configuration +#### Global variable configuration -{{< copyable "" >}} +{{< copyable "sql" >}} -```toml -[readpool] -unify-read-pool = true -[readpool.unified] -min-thread-count = 5 -max-thread-count = 6 -[readpool.storage] -[readpool.coprocessor] -[storage] -scheduler-worker-pool-size = 3 -[raftstore] -store-pool-size = 3 -[rocksdb] -max-background-jobs = 3 -wal-dir = "" -[raftdb] -max-background-jobs = 3 -allow-concurrent-memtable-write = true -[server] -request-batch-enable-cross-command = false -[pessimistic-txn] -pipelined = true +```sql +set global tidb_hashagg_final_concurrency=1; +set global tidb_hashagg_partial_concurrency=1; +set global tidb_disable_txn_auto_retry=0; ``` -### Cluster topology +## Test plan + +1. Deploy TiDB v4.0 and v3.0 using TiUP. + +2. Use BenchmarkSQL to import the TPC-C 5000 Warehouse data. + + 1. Compile BenchmarkSQL： + + {{< copyable "bash" >}} + + ```bash + git clone https://github.com/pingcap/benchmarksql && cd benchmarksql && ant + ``` + + 2. Enter the `run` directory, edit the `props.mysql` file according to the actual situation, and modify the `conn`, `warehouses`, `loadWorkers`, `terminals`, and `runMins` configuration items. + + 3. Execute the `runSQL.sh ./props.mysql sql.mysql/tableCreates.sql` command. + + 4. Execute the `runSQL.sh ./props.mysql sql.mysql/indexCreates.sql` command. + + 5. Run MySQL client and execute the `analyze table` statement on every table. + +3. Execute the `runBenchmark.sh ./props.mysql` command. -* 1 TiKV instance is deployed on each `i3.4xlarge` machine, 3 in total. -* 1 TiDB instance is deployed on each `c5.4xlarge` machine, 2 in total. -* 1 PD instance is deployed on each `m5.xlarge` machine, 3 in total. +4. Extract the tpmC data of New Order from the result. ## Test result -![tpcc](/media/tpcc-3.0-4.0.png) +According to the test statistics, the TPC-C performance of TiDB v4.0 has **increased by 50%** compared with that of TiDB v3.0. -According to the testing statistics, the performance of TiDB 4.0.0-rc **has increased by 45%** than that of TiDB 3.0.8. +![TPC-C](/media/tpcc-v4vsv3.png) diff --git a/benchmark/v4.0-performance-benchmarking-with-tpch.md b/benchmark/v4.0-performance-benchmarking-with-tpch.md new file mode 100644 index 0000000000000..cf1860900acee --- /dev/null +++ b/benchmark/v4.0-performance-benchmarking-with-tpch.md @@ -0,0 +1,201 @@ +--- +title: TiDB TPC-H Performance Test Report -- v4.0 vs. v3.0 +summary: Compare the TPC-H performance of TiDB 4.0 and TiDB 3.0. +category: benchmark +--- + +# TiDB TPC-H Performance Test Report -- v4.0 vs. v3.0 + +## Test purpose + +This test aims to compare the TPC-H performance of TiDB 4.0 and TiDB 3.0 in the online analytical processing (OLAP) scenario. + +Because [TiFlash](/tiflash/tiflash-overview.md) is introduced in TiDB v4.0, which enhances TiDB's Hybrid Transactional and Analytical Processing (HTAP) capabilities, test objects in this report are as follows: + ++ TiDB v3.0 that reads data only from TiKV. ++ TiDB v4.0 that reads data only from TiKV. ++ TiDB v4.0 that reads data from TiKV and TiFlash automatically based on intelligent choice. + +## Test environment (AWS EC2) + +### Hardware configuration + +| Service type | EC2 type | Instance count | +|:----------------|:------------|:----| +| PD | m5.xlarge | 3 | +| TiDB | c5.4xlarge | 2 | +| TiKV & TiFlash | i3.4xlarge | 3 | +| TPC-H | m5.xlarge | 1 | + +### Software version + +| Service type | Software version | +|:----------|:-----------| +| PD | 3.0 and 4.0 | +| TiDB | 3.0 and 4.0 | +| TiKV | 3.0 and 4.0 | +| TiFlash | 4.0 | +| tiup-bench | 0.2 | + +### Parameter configuration + +#### v3.0 + +For v3.0, TiDB, TiKV, and PD use the default parameter configuration. + +##### Global variable configuration + +{{< copyable "sql" >}} + +```sql +set global tidb_distsql_scan_concurrency = 30; +set global tidb_projection_concurrency = 16; +set global tidb_hashagg_partial_concurrency = 16; +set global tidb_hashagg_final_concurrency = 16; +set global tidb_hash_join_concurrency = 16; +set global tidb_index_lookup_concurrency = 16; +set global tidb_index_lookup_join_concurrency = 16; +``` + +#### v4.0 + +For v4.0, TiDB uses the default parameter configuration. + +##### TiKV configuration + +{{< copyable "" >}} + +```yaml +readpool.storage.use-unified-pool: false +readpool.coprocessor.use-unified-pool: true +``` + +##### PD configuration + +{{< copyable "" >}} + +```yaml +replication.enable-placement-rules: true +``` + +##### TiFlash configuration + +{{< copyable "" >}} + +```yaml +logger.level: "info" +learner_config.log-level: "info" +``` + +##### Global variable configuration + +{{< copyable "sql" >}} + +```sql +set global tidb_allow_batch_cop = 1; +set global tidb_opt_distinct_agg_push_down = 1; +set global tidb_distsql_scan_concurrency = 30; +set global tidb_projection_concurrency = 16; +set global tidb_hashagg_partial_concurrency = 16; +set global tidb_hashagg_final_concurrency = 16; +set global tidb_hash_join_concurrency = 16; +set global tidb_index_lookup_concurrency = 16; +set global tidb_index_lookup_join_concurrency = 16; +``` + +## Test plan + +### Hardware prerequisite + +To avoid TiKV and TiFlash racing for disk and I/O resources, mount the two NVMe SSD disks configured on EC2 to `/data1` and `/data2`. Deploy TiKV on `/data1` and deploy TiFlash on `/data2`. + +### Test process + +1. Deploy TiDB v4.0 and v3.0 using [TiUP](/tiup/tiup-overview.md#tiup-overview). + +2. Use the bench tool of TiUP to import the TPC-H data with the scale factor 10. + + * Execute the following command to import data into v3.0: + + {{< copyable "bash" >}} + + ```bash + tiup bench tpch prepare \ + --host ${tidb_v3_host} --port ${tidb_v3_port} --db tpch_10 \ + --sf 10 \ + --analyze --tidb_build_stats_concurrency 8 --tidb_distsql_scan_concurrency 30 + ``` + + * Execute the following command to import data into v4.0: + + {{< copyable "bash" >}} + + ```bash + tiup bench tpch prepare \ + --host ${tidb_v4_host} --port ${tidb_v4_port} --db tpch_10 --password ${password} \ + --sf 10 \ + --tiflash \ + --analyze --tidb_build_stats_concurrency 8 --tidb_distsql_scan_concurrency 30 + ``` + +3. Execute the TPC-H queries. + + 1. Download the TPC-H SQL query file: + + {{< copyable "" >}} + + ```bash + git clone https://github.com/pingcap/tidb-bench.git && cd tpch/queries + ``` + + 2. Execute TPC-H queries and record the executing time of each query. + + * For TiDB v3.0, use the MySQL client to connect to TiDB, execute the queries, and record the execution time of each query. + * For TiDB v4.0, use the MySQL client to connect to TiDB, and choose one of the following operations based on where data is read from: + * If data is read only from TiKV, set `set @@session.tidb_isolation_read_engines = 'tikv,tidb';`, execute the queries, and record the execution time of each query. + * If data is read from TiKV and TiFlash automatically based on cost-based intelligent choice, set `set @@session.tidb_isolation_read_engines = 'tikv,tiflash,tidb';`, execute the query, and record the execution time of each query. + +4. Extract and organize the data of query execution time. + +## Test result + +| Query ID | v3.0 | v4.0 TiKV Only | v4.0 TiKV/TiFlash Automatically | +| :-------- | :----------- | :------------ | :-------------- | +| 1 | 7.78 s | 7.45 s | 2.09 s | +| 2 | 3.15 s | 1.71 s | 1.71 s | +| 3 | 6.61 s | 4.10 s | 4.05 s | +| 4 | 2.98 s | 2.56 s | 1.87 s | +| 5 | 20.35 s | 5.71 s | 8.53 s | +| 6 | 4.75 s | 2.44 s | 0.39 s | +| 7 | 7.97 s | 3.72 s | 3.59 s | +| 8 | 5.89 s | 3.22 s | 8.59 s | +| 9 | 34.08 s | 11.87 s | 15.41 s | +| 10 | 4.83 s | 2.75 s | 3.35 s | +| 11 | 3.98 s | 1.60 s | 1.59 s | +| 12 | 5.63 s | 3.40 s | 1.03 s | +| 13 | 5.41 s | 4.56 s | 4.02 s | +| 14 | 5.19 s | 3.10 s | 0.78 s | +| 15 | 10.25 s | 1.82 s | 1.26 s | +| 16 | 2.46 s | 1.51 s | 1.58 s | +| 17 | 23.76 s | 12.38 s | 8.52 s | +| 18 | 17.14 s | 16.38 s | 16.06 s | +| 19 | 5.70 s | 4.59 s | 3.20 s | +| 20 | 4.98 s | 1.89 s | 1.29 s | +| 21 | 11.12 s | 6.23 s | 6.26 s | +| 22 | 4.49 s | 3.05 s | 2.31 s | + +![TPC-H](/media/tpch-v4vsv3.png) + +In the performance diagram above: + ++ Blue lines represent v3.0; ++ Red lines represent v4.0 (data read only from TiKV); ++ Yellow lines represent v4.0 (data read from TiKV and TiFlash automatically based on intelligent choice). ++ The y-axis represents the execution time of the query. The less the time, the better the performance. + +Result description: + ++ **v4.0 TiKV Only** means that TiDB reads data only from TiKV. The result shows that the TPC-H performance increased after TiDB and TiKV are upgraded to v4.0. ++ **v4.0 TiKV/TiFlash Automatically** means that the TiDB optimizer automatically determines whether to read data from the TiFlash replica according to the cost estimation. The result shows that the TPC-H performance increased in the full HTAP form of v4.0. + +From the diagram above, you can see that TPC-H performance increases by about 100% on average over a set of 22 queries. diff --git a/best-practices/haproxy-best-practices.md b/best-practices/haproxy-best-practices.md index 051575f1d5c56..0b01678a0a61e 100644 --- a/best-practices/haproxy-best-practices.md +++ b/best-practices/haproxy-best-practices.md @@ -113,18 +113,18 @@ haproxy --help | `-vv` | Displays the version, build options, libraries versions and usable pollers. | | `-d` | Enables debug mode. | | `-db` | Disables background mode and multi-process mode. | -| `-dM []` | Forces memory poisoning, which means that each and every memory region allocated with malloc() or pool_alloc2() will be filled with \ before being passed to the caller. | +| `-dM []` | Forces memory poisoning, which means that each and every memory region allocated with malloc() or pool_alloc2() will be filled with `` before being passed to the caller. | | `-V` | Enables verbose mode (disables quiet mode). | | `-D` | Starts as a daemon.| -| `-C ` | Changes to directory \ before loading configuration files. | +| `-C ` | Changes to directory `` before loading configuration files. | | `-W` | Master-worker mode. | | `-q` | Sets "quiet" mode: This disables some messages during the configuration parsing and during startup. | | `-c` | Only performs a check of the configuration files and exits before trying to bind. | -| `-n ` | Limits the per-process connection limit to \. | -| `-m ` | Limits the total allocatable memory to \ megabytes across all processes. | -| `-N ` | Sets the default per-proxy maxconn to \ instead of the builtin default value (usually 2000). | -| `-L ` | Changes the local peer name to \, which defaults to the local hostname. | -| `-p ` | Writes all processes' PIDs into \ during startup. | +| `-n ` | Limits the per-process connection limit to ``. | +| `-m ` | Limits the total allocatable memory to `` megabytes across all processes. | +| `-N ` | Sets the default per-proxy maxconn to `` instead of the builtin default value (usually 2000). | +| `-L ` | Changes the local peer name to ``, which defaults to the local hostname. | +| `-p ` | Writes all processes' PIDs into `` during startup. | | `-de` | Disables the use of epoll(7). epoll(7) is available only on Linux 2.6 and some custom Linux 2.4 systems. | | `-dp` | Disables the use of poll(2). select(2) might be used instead. | | `-dS` | Disables the use of splice(2), which is broken on older kernels. | diff --git a/best-practices/massive-regions-best-practices.md b/best-practices/massive-regions-best-practices.md index f06cb6f9e72b3..2850c2f0989e8 100644 --- a/best-practices/massive-regions-best-practices.md +++ b/best-practices/massive-regions-best-practices.md @@ -40,7 +40,7 @@ You can check the following monitoring metrics in Grafana's **TiKV Dashboard**: + `Raft store CPU` in the **Thread-CPU** panel - Reference value: lower than `raftstore.store-pool-size * 85%`. TiDB v2.1 does not have the `raftstore.store-pool-size` configuration item, so you can take this item's value as `1` in v2.1 versions. + Reference value: lower than `raftstore.store-pool-size * 85%`. ![Check Raftstore CPU](/media/best-practices/raft-store-cpu.png) @@ -61,7 +61,7 @@ After finding out the cause of a performance problem, try to solve it from the f ### Method 1: Increase Raftstore concurrency -Raftstore in TiDB v3.0 has been upgraded to a multi-threaded module, which greatly reduces the possibility that a Raftstore thread becomes the bottleneck. +Raftstore has been upgraded to a multi-threaded module since TiDB v3.0, which greatly reduces the possibility that a Raftstore thread becomes the bottleneck. By default, `raftstore.store-pool-size` is configured to `2` in TiKV. If a bottleneck occurs in Raftstore, you can properly increase the value of this configuration item according to the actual situation. But to avoid introducing unnecessary thread switching overhead, it is recommended that you do not set this value too high. @@ -69,13 +69,13 @@ By default, `raftstore.store-pool-size` is configured to `2` in TiKV. If a bottl In the actual situation, read and write requests are not evenly distributed on every Region. Instead, they are concentrated on a few Regions. Then you can minimize the number of messages between the Raft leader and the followers for the temporarily idle Regions, which is the feature of Hibernate Region. In this feature, Raftstore does sent tick messages to the Raft state machines of idle Regions if not necessary. Then these Raft state machines will not be triggered to generate heartbeat messages, which can greatly reduce the workload of Raftstore. -Up to TiDB v3.0.9 or v3.1.0-beta.1, Hibernate Region is still an experimental feature, which is enabled by default in [TiKV master](https://github.com/tikv/tikv/tree/master). You can enable this feature according to your needs. For the configuration of Hibernate Region, refer to [Configure Hibernate Region](https://github.com/tikv/tikv/blob/master/docs/reference/configuration/raftstore-config.md#hibernate-region). +Hibernate Region is enabled by default in [TiKV master](https://github.com/tikv/tikv/tree/master). You can enable this feature according to your needs. For the configuration of Hibernate Region, refer to [Configure Hibernate Region](https://github.com/tikv/tikv/blob/master/docs/reference/configuration/raftstore-config.md#hibernate-region). ### Method 3: Enable `Region Merge` > **Note:** > -> `Region Merge` is enabled in TiDB v3.0 by default. +> `Region Merge` is enabled by default since TiDB v3.0. You can also reduce the number of Regions by enabling `Region Merge`. Contrary to `Region Split`, `Region Merge` is the process of merging adjacent small Regions through scheduling. After dropping data or executing the `Drop Table` or `Truncate Table` statement, you can merge small Regions or even empty Regions to reduce resource consumption. @@ -133,7 +133,7 @@ This section describes some other problems and solutions. PD needs to persist Region Meta information on etcd to ensure that PD can quickly resume to provide Region routing services after switching the PD Leader node. As the number of Regions increases, the performance problem of etcd appears, making it slower for PD to get Region Meta information from etcd when PD is switching the Leader. With millions of Regions, it might take more than ten seconds or even tens of seconds to get the meta information from etcd. -To address this problem, `use-region-storage` is enabled by default in PD in TiDB v3.0. With this feature enabled, PD stores Region Meta information on local LevelDB and synchronizes the information among PD nodes through other mechanisms. +To address this problem, `use-region-storage` is enabled by default in PD since TiDB v3.0. With this feature enabled, PD stores Region Meta information on local LevelDB and synchronizes the information among PD nodes through other mechanisms. ### PD routing information is not updated in time @@ -143,8 +143,8 @@ You can check **Worker pending tasks** under **Task** in the **TiKV Grafana** pa ![Check pd-worker](/media/best-practices/pd-worker-metrics.png) -Currently, pd-worker is optimized for better efficiency in [#5620](https://github.com/tikv/tikv/pull/5620) on [TiKV master](https://github.com/tikv/tikv/tree/master), which is applied since [v3.0.5](/releases/release-3.0.5.md#tikv). If you encounter a similar problem, it is recommended to upgrade to v3.0.5 or later versions. +pd-worker has been optimized for better performance since [v3.0.5](/releases/release-3.0.5.md#tikv). If you encounter a similar problem, it is recommended to upgrade to the latest version. ### Prometheus is slow to query metrics -In a large-scale cluster, as the number of TiKV instances increases, Prometheus has greater pressure to query metrics, making it slower for Grafana to display these metrics. To ease this problem, metrics pre-calculation is configured in v3.0. +In a large-scale cluster, as the number of TiKV instances increases, Prometheus has greater pressure to query metrics, making it slower for Grafana to display these metrics. To ease this problem, metrics pre-calculation is configured since v3.0. diff --git a/br/backup-and-restore-faq.md b/br/backup-and-restore-faq.md new file mode 100644 index 0000000000000..3703ac0733913 --- /dev/null +++ b/br/backup-and-restore-faq.md @@ -0,0 +1,55 @@ +--- +title: Backup & Restore FAQ +summary: Learn about Frequently Asked Questions (FAQ) and the solutions of BR. +category: FAQ +--- + +# Backup & Restore FAQ + +This document lists the frequently asked questions (FAQs) and the solutions about Backup & Restore (BR). + +## What should I do if the error message `could not read local://...:download sst failed` is returned during data restoration? + +When you restore data, each node must have access to **all** backup files (SST files). By default, if `local` storage is used, you cannot restore data because the backup files are scattered among different nodes. Therefore, you have to copy the backup file of each TiKV node to the other TiKV nodes. + +It is recommended to mount an NFS disk as a backup disk during backup. For details, see [Back up a single table to a network disk](/br/backup-and-restore-use-cases.md#back-up-a-single-table-to-a-network-disk-recommended). + +## How much does it affect the cluster during backup using BR? + +When you use the `oltp_read_only` scenario of `sysbench` to back up to a disk (make sure the backup disk and the service disk are different) at full rate, the cluster QPS is decreased by 15%-25%. The impact on the cluster depends on the table schema. + +To reduce the impact on the cluster, you can use the `--ratelimit` parameter to limit the backup rate. + +## Does BR back up system tables? During data restoration, do they raise conflict? + +The system libraries (`information_schema`, `performance_schema`, `mysql`) are filtered out during full backup. For more details, refer to the [Backup Principle](/br/backup-and-restore-tool.md#backup-principle). + +Because these system libraries do not exist in the backup files, no conflict occurs among system tables during data restoration. + +## What should I do to resolve the `Permission denied` error, even if I have tried to run BR using root in vain? + +You need to confirm whether TiKV has access to the backup directory. To back up data, confirm whether TiKV has the write permission. To restore data, confirm whether it has the read permission. + +Running BR with the root access might fail due to the disk permission, because the backup files (SST files) are saved by TiKV. + +> **Note:** +> +> You might encounter the same problem during data restoration. When the SST files are read for the first time, the read permission is verified. The execution duration of DDL suggests that there might be a long interval between checking the permission and running BR. You might receive the error message `Permission denied` after waiting for a long time. +> +> Therefore, It is recommended to check the permission before data restoration. + +## What should I do to resolve the `Io(Os...)` error? + +Almost all of these problems are system call errors that occur when TiKV writes data to the disk. You can check the mounting method and the file system of the backup directory, and try to back up data to another folder or another hard disk. +  +For example, you might encounter the `Code: 22(invalid argument)` error when backing up data to the network disk built by `samba`. + +## Where are the backed up files stored when I use `local` storage? +  +When you use `local` storage, `backupmeta` is generated on the node where BR is running, and backup files are generated on the Leader nodes of each Region. + +## How about the size of the backup data? Are there replicas of the backup? + +During data backup, backup files are generated on the Leader nodes of each Region. The size of the backup is equal to the data size, with no redundant replicas. Therefore, the total data size is approximately the total number of TiKV data divided by the number of replicas. + +However, if you want to restore data from local storage, the number of replicas is equal to that of the TiKV nodes, because each TiKV must have access to all backup files. diff --git a/br/backup-and-restore-storages.md b/br/backup-and-restore-storages.md new file mode 100644 index 0000000000000..973f06a876715 --- /dev/null +++ b/br/backup-and-restore-storages.md @@ -0,0 +1,82 @@ +--- +title: BR storages +summary: Describes the storage URL format used in BR. +category: reference +--- + +# BR storages + +BR supports reading and writing data on the local filesystem, as well as on Amazon S3 and Google Cloud Storage. These are distinguished by the URL scheme in the `--storage` parameter passed into BR. + +## Schemes + +The following services are supported: + +| Service | Schemes | Example URL | +|---------|---------|-------------| +| Local filesystem, distributed on every node | local | `local:///path/to/dest/` | +| Amazon S3 and compatible services | s3 | `s3://bucket-name/prefix/of/dest/` | +| Google Cloud Storage (GCS) | gcs, gs | `gcs://bucket-name/prefix/of/dest/` | +| Write to nowhere (for benchmarking only) | noop | `noop://` | + +## Parameters + +Cloud storages such as S3 and GCS sometimes require additional configuration for connection. You can specify parameters for such configuration. For example: + +{{< copyable "shell-regular" >}} + +```shell +./br backup full -u 127.0.0.1:2379 -s 's3://bucket-name/prefix?region=us-west-2' +``` + +### S3 parameters + +| Parameter | Description | +|----------:|---------| +| `access-key` | The access key | +| `secret-access-key` | The secret access key | +| `region` | Service Region for Amazon S3 (default to `us-east-1`) | +| `use-accelerate-endpoint` | Whether to use the accelerate endpoint on Amazon S3 (default to `false`) | +| `endpoint` | URL of custom endpoint for S3-compatible services (for example, `https://s3.example.com/`) | +| `force-path-style` | Use path style access rather than virtual hosted style access (default to `false`) | +| `storage-class` | Storage class of the uploaded objects (for example, `STANDARD`, `STANDARD_IA`) | +| `sse` | Server-side encryption algorithm used to encrypt the upload (empty, `AES256` or `aws:kms`) | +| `sse-kms-key-id` | If `sse` is set to `aws:kms`, specifies the KMS ID | +| `acl` | Canned ACL of the uploaded objects (for example, `private`, `authenticated-read`) | + +> **Note:** +> +> It is not recommended to pass in the access key and secret access key directly in the storage URL, because these keys are logged in plain text. BR tries to infer these keys from the environment in the following order: + +1. `$AWS_ACCESS_KEY_ID` and `$AWS_SECRET_ACCESS_KEY` environment variables +2. `$AWS_ACCESS_KEY` and `$AWS_SECRET_KEY` environment variables +3. Shared credentials file on the BR node at the path specified by the `$AWS_SHARED_CREDENTIALS_FILE` environment variable +4. Shared credentials file on the BR node at `~/.aws/credentials` +5. Current IAM role of the Amazon EC2 container +6. Current IAM role of the Amazon ECS task + +### GCS parameters + +| Parameter | Description | +|----------:|---------| +| `credentials-file` | The path to the credentials JSON file on the TiDB node | +| `storage-class` | Storage class of the uploaded objects (for example, `STANDARD`, `COLDLINE`) | +| `predefined-acl` | Predefined ACL of the uploaded objects (for example, `private`, `project-private`) | + +When `credentials-file` is not specified, BR will try to infer the credentials from the environment, in the following order: + +1. Content of the file on the BR node at the path specified by the `$GOOGLE_APPLICATION_CREDENTIALS` environment variable +2. Content of the file on the BR node at `~/.config/gcloud/application_default_credentials.json` +3. When running in GCE or GAE, the credentials fetched from the metadata server. + +## Sending credentials to TiKV + +By default, when using S3 and GCS destinations, BR will send the credentials to every TiKV nodes to reduce setup complexity. + +However, this is unsuitable on cloud environment, where every node has their own role and permission. In such cases, you need to disable credentials sending with `--send-credentials-to-tikv=false` (or the short form `-c=0`): + +{{< copyable "shell-regular" >}} + +```shell +./br backup full -c=0 -u pd-service:2379 -s 's3://bucket-name/prefix' +``` diff --git a/br/backup-and-restore-tool.md b/br/backup-and-restore-tool.md index bec3bab903059..e874fa3af518f 100644 --- a/br/backup-and-restore-tool.md +++ b/br/backup-and-restore-tool.md @@ -52,6 +52,7 @@ BackupRequest{ ClusterId, // The cluster ID. StartKey, // The starting key of the backup (backed up). EndKey, // The ending key of the backup (not backed up). + StartVersion, // The version of the last backup snapshot, used for the incremental backup. EndVersion, // The backup snapshot time. StorageBackend, // The path where backup files are stored. RateLimit, // Backup speed (MB/s). @@ -62,6 +63,8 @@ After receiving the backup request, the TiKV node traverses all Region leaders o After finishing backing up the data of the corresponding Region, the TiKV node returns the metadata to BR. BR collects the metadata and stores it in the `backupmeta` file which is used for restoration. +If `StartVersion` is not `0`, the backup is seen as an incremental backup. In addition to KVs, BR also collects DDLs between `[StartVersion, EndVersion)`. During data restoration, these DDLs are restored first. + If checksum is enabled when you execute the backup command, BR calculates the checksum of each backed up table for data check. #### Types of backup files @@ -122,17 +125,26 @@ Explanations for the above command are as follows: * `backup`: the sub-command of `br`. * `full`: the sub-command of `backup`. * `-s` (or `--storage`): the option that specifies the path where the backup files are stored. -* `"local:///tmp/backup"`: the parameter of `-s`. `/tmp/backup` is the path in the local disk where the backup files are stored. +* `"local:///tmp/backup"`: the parameter of `-s`. `/tmp/backup` is the path in the local disk where the backed up files of each TiKV node are stored. * `--pd`: the option that specifies the Placement Driver (PD) service address. * `"${PDIP}:2379"`: the parameter of `--pd`. +> **Note:** +> +> - When the `local` storage is used, the backup data are scattered in the local file system of each node. +> +> - It is **not recommended** to back up to a local disk in the production environment because you **have to** manually aggregate these data to complete the data restoration. For more information, see [Restore Cluster Data](#restore-cluster-data). +> +> - Aggregating these backup data might cause redundancy and bring troubles to operation and maintenance. Even worse, if restoring data without aggregating these data, you can receive a rather confusing error message `SST file not found`. +> +> - It is recommended to mount the NFS disk on each node, or back up to the `S3` object storage. + ### Sub-commands A `br` command consists of multiple layers of sub-commands. Currently, BR has the following three sub-commands: * `br backup`: used to back up the data of the TiDB cluster. * `br restore`: used to restore the data of the TiDB cluster. -* `br version`: used to check the version of BR. Each of the above three sub-commands might still include the following three sub-commands to specify the scope of an operation: @@ -144,6 +156,7 @@ Each of the above three sub-commands might still include the following three sub * `--pd`: used for connection, specifying the PD server address. For example, `"${PDIP}:2379"`. * `-h` (or `--help`): used to get help on all sub-commands. For example, `br backup --help`. +* `-V` (or `--version`): used to check the version of BR. * `--ca`: specifies the path to the trusted CA certificate in the PEM format. * `--cert`: specifies the path to the SSL certificate in the PEM format. * `--key`: specifies the path to the SSL certificate key in the PEM format. @@ -172,6 +185,12 @@ To back up all the cluster data, execute the `br backup full` command. To get he Back up all the cluster data to the `/tmp/backup` path of each TiKV node and write the `backupmeta` file to this path. +> **Note:** +> +> + If the backup disk and the service disk are different, it has been tested that online backup reduces QPS of the read-only online service by about 15%-25% in case of full-speed backup. If you want to reduce the impact on QPS, use `--ratelimit` to limit the rate. +> +> + If the backup disk and the service disk are the same, the backup competes with the service for I/O resources. This might decrease the QPS of the read-only online service by more than half. Therefore, it is **highly not recommended** to back up the online service data to the TiKV data disk. + {{< copyable "shell-regular" >}} ```shell @@ -246,10 +265,95 @@ The `table` sub-command has two options: * `--db`: specifies the database name * `--table`: specifies the table name. -For descriptions of other options, see [Back up all cluster data](#back-up-all-cluster-data). +For descriptions of other options, see [Back up all cluster data](#back-up-all-the-cluster-data). A progress bar is displayed in the terminal during the backup operation. When the progress bar advances to 100%, the backup is complete. Then the BR also checks the backup data to ensure data safety. +### Back up data to Amazon S3 backend + +If you back up the data to the Amazon S3 backend, instead of `local` storage, you need to specify the S3 storage path in the `storage` sub-command, and allow the BR node and the TiKV node to access Amazon S3. + +You can refer to the [AWS Official Document](https://docs.aws.amazon.com/AmazonS3/latest/user-guide/create-bucket.html) to create an S3 `Bucket` in the specified `Region`. You can also refer to another [AWS Official Document](https://docs.aws.amazon.com/AmazonS3/latest/user-guide/create-folder.html) to create a `Folder` in the `Bucket`. + +Pass `SecretKey` and `AccessKey` of the account that has privilege to access the S3 backend to the BR node. Here `SecretKey` and `AccessKey` are passed as environment variables. Then pass the privilege to the TiKV node through BR. + +{{< copyable "shell-regular" >}} + +```shell +export AWS_ACCESS_KEY_ID=${AccessKey} +export AWS_SECRET_ACCESS_KEY=${SecretKey} +``` + +When backing up using BR, explicitly specify the parameters `--s3.region` and `--send-credentials-to-tikv`. `--s3.region` indicates the region where S3 is located, and `--send-credentials-to-tikv` means passing the privilege to access S3 to the TiKV node. + +{{< copyable "shell-regular" >}} + +```shell +br backup full \ + --pd "${PDIP}:2379" \ + --storage "s3://${Bucket}/${Folder}" \ + --s3.region "${region}" \ + --send-credentials-to-tikv=true \ + --log-file backuptable.log +``` + +### Back up incremental data + +If you want to back up incrementally, you only need to specify the **last backup timestamp** `--lastbackupts`. + +The incremental backup has two limitations: + +- The incremental backup needs to be under a different path from the previous full backup. +- No GC (Garbage Collection) happens between the start time of the incremental backup and `lastbackupts`. + +To back up the incremental data between `(LAST_BACKUP_TS, current PD timestamp]`, execute the following command: + +{{< copyable "shell-regular" >}} + +```shell +br backup full\ + --pd ${PDIP}:2379 \ + -s local:///home/tidb/backupdata/incr \ + --lastbackupts ${LAST_BACKUP_TS} +``` + +To get the timestamp of the last backup, execute the `validate` command. For example: + +{{< copyable "shell-regular" >}} + +```shell +LAST_BACKUP_TS=`br validate decode --field="end-version" -s local:///home/tidb/backupdata` +``` + +In the above example, the incremental backup data includes the newly written data and the DDLs between `(LAST_BACKUP_TS, current PD timestamp]`. When restoring data, BR restores DDLs first and then restores the written data. + +### Back up Raw KV (experimental feature) + +> **Warning:** +> +> This feature is experimental and not thoroughly tested. It is highly **not recommended** to use this feature in the production environment. + +In some scenarios, TiKV might run independently of TiDB. Given that, BR also supports bypassing the TiDB layer and backing up data in TiKV. + +For example, you can execute the following command to back up all keys between `[0x31, 0x3130303030303030)` in the default CF to `$BACKUP_DIR`: + +{{< copyable "shell-regular" >}} + +```shell +br backup raw --pd $PD_ADDR \ + -s "local://$BACKUP_DIR" \ + --start 31 \ + --end 3130303030303030 \ + --format hex \ + --cf default +``` + +Here, the parameters of `--start` and `--end` are decoded using the method specified by `--format` before being sent to TiKV. Currently, the following methods are available: + +- "raw": The input string is directly encoded as a key in binary format. +- "hex": The default encoding method. The input string is treated as a hexadecimal number. +- "escape": First escape the input string, and then encode it into binary format. + ## Restore cluster data To restore the cluster data, use the `br restore` command. You can add the `full`, `db` or `table` sub-command to specify the scope of your restoration: the whole cluster, a database or a single table. @@ -263,7 +367,7 @@ To restore the cluster data, use the `br restore` command. You can add the `full > - Data are replicated into multiple peers. When ingesting SSTs, these files have to be present on *all* peers. This is unlike back up where reading from a single node is enough. > - Where each peer is scattered to during restore is random. We don't know in advance which node will read which file. > -> These can be avoided using shared storage, e.g. mounting an NFS on the local path, or using S3. With network storage, every node can automatically read every SST file, so these caveats no longer apply. +> These can be avoided using shared storage, for example mounting an NFS on the local path, or using S3. With network storage, every node can automatically read every SST file, so these caveats no longer apply. ### Restore all the backup data @@ -316,7 +420,7 @@ br restore db \ --log-file restorefull.log ``` -In the above command, `--db` specifies the name of the database to be restored. For descriptions of other options, see [Restore all backup data](#restore-all-backup-data). +In the above command, `--db` specifies the name of the database to be restored. For descriptions of other options, see [Restore all backup data](#restore-all-the-backup-data)). ### Restore a table @@ -337,7 +441,96 @@ br restore table \ --log-file restorefull.log ``` -In the above command, `--table` specifies the name of the table to be restored. For descriptions of other options, see [Restore all backup data](#restore-all-backup-data) and [Restore a database](#restore-a-database). +In the above command, `--table` specifies the name of the table to be restored. For descriptions of other options, see [Restore all backup data](#restore-all-the-backup-data) and [Restore a database](#restore-a-database). + +### Restore data from Amazon S3 backend + +If you restore data from the Amazon S3 backend, instead of `local` storage, you need to specify the S3 storage path in the `storage` sub-command, and allow the BR node and the TiKV node to access Amazon S3. + +Pass `SecretKey` and `AccessKey` of the account that has privilege to access the S3 backend to the BR node. Here `SecretKey` and `AccessKey` are passed as environment variables. Then pass the privilege to the TiKV node through BR. + +{{< copyable "shell-regular" >}} + +```shell +export AWS_ACCESS_KEY_ID=${AccessKey} +export AWS_SECRET_ACCESS_KEY=${SecretKey} +``` + +When restoring data using BR, explicitly specify the parameters `--s3.region` and `--send-credentials-to-tikv`. `--s3.region` indicates the region where S3 is located, and `--send-credentials-to-tikv` means passing the privilege to access S3 to the TiKV node. + +`Bucket` and `Folder` in the `--storage` parameter represent the S3 bucket and the folder where the data to be restored is located. + +{{< copyable "shell-regular" >}} + +```shell +br restore full \ + --pd "${PDIP}:2379" \ + --storage "s3://${Bucket}/${Folder}" \ + --s3.region "${region}" \ + --send-credentials-to-tikv=true \ + --log-file restorefull.log +``` + +In the above command, `--table` specifies the name of the table to be restored. For descriptions of other options, see [Restore a database](#restore-a-database). + +### Restore incremental data + +Restoring incremental data is similar to [restoring full data using BR](#restore-all-the-backup-data). Note that when restoring incremental data, make sure that all the data backed up before `last backup ts` has been restored to the target cluster. + +### Restore Raw KV (experimental feature) + +> **Warning:** +> +> This feature is in the experiment, without being thoroughly tested. It is highly **not recommended** to use this feature in the production environment. + +Similar to [backing up Raw KV](#back-up-raw-kv-experimental-feature), you can execute the following command to restore Raw KV: + +{{< copyable "shell-regular" >}} + +br restore raw --pd $PD_ADDR \ + -s "local://$BACKUP_DIR" \ + --start 31 \ + --end 3130303030303030 \ + --format hex \ + --cf default + +In the above example, all the backed up keys in the range `[0x31, 0x3130303030303030)` are restored to the TiKV cluster. The coding methods of these keys are identical to that of [keys during the backup process](#back-up-raw-kv-experimental-feature) + +### Online restore (experimental feature) + +> **Warning:** +> +> This feature is in the experiment, without being thoroughly tested. It also relies on the unstable `Placement Rules` feature of PD. It is highly **not recommended** to use this feature in the production environment. + +During data restoration, writing too much data affects the performance of the online cluster. To avoid this effect as much as possible, BR supports [Placement rules](/configure-placement-rules.md) to isolate resources. In this case, downloading and importing SST are only performed on a few specified nodes (or "restore nodes" for short). To complete the online restore, take the following steps. + +1. Configure PD, and start Placement rules: + + {{< copyable "shell-regular" >}} + + ```shell + echo "config set enable-placement-rules true" | pd-ctl + ``` + +2. Edit the configuration file of the "restore node" in TiKV, and specify "restore" to the `server` configuration item: + + {{< copyable "" >}} + + ``` + [server] + labels = { exclusive = "restore" } + ``` + +3. Start TiKV of the "restore node" and restore the backed up files using BR. Compared with the offline restore, you only need to add the `--online` flag: + + {{< copyable "shell-regular" >}} + + ``` + br restore full \ + -s "local://$BACKUP_DIR" \ + --pd $PD_ADDR \ + --online + ``` ## Best practices diff --git a/br/backup-and-restore-use-cases.md b/br/backup-and-restore-use-cases.md index bb2c9307df9c4..408ef05e48539 100644 --- a/br/backup-and-restore-use-cases.md +++ b/br/backup-and-restore-use-cases.md @@ -84,7 +84,7 @@ For the detailed usage of the `br backup` command, refer to [BR command-line des {{< copyable "sql" >}} ```sql - update mysql.tidb set VARIABLE_VALUE = '10m' where VARIABLE_NAME = 'tikv_gc_life_time'; + UPDATE mysql.tidb SET VARIABLE_VALUE = '10m' WHERE VARIABLE_NAME = 'tikv_gc_life_time'; ``` ### Preparation for restoration @@ -121,7 +121,12 @@ Execute the `br backup` command: {{< copyable "shell-regular" >}} ```shell -bin/br backup table --db batchmark --table order_line -s local:///br_data --pd 172.16.5.198:2379 --log-file backup-nfs.log +bin/br backup table \ + --db batchmark \ + --table order_line \ + -s local:///br_data \ + --pd ${PD_ADDR}:2379 \ + --log-file backup-nfs.log ``` #### Monitoring metrics for the backup @@ -166,7 +171,17 @@ During the backup process, pay attention to the following metrics on the monitor Before executing the backup command, a path in which the log is stored has been specified. You can get the statistical information of the backup operation from this log. Search "summary" in this log, you can see the following information: ``` -["Table backup summary: total backup ranges: 4, total success: 4, total failed: 0, total take(s): 986.43, total kv: 5659888624, total size(MB): 353227.18, avg speed(MB/s): 358.09"] ["backup total regions"=7196] ["backup checksum"=6m28.291772955s] ["backup fast checksum"=24.950298ms] +["Table backup summary: + total backup ranges: 4, + total success: 4, + total failed: 0, + total take(s): 986.43, + total kv: 5659888624, + total size(MB): 353227.18, + avg speed(MB/s): 358.09"] + ["backup total regions"=7196] + ["backup checksum"=6m28.291772955s] + ["backup fast checksum"=24.950298ms] ``` The above log includes the following information: @@ -185,7 +200,13 @@ If the resource usage of TiKV does not become an obvious bottleneck during the b {{< copyable "shell-regular" >}} ```shell -bin/br backup table --db batchmark --table order_line -s local:///br_data/ --pd 172.16.5.198:2379 --log-file backup-nfs.log --concurrency 16 +bin/br backup table \ + --db batchmark \ + --table order_line \ + -s local:///br_data/ \ + --pd ${PD_ADDR}:2379 \ + --log-file backup-nfs.log \ + --concurrency 16 ``` ![img](/media/br/backup-diff.png) @@ -261,7 +282,18 @@ During the restoration process, pay attention to the following metrics on the mo Before executing the restoration command, a path in which the log is stored has been specified. You can get the statistical information of the restoration operation from this log. Search "summary" in this log, you can see the following information: ``` -["Table Restore summary: total restore tables: 1, total success: 1, total failed: 0, total take(s): 961.37, total kv: 5659888624, total size(MB): 353227.18, avg speed(MB/s): 367.42"] ["restore files"=9263] ["restore ranges"=6888] ["split region"=49.049182743s] ["restore checksum"=6m34.879439498s] +["Table Restore summary: + total restore tables: 1, + total success: 1, + total failed: 0, + total take(s): 961.37, + total kv: 5659888624, + total size(MB): 353227.18, + avg speed(MB/s): 367.42"] + ["restore files"=9263] + ["restore ranges"=6888] + ["split region"=49.049182743s] + ["restore checksum"=6m34.879439498s] ``` The above log includes the following information: @@ -321,7 +353,12 @@ Execute the `br backup` command: {{< copyable "shell-regular" >}} ```shell -bin/br backup table --db batchmark --table order_line -s local:///home/tidb/backup_local/ --pd 172.16.5.198:2379 --log-file backup_local.log +bin/br backup table \ + --db batchmark \ + --table order_line \ + -s local:///home/tidb/backup_local/ \ + --pd ${PD_ADDR}:2379 \ + --log-file backup_local.log ``` During the backup process, pay attention to the metrics on the monitoring panels to get the status of the backup process. See [Monitoring metrics for the backup](#monitoring-metrics-for-the-backup) for details. diff --git a/certificate-authentication.md b/certificate-authentication.md index ab906bbbdecdf..585867936b21d 100644 --- a/certificate-authentication.md +++ b/certificate-authentication.md @@ -5,9 +5,9 @@ category: reference aliases: ['/docs/dev/reference/security/cert-based-authentication/'] --- -# Certificate-Based Authentication for Login New in v3.0.8 +# Certificate-Based Authentication for Login -Starting from v3.0.8, TiDB supports a certificate-based authentication method for users to log into TiDB. With this method, TiDB issues certificates to different users, uses encrypted connections to transfer data, and verifies certificates when users log in. This approach is more secure than the traditional password-based authentication method commonly used by MySQL users and is thus adopted by an increasing number of users. +TiDB supports a certificate-based authentication method for users to log into TiDB. With this method, TiDB issues certificates to different users, uses encrypted connections to transfer data, and verifies certificates when users log in. This approach is more secure than the traditional password-based authentication method commonly used by MySQL users and is thus adopted by an increasing number of users. To use certificate-based authentication, you might need to perform the following operations: @@ -20,15 +20,7 @@ The rest of the document introduces in detail how to perform these operations. ## Create security keys and certificates -### Install OpenSSL - -It is recommended that you use [OpenSSL](https://www.openssl.org/) to create keys and certificates. Taking the Debian operation system as an example, execute the following command to install OpenSSL: - -{{< copyable "shell-regular" >}} - -```bash -sudo apt-get install openssl -``` +It is recommended that you use [OpenSSL](https://www.openssl.org/) to create keys and certificates. The certificate generation process is similar to the process described in [Enable TLS Between TiDB Clients and Servers](/enable-tls-between-clients-and-servers.md). The following paragraphs demonstrate on how to configure more attribute fields that need to be verified in the certificate. ### Generate CA key and certificate @@ -265,7 +257,7 @@ First, connect TiDB using the client to configure the login verification. Then, ### Get user certificate information -The user certificate information can be specified by `require subject`, `require issuer`, and `require cipher`, which are used to check the X509 certificate attributes. +The user certificate information can be specified by `require subject`, `require issuer`, `require san`, and `require cipher`, which are used to check the X509 certificate attributes. + `require subject`: Specifies the `subject` information of the client certificate when you log in. With this option specified, you do not need to configure `require ssl` or x509. The information to be specified is consistent with the entered `subject` information in [Generate client keys and certificates](#generate-client-keys-and-certificates). @@ -287,6 +279,32 @@ The user certificate information can be specified by `require subject`, `require openssl x509 -noout -subject -in ca-cert.pem | sed 's/.\{8\}//' | sed 's/, /\//g' | sed 's/ = /=/g' | sed 's/^/\//' ``` ++ `require san`: Specifies the `Subject Alternative Name` information of the CA certificate that issues the user certificate. The information to be specified is consistent with the [`alt_names` of the `openssl.cnf` configuration file](/generate-self-signed-certificates.md) used to generate the client certificate. + + + Execute the following command to get the information of the `require san` item in the generated certificate: + + {{< copyable "shell-regular" >}} + + ```shell + openssl x509 -noout -ext subjectAltName -in client.crt + ``` + + + `require san` currently supports the following `Subject Alternative Name` check items: + + - URI + - IP + - DNS + + + Multiple check items can be configured after they are connected by commas. For example, configure `require san` as follows for the `u1` user: + + {{< copyable "sql" >}} + + ```sql + create user 'u1'@'%' require san 'DNS:d1,URI:spiffe://example.org/myservice1,URI:spiffe://example.org/myservice2' + ``` + + The above configuration only allows the `u1` user to log in to TiDB using the certificate with the URI item `spiffe://example.org/myservice1` or `spiffe://example.org/myservice2` and the DNS item `d1`. + + `require cipher`: Checks the cipher method supported by the client. Use the following statement to check the list of supported cipher methods: {{< copyable "sql" >}} @@ -297,7 +315,7 @@ The user certificate information can be specified by `require subject`, `require ### Configure user certificate information -After getting the user certificate information (`require subject`, `require issuer`, `require cipher`), configure these information to be verified when creating a user, granting privileges, or altering a user. Replace `` with the corresponding information in the following statements. +After getting the user certificate information (`require subject`, `require issuer`, `require san`, `require cipher`), configure these information to be verified when creating a user, granting privileges, or altering a user. Replace `` with the corresponding information in the following statements. You can configure one option or multiple options using the space or `and` as the separator. @@ -306,7 +324,7 @@ You can configure one option or multiple options using the space or `and` as the {{< copyable "sql" >}} ```sql - create user 'u1'@'%' require issuer '' subject '' cipher ''; + create user 'u1'@'%' require issuer '' subject '' san '' cipher ''; ``` + Configure user certificate when granting privileges: @@ -314,7 +332,7 @@ You can configure one option or multiple options using the space or `and` as the {{< copyable "sql" >}} ```sql - grant all on *.* to 'u1'@'%' require issuer '' subject '' cipher ''; + grant all on *.* to 'u1'@'%' require issuer '' subject '' san '' cipher ''; ``` + Configure user certificate when altering a user: @@ -322,7 +340,7 @@ You can configure one option or multiple options using the space or `and` as the {{< copyable "sql" >}} ```sql - alter user 'u1'@'%' require issuer '' subject '' cipher ''; + alter user 'u1'@'%' require issuer '' subject '' san '' cipher ''; ``` After the above configuration, the following items will be verified when you log in: @@ -330,6 +348,7 @@ After the above configuration, the following items will be verified when you log + SSL is used; the CA that issues the client certificate is consistent with the CA configured in the server. + The `issuer` information of the client certificate matches the information specified in `require issuer`. + The `subject` information of the client certificate matches the information specified in `require cipher`. ++ The `Subject Alternative Name` information of the client certificate matches the information specified in `require san`. You can log into TiDB only after all the above items are verified. Otherwise, the `ERROR 1045 (28000): Access denied` error is returned. You can use the following command to check the TLS version, the cipher algorithm and whether the current connection uses the certificate for the login. diff --git a/check-before-deployment.md b/check-before-deployment.md new file mode 100644 index 0000000000000..7cfe54ef610c5 --- /dev/null +++ b/check-before-deployment.md @@ -0,0 +1,342 @@ +--- +title: TiDB Environment and System Configuration Check +summary: Learn the environment check operations before deploying TiDB. +category: how-to +--- + +# TiDB Environment and System Configuration Check + +This document describes the environment check operations before deploying TiDB. The following steps are ordered by priorities. + +## Mount the data disk ext4 filesystem with options on the target machines that deploy TiKV + +For production deployments, it is recommended to use NVMe SSD of EXT4 filesystem to store TiKV data. This configuration is the best practice, whose reliability, security, and stability have been proven in a large number of online scenarios. + +Log in to the target machines using the `root` user account. + +Format your data disks to the ext4 filesystem and add the `nodelalloc` and `noatime` mount options to the filesystem. It is required to add the `nodelalloc` option, or else the TiUP deployment cannot pass the precheck. The `noatime` option is optional. + +> **Note:** +> +> If your data disks have been formatted to ext4 and have added the mount options, you can uninstall it by running the `umount /dev/nvme0n1p1` command, skip directly to the fifth step below to edit the `/etc/fstab` file, and add the options again to the filesystem. + +Take the `/dev/nvme0n1` data disk as an example: + +1. View the data disk. + + {{< copyable "shell-root" >}} + + ```bash + fdisk -l + ``` + + ``` + Disk /dev/nvme0n1: 1000 GB + ``` + +2. Create the partition. + + {{< copyable "shell-root" >}} + + ```bash + parted -s -a optimal /dev/nvme0n1 mklabel gpt -- mkpart primary ext4 1 -1 + ``` + + > **Note:** + > + > Use the `lsblk` command to view the device number of the partition: for a NVMe disk, the generated device number is usually `nvme0n1p1`; for a regular disk (for example, `/dev/sdb`), the generated device number is usually `sdb1`. + +3. Format the data disk to the ext4 filesystem. + + {{< copyable "shell-root" >}} + + ```bash + mkfs.ext4 /dev/nvme0n1p1 + ``` + +4. View the partition UUID of the data disk. + + In this example, the UUID of nvme0n1p1 is `c51eb23b-195c-4061-92a9-3fad812cc12f`. + + {{< copyable "shell-root" >}} + + ```bash + lsblk -f + ``` + + ``` + NAME FSTYPE LABEL UUID MOUNTPOINT + sda + ├─sda1 ext4 237b634b-a565-477b-8371-6dff0c41f5ab /boot + ├─sda2 swap f414c5c0-f823-4bb1-8fdf-e531173a72ed + └─sda3 ext4 547909c1-398d-4696-94c6-03e43e317b60 / + sr0 + nvme0n1 + └─nvme0n1p1 ext4 c51eb23b-195c-4061-92a9-3fad812cc12f + ``` + +5. Edit the `/etc/fstab` file and add the `nodelalloc` mount options. + + {{< copyable "shell-root" >}} + + ```bash + vi /etc/fstab + ``` + + ``` + UUID=c51eb23b-195c-4061-92a9-3fad812cc12f /data1 ext4 defaults,nodelalloc,noatime 0 2 + ``` + +6. Mount the data disk. + + {{< copyable "shell-root" >}} + + ```bash + mkdir /data1 && \ + mount -a + ``` + +7. Check using the following command. + + {{< copyable "shell-root" >}} + + ```bash + mount -t ext4 + ``` + + ``` + /dev/nvme0n1p1 on /data1 type ext4 (rw,noatime,nodelalloc,data=ordered) + ``` + + If the filesystem is ext4 and `nodelalloc` is included in the mount options, you have successfully mount the data disk ext4 filesystem with options on the target machines. + +## Check and disable system swap + +This section describes how to disable swap. + +TiDB requires sufficient memory space for operation. It is not recommended to use swap as a buffer for insufficient memory, which might reduce performance. Therefore, it is recommended to disable the system swap permanently. + +Do not disable the system swap by executing `swapoff -a`, or this setting will be invalid after the machine is restarted. + +To disable the system swap, execute the following command: + +{{< copyable "shell-regular" >}} + +```bash +echo "vm.swappiness = 0">> /etc/sysctl.conf +swapoff -a && swapon -a +sysctl -p +``` + +## Check and stop the firewall service of target machines + +In TiDB clusters, the access ports between nodes must be open to ensure the transmission of information such as read and write requests and data heartbeats. In common online scenarios, the data interaction between the database and the application service and between the database nodes are all made within a secure network. Therefore, if there are no special security requirements, it is recommended to stop the firewall of the target machine. Otherwise, refer to [the port usage](/hardware-and-software-requirements.md#network-requirements) and add the needed port information to the whitelist of the firewall service. + +The rest of this section describes how to stop the firewall service of a target machine. + +1. Check the firewall status. Take CentOS Linux release 7.7.1908 (Core) as an example. + + {{< copyable "shell-regular" >}} + + ```shell + sudo firewall-cmd --state + sudo systemctl status firewalld.service + ``` + +2. Stop the firewall service. + + {{< copyable "shell-regular" >}} + + ```bash + sudo systemctl stop firewalld.service + ``` + +3. Disable automatic start of the firewall service. + + {{< copyable "shell-regular" >}} + + ```bash + sudo systemctl disable firewalld.service + ``` + +4. Check the firewall status. + + {{< copyable "shell-regular" >}} + + ```bash + sudo systemctl status firewalld.service + ``` + +## Check and install the NTP service + +TiDB is a distributed database system that requires clock synchronization between nodes to guarantee linear consistency of transactions in the ACID model. + +At present, the common solution to clock synchronization is to use the Network Time Protocol (NTP) services. You can use the `pool.ntp.org` timing service on the Internet, or build your own NTP service in an offline environment. + +To check whether the NTP service is installed and whether it synchronizes with the NTP server normally, take the following steps: + +1. Run the following command. If it returns `running`, then the NTP service is running. + + {{< copyable "shell-regular" >}} + + ```bash + sudo systemctl status ntpd.service + ``` + + ``` + ntpd.service - Network Time Service + Loaded: loaded (/usr/lib/systemd/system/ntpd.service; disabled; vendor preset: disabled) + Active: active (running) since 一 2017-12-18 13:13:19 CST; 3s ago + ``` + +2. Run the `ntpstat` command to check whether the NTP service synchronizes with the NTP server. + + > **Note:** + > + > For the Ubuntu system, you need to install the `ntpstat` package. + + {{< copyable "shell-regular" >}} + + ```bash + ntpstat + ``` + + - If it returns `synchronised to NTP server` (synchronizing with the NTP server), then the synchronization process is normal. + + ``` + synchronised to NTP server (85.199.214.101) at stratum 2 + time correct to within 91 ms + polling server every 1024 s + ``` + + - The following situation indicates the NTP service is not synchronizing normally: + + ``` + unsynchronised + ``` + + - The following situation indicates the NTP service is not running normally: + + ``` + Unable to talk to NTP daemon. Is it running? + ``` + +To make the NTP service start synchronizing as soon as possible, run the following command. Replace `pool.ntp.org` with your NTP server. + +{{< copyable "shell-regular" >}} + +```bash +sudo systemctl stop ntpd.service && \ +sudo ntpdate pool.ntp.org && \ +sudo systemctl start ntpd.service +``` + +To install the NTP service manually on the CentOS 7 system, run the following command: + +{{< copyable "shell-regular" >}} + +```bash +sudo yum install ntp ntpdate && \ +sudo systemctl start ntpd.service && \ +sudo systemctl enable ntpd.service +``` + +## Manually configure the SSH mutual trust and sudo without password + +This section describes how to manually configure the SSH mutual trust and sudo without password. It is recommended to use TiUP for deployment, which automatically configure SSH mutual trust and login without password. If you deploy TiDB clusters using TiUP, ignore this section. + +1. Log in to the target machine respectively using the `root` user account, create the `tidb` user and set the login password. + + {{< copyable "shell-root" >}} + + ```bash + useradd tidb && \ + passwd tidb + ``` + +2. To configure sudo without password, run the following command, and add `tidb ALL=(ALL) NOPASSWD: ALL` to the end of the file: + + {{< copyable "shell-root" >}} + + ```bash + visudo + ``` + + ``` + tidb ALL=(ALL) NOPASSWD: ALL + ``` + +3. Use the `tidb` user to log in to the control machine, and run the following command. Replace `10.0.1.1` with the IP of your target machine, and enter the `tidb` user password of the target machine as prompted. After the command is executed, SSH mutual trust is already created. This applies to other machines as well. + + {{< copyable "shell-regular" >}} + + ```bash + ssh-copy-id -i ~/.ssh/id_rsa.pub 10.0.1.1 + ``` + +4. Log in to the control machine using the `tidb` user account, and log in to the IP of the target machine using `ssh`. If you do not need to enter the password and can successfully log in, then the SSH mutual trust is successfully configured. + + {{< copyable "shell-regular" >}} + + ```bash + ssh 10.0.1.1 + ``` + + ``` + [tidb@10.0.1.1 ~]$ + ``` + +5. After you log in to the target machine using the `tidb` user, run the following command. If you do not need to enter the password and can switch to the `root` user, then sudo without password of the `tidb` user is successfully configured. + + {{< copyable "shell-regular" >}} + + ```bash + sudo -su root + ``` + + ``` + [root@10.0.1.1 tidb]# + ``` + +## Install the `numactl` tool + +This section describes how to install the NUMA tool. In online environments, because the hardware configuration is usually higher than required, to better plan the hardware resources, multiple instances of TiDB or TiKV can be deployed on a single machine. In such scenarios, you can use NUMA tools to prevent the competition for CPU resources which might cause reduced performance. + +> **Note:** +> +> - Binding cores using NUMA is a method to isolate CPU resources and is suitable for deploying multiple instances on highly configured physical machines. +> - After completing deployment using `tiup cluster deploy`, you can use the `exec` command to perform cluster level management operations. + +1. Log in to the target node to install. Take CentOS Linux release 7.7.1908 (Core) as an example. + + {{< copyable "shell-regular" >}} + + ```bash + sudo yum -y install numactl + ``` + +2. Run the `exec` command using `tiup cluster` to install in batches. + + {{< copyable "shell-regular" >}} + + ```bash + tiup cluster exec --help + ``` + + ``` + Run shell command on host in the tidb cluster + Usage: + cluster exec [flags] + Flags: + --command string the command run on cluster host (default "ls") + -h, --help help for exec + --sudo use root permissions (default false) + ``` + + To use the sudo privilege to execute the installation command for all the target machines in the `tidb-test` cluster, run the following command: + + {{< copyable "shell-regular" >}} + + ```bash + tiup cluster exec tidb-test --sudo --command "yum -y install numactl" + ``` diff --git a/command-line-flags-for-pd-configuration.md b/command-line-flags-for-pd-configuration.md index 54685c083ac7b..bf4ed6144d9f2 100644 --- a/command-line-flags-for-pd-configuration.md +++ b/command-line-flags-for-pd-configuration.md @@ -102,12 +102,6 @@ PD is configurable using command-line flags and environment variables. - The path of the PEM file including the X509 key, used to enable TLS - Default: "" -## `--namespace-classifier` - -- To specify the namespace classifier used by PD -- Default: "table" -- If you use TiKV separately, not in the entire TiDB cluster, it is recommended to configure the value to 'default'. - ## `--metrics-addr` - The address of Prometheus Pushgateway, which does not push data to Promethus by default. diff --git a/config-templates/complex-cdc.yaml b/config-templates/complex-cdc.yaml new file mode 100644 index 0000000000000..e4267e371912a --- /dev/null +++ b/config-templates/complex-cdc.yaml @@ -0,0 +1,126 @@ +# # Global variables are applied to all deployments and used as the default value of +# # the deployments if a specific deployment value is missing. +global: + user: "tidb" + ssh_port: 22 + deploy_dir: "/tidb-deploy" + data_dir: "/tidb-data" + +# # Monitored variables are applied to all the machines. +monitored: + node_exporter_port: 9100 + blackbox_exporter_port: 9115 + # deploy_dir: "/tidb-deploy/monitored-9100" + # data_dir: "/tidb-data/monitored-9100" + # log_dir: "/tidb-deploy/monitored-9100/log" + +# # Server configs are used to specify the runtime configuration of TiDB components. +# # All configuration items can be found in TiDB docs: +# # - TiDB: https://pingcap.com/docs/stable/reference/configuration/tidb-server/configuration-file/ +# # - TiKV: https://pingcap.com/docs/stable/reference/configuration/tikv-server/configuration-file/ +# # - PD: https://pingcap.com/docs/stable/reference/configuration/pd-server/configuration-file/ +# # All configuration items use points to represent the hierarchy, e.g: +# # readpool.storage.use-unified-pool +# # +# # You can overwrite this configuration via the instance-level `config` field. + +server_configs: + tidb: + log.slow-threshold: 300 + tikv: + # server.grpc-concurrency: 4 + # raftstore.apply-pool-size: 2 + # raftstore.store-pool-size: 2 + # rocksdb.max-sub-compactions: 1 + # storage.block-cache.capacity: "16GB" + # readpool.unified.max-thread-count: 12 + readpool.storage.use-unified-pool: false + readpool.coprocessor.use-unified-pool: true + pd: + schedule.leader-schedule-limit: 4 + schedule.region-schedule-limit: 2048 + schedule.replica-schedule-limit: 64 + +pd_servers: + - host: 10.0.1.4 + # ssh_port: 22 + # name: "pd-1" + # client_port: 2379 + # peer_port: 2380 + # deploy_dir: "/tidb-deploy/pd-2379" + # data_dir: "/tidb-data/pd-2379" + # log_dir: "/tidb-deploy/pd-2379/log" + # numa_node: "0,1" + # # The following configs are used to overwrite the `server_configs.pd` values. + # config: + # schedule.max-merge-region-size: 20 + # schedule.max-merge-region-keys: 200000 + - host: 10.0.1.5 + - host: 10.0.1.6 + +tidb_servers: + - host: 10.0.1.1 + # ssh_port: 22 + # port: 4000 + # status_port: 10080 + # deploy_dir: "/tidb-deploy/tidb-4000" + # log_dir: "/tidb-deploy/tidb-4000/log" + # numa_node: "0,1" + # # The following configs are used to overwrite the `server_configs.tidb` values. + # config: + # log.slow-query-file: tidb-slow-overwrited.log + - host: 10.0.1.2 + - host: 10.0.1.3 + +tikv_servers: + - host: 10.0.1.7 + # ssh_port: 22 + # port: 20160 + # status_port: 20180 + # deploy_dir: "/tidb-deploy/tikv-20160" + # data_dir: "/tidb-data/tikv-20160" + # log_dir: "/tidb-deploy/tikv-20160/log" + # numa_node: "0,1" + # # The following configs are used to overwrite the `server_configs.tikv` values. + # config: + # server.grpc-concurrency: 4 + # server.labels: { zone: "zone1", dc: "dc1", host: "host1" } + + - host: 10.0.1.8 + - host: 10.0.1.9 + +cdc_servers: + - host: 10.0.1.1 + port: 8300 + deploy_dir: "/tidb-deploy/cdc-8300" + log_dir: "/tidb-deploy/cdc-8300/log" + - host: 10.0.1.2 + port: 8300 + deploy_dir: "/tidb-deploy/cdc-8300" + log_dir: "/tidb-deploy/cdc-8300/log" + - host: 10.0.1.3 + port: 8300 + deploy_dir: "/tidb-deploy/cdc-8300" + log_dir: "/tidb-deploy/cdc-8300/log" + +monitoring_servers: + - host: 10.0.1.10 + # ssh_port: 22 + # port: 9090 + # deploy_dir: "/tidb-deploy/prometheus-8249" + # data_dir: "/tidb-data/prometheus-8249" + # log_dir: "/tidb-deploy/prometheus-8249/log" + +grafana_servers: + - host: 10.0.1.10 + # port: 3000 + # deploy_dir: /tidb-deploy/grafana-3000 + +alertmanager_servers: + - host: 10.0.1.10 + # ssh_port: 22 + # web_port: 9093 + # cluster_port: 9094 + # deploy_dir: "/tidb-deploy/alertmanager-9093" + # data_dir: "/tidb-data/alertmanager-9093" + # log_dir: "/tidb-deploy/alertmanager-9093/log" \ No newline at end of file diff --git a/config-templates/complex-mini.yaml b/config-templates/complex-mini.yaml new file mode 100644 index 0000000000000..4a4807867a2ac --- /dev/null +++ b/config-templates/complex-mini.yaml @@ -0,0 +1,113 @@ +# # Global variables are applied to all deployments and used as the default value of +# # the deployments if a specific deployment value is missing. +global: + user: "tidb" + ssh_port: 22 + deploy_dir: "/tidb-deploy" + data_dir: "/tidb-data" + +# # Monitored variables are applied to all the machines. +monitored: + node_exporter_port: 9100 + blackbox_exporter_port: 9115 + # deploy_dir: "/tidb-deploy/monitored-9100" + # data_dir: "/tidb-data/monitored-9100" + # log_dir: "/tidb-deploy/monitored-9100/log" + +# # Server configs are used to specify the runtime configuration of TiDB components. +# # All configuration items can be found in TiDB docs: +# # - TiDB: https://pingcap.com/docs/stable/reference/configuration/tidb-server/configuration-file/ +# # - TiKV: https://pingcap.com/docs/stable/reference/configuration/tikv-server/configuration-file/ +# # - PD: https://pingcap.com/docs/stable/reference/configuration/pd-server/configuration-file/ +# # All configuration items use points to represent the hierarchy, e.g: +# # readpool.storage.use-unified-pool +# # +# # You can overwrite this configuration via the instance-level `config` field. + +server_configs: + tidb: + log.slow-threshold: 300 + binlog.enable: false + binlog.ignore-error: false + tikv: + # server.grpc-concurrency: 4 + # raftstore.apply-pool-size: 2 + # raftstore.store-pool-size: 2 + # rocksdb.max-sub-compactions: 1 + # storage.block-cache.capacity: "16GB" + # readpool.unified.max-thread-count: 12 + readpool.storage.use-unified-pool: false + readpool.coprocessor.use-unified-pool: true + pd: + schedule.leader-schedule-limit: 4 + schedule.region-schedule-limit: 2048 + schedule.replica-schedule-limit: 64 + +pd_servers: + - host: 10.0.1.4 + # ssh_port: 22 + # name: "pd-1" + # client_port: 2379 + # peer_port: 2380 + # deploy_dir: "/tidb-deploy/pd-2379" + # data_dir: "/tidb-data/pd-2379" + # log_dir: "/tidb-deploy/pd-2379/log" + # numa_node: "0,1" + # # The following configs are used to overwrite the `server_configs.pd` values. + # config: + # schedule.max-merge-region-size: 20 + # schedule.max-merge-region-keys: 200000 + - host: 10.0.1.5 + - host: 10.0.1.6 + +tidb_servers: + - host: 10.0.1.1 + # ssh_port: 22 + # port: 4000 + # status_port: 10080 + # deploy_dir: "/tidb-deploy/tidb-4000" + # log_dir: "/tidb-deploy/tidb-4000/log" + # numa_node: "0,1" + # # The following configs are used to overwrite the `server_configs.tidb` values. + # config: + # log.slow-query-file: tidb-slow-overwrited.log + - host: 10.0.1.2 + - host: 10.0.1.3 + +tikv_servers: + - host: 10.0.1.7 + # ssh_port: 22 + # port: 20160 + # status_port: 20180 + # deploy_dir: "/tidb-deploy/tikv-20160" + # data_dir: "/tidb-data/tikv-20160" + # log_dir: "/tidb-deploy/tikv-20160/log" + # numa_node: "0,1" + # # The following configs are used to overwrite the `server_configs.tikv` values. + # config: + # server.grpc-concurrency: 4 + # server.labels: { zone: "zone1", dc: "dc1", host: "host1" } + - host: 10.0.1.8 + - host: 10.0.1.9 + +monitoring_servers: + - host: 10.0.1.10 + # ssh_port: 22 + # port: 9090 + # deploy_dir: "/tidb-deploy/prometheus-8249" + # data_dir: "/tidb-data/prometheus-8249" + # log_dir: "/tidb-deploy/prometheus-8249/log" + +grafana_servers: + - host: 10.0.1.10 + # port: 3000 + # deploy_dir: /tidb-deploy/grafana-3000 + +alertmanager_servers: + - host: 10.0.1.10 + # ssh_port: 22 + # web_port: 9093 + # cluster_port: 9094 + # deploy_dir: "/tidb-deploy/alertmanager-9093" + # data_dir: "/tidb-data/alertmanager-9093" + # log_dir: "/tidb-deploy/alertmanager-9093/log" \ No newline at end of file diff --git a/config-templates/complex-multi-instance.yaml b/config-templates/complex-multi-instance.yaml new file mode 100644 index 0000000000000..2513d0b3869ca --- /dev/null +++ b/config-templates/complex-multi-instance.yaml @@ -0,0 +1,150 @@ +# # Global variables are applied to all deployments and used as the default value of +# # the deployments if a specific deployment value is missing. +global: + user: "tidb" + ssh_port: 22 + deploy_dir: "/tidb-deploy" + data_dir: "/tidb-data" + +monitored: + node_exporter_port: 9100 + blackbox_exporter_port: 9115 + deploy_dir: "/tidb-deploy/monitored-9100" + data_dir: "/tidb-data-monitored-9100" + log_dir: "/tidb-deploy/monitored-9100/log" + +server_configs: + tidb: + log.slow-threshold: 300 + tikv: + readpool.unified.max-thread-count: + readpool.storage.use-unified-pool: false + readpool.coprocessor.use-unified-pool: true + storage.block-cache.capacity: "" + raftstore.capacity: "" + pd: + replication.location-labels: ["host"] + schedule.leader-schedule-limit: 4 + schedule.region-schedule-limit: 2048 + schedule.replica-schedule-limit: 64 + +pd_servers: + - host: 10.0.1.4 + - host: 10.0.1.5 + - host: 10.0.1.6 + +tidb_servers: + - host: 10.0.1.1 + port: 4000 + status_port: 10080 + deploy_dir: "/tidb-deploy/tidb-4000" + log_dir: "/tidb-deploy/tidb-4000/log" + numa_node: "0" + - host: 10.0.1.1 + port: 4001 + status_port: 10081 + deploy_dir: "/tidb-deploy/tidb-4001" + log_dir: "/tidb-deploy/tidb-4001/log" + numa_node: "1" + - host: 10.0.1.2 + port: 4000 + status_port: 10080 + deploy_dir: "/tidb-deploy/tidb-4000" + log_dir: "/tidb-deploy/tidb-4000/log" + numa_node: "0" + - host: 10.0.1.2 + port: 4001 + status_port: 10081 + deploy_dir: "/tidb-deploy/tidb-4001" + log_dir: "/tidb-deploy/tidb-4001/log" + numa_node: "1" + - host: 10.0.1.3 + port: 4000 + status_port: 10080 + deploy_dir: "/tidb-deploy/tidb-4000" + log_dir: "/tidb-deploy/tidb-4000/log" + numa_node: "0" + - host: 10.0.1.3 + port: 4001 + status_port: 10081 + deploy_dir: "/tidb-deploy/tidb-4001" + log_dir: "/tidb-deploy/tidb-4001/log" + numa_node: "1" + +tikv_servers: + - host: 10.0.1.7 + port: 20160 + status_port: 20180 + deploy_dir: "/tidb-deploy/tikv-20160" + data_dir: "/tidb-data/tikv-20160" + log_dir: "/tidb-deploy/tikv-20160/log" + numa_node: "0" + config: + server.labels: { host: "tikv1" } + - host: 10.0.1.7 + port: 20161 + status_port: 20181 + deploy_dir: "/tidb-deploy/tikv-20161" + data_dir: "/tidb-data/tikv-20161" + log_dir: "/tidb-deploy/tikv-20161/log" + numa_node: "1" + config: + server.labels: { host: "tikv1" } + - host: 10.0.1.8 + port: 20160 + status_port: 20180 + deploy_dir: "/tidb-deploy/tikv-20160" + data_dir: "/tidb-data/tikv-20160" + log_dir: "/tidb-deploy/tikv-20160/log" + numa_node: "0" + config: + server.labels: { host: "tikv2" } + - host: 10.0.1.8 + port: 20161 + status_port: 20181 + deploy_dir: "/tidb-deploy/tikv-20161" + data_dir: "/tidb-data/tikv-20161" + log_dir: "/tidb-deploy/tikv-20161/log" + numa_node: "1" + config: + server.labels: { host: "tikv2" } + - host: 10.0.1.9 + port: 20160 + status_port: 20180 + deploy_dir: "/tidb-deploy/tikv-20160" + data_dir: "/tidb-data/tikv-20160" + log_dir: "/tidb-deploy/tikv-20160/log" + numa_node: "0" + config: + server.labels: { host: "tikv3" } + - host: 10.0.1.9 + port: 20161 + status_port: 20181 + deploy_dir: "/tidb-deploy/tikv-20161" + data_dir: "/tidb-data/tikv-20161" + log_dir: "/tidb-deploy/tikv-20161/log" + numa_node: "1" + config: + server.labels: { host: "tikv3" } + +monitoring_servers: + - host: 10.0.1.10 + # ssh_port: 22 + # port: 9090 + # deploy_dir: "/tidb-deploy/prometheus-8249" + # data_dir: "/tidb-data/prometheus-8249" + # log_dir: "/tidb-deploy/prometheus-8249/log" + +grafana_servers: + - host: 10.0.1.10 + # port: 3000 + # deploy_dir: /tidb-deploy/grafana-3000 + +alertmanager_servers: + - host: 10.0.1.10 + # ssh_port: 22 + # web_port: 9093 + # cluster_port: 9094 + # deploy_dir: "/tidb-deploy/alertmanager-9093" + # data_dir: "/tidb-data/alertmanager-9093" + # log_dir: "/tidb-deploy/alertmanager-9093/log" \ No newline at end of file diff --git a/config-templates/complex-tidb-binlog.yaml b/config-templates/complex-tidb-binlog.yaml new file mode 100644 index 0000000000000..36db4f2ca451b --- /dev/null +++ b/config-templates/complex-tidb-binlog.yaml @@ -0,0 +1,152 @@ +# # Global variables are applied to all deployments and used as the default value of +# # the deployments if a specific deployment value is missing. +global: + user: "tidb" + ssh_port: 22 + deploy_dir: "/tidb-deploy" + data_dir: "/tidb-data" + +# # Monitored variables are applied to all the machines. +monitored: + node_exporter_port: 9100 + blackbox_exporter_port: 9115 + # deploy_dir: "/tidb-deploy/monitored-9100" + # data_dir: "/tidb-data/monitored-9100" + # log_dir: "/tidb-deploy/monitored-9100/log" + +# # Server configs are used to specify the runtime configuration of TiDB components. +# # All configuration items can be found in TiDB docs: +# # - TiDB: https://pingcap.com/docs/stable/reference/configuration/tidb-server/configuration-file/ +# # - TiKV: https://pingcap.com/docs/stable/reference/configuration/tikv-server/configuration-file/ +# # - PD: https://pingcap.com/docs/stable/reference/configuration/pd-server/configuration-file/ +# # All configuration items use points to represent the hierarchy, e.g: +# # readpool.storage.use-unified-pool +# # +# # You can overwrite this configuration via the instance-level `config` field. + +server_configs: + tidb: + log.slow-threshold: 300 + binlog.enable: true + binlog.ignore-error: true + tikv: + # server.grpc-concurrency: 4 + # raftstore.apply-pool-size: 2 + # raftstore.store-pool-size: 2 + # rocksdb.max-sub-compactions: 1 + # storage.block-cache.capacity: "16GB" + # readpool.unified.max-thread-count: 12 + readpool.storage.use-unified-pool: false + readpool.coprocessor.use-unified-pool: true + pd: + schedule.leader-schedule-limit: 4 + schedule.region-schedule-limit: 2048 + schedule.replica-schedule-limit: 64 + +pd_servers: + - host: 10.0.1.4 + # ssh_port: 22 + # name: "pd-1" + # client_port: 2379 + # peer_port: 2380 + # deploy_dir: "/tidb-deploy/pd-2379" + # data_dir: "/tidb-data/pd-2379" + # log_dir: "/tidb-deploy/pd-2379/log" + # numa_node: "0,1" + # # The following configs are used to overwrite the `server_configs.pd` values. + # config: + # schedule.max-merge-region-size: 20 + # schedule.max-merge-region-keys: 200000 + - host: 10.0.1.5 + - host: 10.0.1.6 +tidb_servers: + - host: 10.0.1.1 + # ssh_port: 22 + # port: 4000 + # status_port: 10080 + # deploy_dir: "/tidb-deploy/tidb-4000" + # log_dir: "/tidb-deploy/tidb-4000/log" + # numa_node: "0,1" + # # The following configs are used to overwrite the `server_configs.tidb` values. + # config: + # log.slow-query-file: tidb-slow-overwrited.log + - host: 10.0.1.2 + - host: 10.0.1.3 +tikv_servers: + - host: 10.0.1.7 + # ssh_port: 22 + # port: 20160 + # status_port: 20180 + # deploy_dir: "/tidb-deploy/tikv-20160" + # data_dir: "/tidb-data/tikv-20160" + # log_dir: "/tidb-deploy/tikv-20160/log" + # numa_node: "0,1" + # # The following configs are used to overwrite the `server_configs.tikv` values. + # config: + # server.grpc-concurrency: 4 + # server.labels: { zone: "zone1", dc: "dc1", host: "host1" } + - host: 10.0.1.8 + - host: 10.0.1.9 + +pump_servers: + - host: 10.0.1.1 + ssh_port: 22 + port: 8250 + deploy_dir: "/tidb-deploy/pump-8249" + data_dir: "/tidb-data/pump-8249" + # The following configs are used to overwrite the `server_configs.drainer` values. + config: + gc: 7 + - host: 10.0.1.2 + ssh_port: 22 + port: 8250 + deploy_dir: "/tidb-deploy/pump-8249" + data_dir: "/tidb-data/pump-8249" + # The following configs are used to overwrite the `server_configs.drainer` values. + config: + gc: 7 + - host: 10.0.1.3 + ssh_port: 22 + port: 8250 + deploy_dir: "/tidb-deploy/pump-8249" + data_dir: "/tidb-data/pump-8249" + # The following configs are used to overwrite the `server_configs.drainer` values. + config: + gc: 7 +drainer_servers: + - host: 10.0.1.12 + port: 8249 + data_dir: "/tidb-data/drainer-8249" + # If drainer doesn't have a checkpoint, use initial commitTS as the initial checkpoint. + # Will get a latest timestamp from pd if commit_ts is set to -1 (the default value). + commit_ts: -1 + deploy_dir: "/tidb-deploy/drainer-8249" + # The following configs are used to overwrite the `server_configs.drainer` values. + config: + syncer.db-type: "tidb" + syncer.to.host: "10.0.1.12" + syncer.to.user: "root" + syncer.to.password: "" + syncer.to.port: 4000 + +monitoring_servers: + - host: 10.0.1.10 + # ssh_port: 22 + # port: 9090 + # deploy_dir: "/tidb-deploy/prometheus-8249" + # data_dir: "/tidb-data/prometheus-8249" + # log_dir: "/tidb-deploy/prometheus-8249/log" + +grafana_servers: + - host: 10.0.1.10 + # port: 3000 + # deploy_dir: /tidb-deploy/grafana-3000 + +alertmanager_servers: + - host: 10.0.1.10 + # ssh_port: 22 + # web_port: 9093 + # cluster_port: 9094 + # deploy_dir: "/tidb-deploy/alertmanager-9093" + # data_dir: "/tidb-data/alertmanager-9093" + # log_dir: "/tidb-deploy/alertmanager-9093/log" \ No newline at end of file diff --git a/config-templates/complex-tiflash.yaml b/config-templates/complex-tiflash.yaml new file mode 100644 index 0000000000000..99cf55e0f52b5 --- /dev/null +++ b/config-templates/complex-tiflash.yaml @@ -0,0 +1,131 @@ +# # Global variables are applied to all deployments and used as the default value of +# # the deployments if a specific deployment value is missing. +global: + user: "tidb" + ssh_port: 22 + deploy_dir: "/tidb-deploy" + data_dir: "/tidb-data" + +# # Monitored variables are applied to all the machines. +monitored: + node_exporter_port: 9100 + blackbox_exporter_port: 9115 + # deploy_dir: "/tidb-deploy/monitored-9100" + # data_dir: "/tidb-data/monitored-9100" + # log_dir: "/tidb-deploy/monitored-9100/log" + +# # Server configs are used to specify the runtime configuration of TiDB components. +# # All configuration items can be found in TiDB docs: +# # - TiDB: https://pingcap.com/docs/stable/reference/configuration/tidb-server/configuration-file/ +# # - TiKV: https://pingcap.com/docs/stable/reference/configuration/tikv-server/configuration-file/ +# # - PD: https://pingcap.com/docs/stable/reference/configuration/pd-server/configuration-file/ +# # All configuration items use points to represent the hierarchy, e.g: +# # readpool.storage.use-unified-pool +# # +# # You can overwrite this configuration via the instance-level `config` field. + +server_configs: + tidb: + log.slow-threshold: 300 + tikv: + # server.grpc-concurrency: 4 + # raftstore.apply-pool-size: 2 + # raftstore.store-pool-size: 2 + # rocksdb.max-sub-compactions: 1 + # storage.block-cache.capacity: "16GB" + # readpool.unified.max-thread-count: 12 + readpool.storage.use-unified-pool: false + readpool.coprocessor.use-unified-pool: true + pd: + schedule.leader-schedule-limit: 4 + schedule.region-schedule-limit: 2048 + schedule.replica-schedule-limit: 64 + replication.enable-placement-rules: true + +pd_servers: + - host: 10.0.1.4 + # ssh_port: 22 + # name: "pd-1" + # client_port: 2379 + # peer_port: 2380 + # deploy_dir: "/tidb-deploy/pd-2379" + # data_dir: "/tidb-data/pd-2379" + # log_dir: "/tidb-deploy/pd-2379/log" + # numa_node: "0,1" + # # The following configs are used to overwrite the `server_configs.pd` values. + # config: + # schedule.max-merge-region-size: 20 + # schedule.max-merge-region-keys: 200000 + - host: 10.0.1.5 + - host: 10.0.1.6 +tidb_servers: + - host: 10.0.1.7 + # ssh_port: 22 + # port: 4000 + # status_port: 10080 + # deploy_dir: "/tidb-deploy/tidb-4000" + # log_dir: "/tidb-deploy/tidb-4000/log" + # numa_node: "0,1" + # # The following configs are used to overwrite the `server_configs.tidb` values. + # config: + # log.slow-query-file: tidb-slow-overwrited.log + - host: 10.0.1.8 + - host: 10.0.1.9 +tikv_servers: + - host: 10.0.1.1 + # ssh_port: 22 + # port: 20160 + # status_port: 20180 + # deploy_dir: "/tidb-deploy/tikv-20160" + # data_dir: "/tidb-data/tikv-20160" + # log_dir: "/tidb-deploy/tikv-20160/log" + # numa_node: "0,1" + # # The following configs are used to overwrite the `server_configs.tikv` values. + # config: + # server.grpc-concurrency: 4 + # server.labels: { zone: "zone1", dc: "dc1", host: "host1" } + - host: 10.0.1.2 + - host: 10.0.1.3 + +tiflash_servers: + - host: 10.0.1.11 + data_dir: /tidb-data/tiflash-9000 + deploy_dir: /tidb-deploy/tiflash-9000 + # ssh_port: 22 + # tcp_port: 9000 + # http_port: 8123 + # flash_service_port: 3930 + # flash_proxy_port: 20170 + # flash_proxy_status_port: 20292 + # metrics_port: 8234 + # deploy_dir: /tidb-deploy/tiflash-9000 + # numa_node: "0,1" + # # The following configs are used to overwrite the `server_configs.tiflash` values. + # config: + # logger.level: "info" + # learner_config: + # log-level: "info" + # - host: 10.0.1.12 + # - host: 10.0.1.13 + +monitoring_servers: + - host: 10.0.1.10 + # ssh_port: 22 + # port: 9090 + # deploy_dir: "/tidb-deploy/prometheus-8249" + # data_dir: "/tidb-data/prometheus-8249" + # log_dir: "/tidb-deploy/prometheus-8249/log" + +grafana_servers: + - host: 10.0.1.10 + # port: 3000 + # deploy_dir: /tidb-deploy/grafana-3000 + +alertmanager_servers: + - host: 10.0.1.10 + # ssh_port: 22 + # web_port: 9093 + # cluster_port: 9094 + # deploy_dir: "/tidb-deploy/alertmanager-9093" + # data_dir: "/tidb-data/alertmanager-9093" + # log_dir: "/tidb-deploy/alertmanager-9093/log" \ No newline at end of file diff --git a/config-templates/geo-redundancy-deployment.yaml b/config-templates/geo-redundancy-deployment.yaml new file mode 100644 index 0000000000000..9f18ac900a60c --- /dev/null +++ b/config-templates/geo-redundancy-deployment.yaml @@ -0,0 +1,108 @@ +# Tip: PD priority needs to be manually set using the PD-ctl client tool. such as, member Leader_priority PD-name numbers. +# Global variables are applied to all deployments and used as the default value of +# the deployments if a specific deployment value is missing. + +global: + user: "tidb" + ssh_port: 22 + deploy_dir: "/tidb-deploy" + data_dir: "/tidb-data" +monitored: + node_exporter_port: 9100 + blackbox_exporter_port: 9115 + deploy_dir: "/tidb-deploy/monitored-9100" +server_configs: + tidb: + log.level: debug + log.slow-query-file: tidb-slow.log + tikv: + server.grpc-compression-type: gzip + readpool.storage.use-unified-pool: true + readpool.storage.low-concurrency: 8 + pd: + replication.location-labels: ["zone","dc","rack","host"] + replication.max-replicas: 5 + label-property: + reject-leader: + - key: "dc" + value: "sha" +pd_servers: + - host: 10.0.1.6 + - host: 10.0.1.7 + - host: 10.0.1.8 + - host: 10.0.1.9 + - host: 10.0.1.10 +tidb_servers: + - host: 10.0.1.1 + - host: 10.0.1.2 + - host: 10.0.1.3 + - host: 10.0.1.4 + - host: 10.0.1.5 +tikv_servers: + - host: 10.0.1.11 + ssh_port: 22 + port: 20160 + status_port: 20180 + deploy_dir: "/tidb-deploy/tikv-20160" + data_dir: "/tidb-data/tikv-20160" + config: + server.labels: + zone: bj + dc: bja + rack: rack1 + host: host1 + - host: 10.0.1.12 + ssh_port: 22 + port: 20161 + status_port: 20181 + deploy_dir: "/tidb-deploy/tikv-20161" + data_dir: "/tidb-data/tikv-20161" + config: + server.labels: + zone: bj + dc: bja + rack: rack1 + host: host2 + - host: 10.0.1.13 + ssh_port: 22 + port: 20160 + status_port: 20180 + deploy_dir: "/tidb-deploy/tikv-20160" + data_dir: "/tidb-data/tikv-20160" + config: + server.labels: + zone: bj + dc: bjb + rack: rack1 + host: host1 + - host: 10.0.1.14 + ssh_port: 22 + port: 20161 + status_port: 20181 + deploy_dir: "/tidb-deploy/tikv-20161" + data_dir: "/tidb-data/tikv-20161" + config: + server.labels: + zone: bj + dc: bjb + rack: rack1 + host: host2 + - host: 10.0.1.15 + ssh_port: 22 + port: 20160 + deploy_dir: "/tidb-deploy/tikv-20160" + data_dir: "/tidb-data/tikv-20160" + config: + server.labels: + zone: sh + dc: sha + rack: rack1 + host: host1 + readpool.storage.use-unified-pool: true + readpool.storage.low-concurrency: 10 + raftstore.raft-min-election-timeout-ticks: 1000 + raftstore.raft-max-election-timeout-ticks: 1020 +monitoring_servers: + - host: 10.0.1.16 +grafana_servers: + - host: 10.0.1.16 \ No newline at end of file diff --git a/config-templates/simple-cdc.yaml b/config-templates/simple-cdc.yaml new file mode 100644 index 0000000000000..4d820082de20f --- /dev/null +++ b/config-templates/simple-cdc.yaml @@ -0,0 +1,36 @@ +# # Global variables are applied to all deployments and used as the default value of +# # the deployments if a specific deployment value is missing. +global: + user: "tidb" + ssh_port: 22 + deploy_dir: "/tidb-deploy" + data_dir: "/tidb-data" + +pd_servers: + - host: 10.0.1.4 + - host: 10.0.1.5 + - host: 10.0.1.6 + +tidb_servers: + - host: 10.0.1.1 + - host: 10.0.1.2 + - host: 10.0.1.3 + +tikv_servers: + - host: 10.0.1.7 + - host: 10.0.1.8 + - host: 10.0.1.9 + +cdc_servers: + - host: 10.0.1.7 + - host: 10.0.1.8 + - host: 10.0.1.9 + +monitoring_servers: + - host: 10.0.1.10 + +grafana_servers: + - host: 10.0.1.10 + +alertmanager_servers: + - host: 10.0.1.10 \ No newline at end of file diff --git a/config-templates/simple-mini.yaml b/config-templates/simple-mini.yaml new file mode 100644 index 0000000000000..57e48fd2d7416 --- /dev/null +++ b/config-templates/simple-mini.yaml @@ -0,0 +1,31 @@ +# # Global variables are applied to all deployments and used as the default value of +# # the deployments if a specific deployment value is missing. +global: + user: "tidb" + ssh_port: 22 + deploy_dir: "/tidb-deploy" + data_dir: "/tidb-data" + +pd_servers: + - host: 10.0.1.4 + - host: 10.0.1.5 + - host: 10.0.1.6 + +tidb_servers: + - host: 10.0.1.1 + - host: 10.0.1.2 + - host: 10.0.1.3 + +tikv_servers: + - host: 10.0.1.7 + - host: 10.0.1.8 + - host: 10.0.1.9 + +monitoring_servers: + - host: 10.0.1.10 + +grafana_servers: + - host: 10.0.1.10 + +alertmanager_servers: + - host: 10.0.1.10 \ No newline at end of file diff --git a/config-templates/simple-multi-instance.yaml b/config-templates/simple-multi-instance.yaml new file mode 100644 index 0000000000000..735b3be1af61a --- /dev/null +++ b/config-templates/simple-multi-instance.yaml @@ -0,0 +1,95 @@ +# # Global variables are applied to all deployments and used as the default value of +# # the deployments if a specific deployment value is missing. +global: + user: "tidb" + ssh_port: 22 + deploy_dir: "/tidb-deploy" + data_dir: "/tidb-data" + +server_configs: + tikv: + readpool.unified.max-thread-count: + readpool.storage.use-unified-pool: false + readpool.coprocessor.use-unified-pool: true + storage.block-cache.capacity: "" + raftstore.capacity: "" + pd: + replication.location-labels: ["host"] + +pd_servers: + - host: 10.0.1.4 + - host: 10.0.1.5 + - host: 10.0.1.6 + +tidb_servers: + - host: 10.0.1.1 + port: 4000 + status_port: 10080 + numa_node: "0" + - host: 10.0.1.1 + port: 4001 + status_port: 10081 + numa_node: "1" + - host: 10.0.1.2 + port: 4000 + status_port: 10080 + numa_node: "0" + - host: 10.0.1.2 + port: 4001 + status_port: 10081 + numa_node: "1" + - host: 10.0.1.3 + port: 4000 + status_port: 10080 + numa_node: "0" + - host: 10.0.1.3 + port: 4001 + status_port: 10081 + numa_node: "1" + +tikv_servers: + - host: 10.0.1.7 + port: 20160 + status_port: 20180 + numa_node: "0" + config: + server.labels: { host: "tikv1" } + - host: 10.0.1.7 + port: 20161 + status_port: 20181 + numa_node: "1" + config: + server.labels: { host: "tikv1" } + - host: 10.0.1.8 + port: 20160 + status_port: 20180 + numa_node: "0" + config: + server.labels: { host: "tikv2" } + - host: 10.0.1.8 + port: 20161 + status_port: 20181 + numa_node: "1" + config: + server.labels: { host: "tikv2" } + - host: 10.0.1.9 + port: 20160 + status_port: 20180 + numa_node: "0" + config: + server.labels: { host: "tikv3" } + - host: 10.0.1.9 + port: 20161 + status_port: 20181 + numa_node: "1" + config: + server.labels: { host: "tikv3" } + +monitoring_servers: + - host: 10.0.1.10 + +grafana_servers: + - host: 10.0.1.10 + +alertmanager_servers: + - host: 10.0.1.10 \ No newline at end of file diff --git a/config-templates/simple-tidb-binlog.yaml b/config-templates/simple-tidb-binlog.yaml new file mode 100644 index 0000000000000..558023e003561 --- /dev/null +++ b/config-templates/simple-tidb-binlog.yaml @@ -0,0 +1,47 @@ +# # Global variables are applied to all deployments and used as the default value of +# # the deployments if a specific deployment value is missing. +global: + user: "tidb" + ssh_port: 22 + deploy_dir: "/tidb-deploy" + data_dir: "/tidb-data" + +server_configs: + tidb: + binlog.enable: true + binlog.ignore-error: true + +pd_servers: + - host: 10.0.1.4 + - host: 10.0.1.5 + - host: 10.0.1.6 +tidb_servers: + - host: 10.0.1.1 + - host: 10.0.1.2 + - host: 10.0.1.3 +tikv_servers: + - host: 10.0.1.7 + - host: 10.0.1.8 + - host: 10.0.1.9 + +pump_servers: + - host: 10.0.1.1 + - host: 10.0.1.2 + - host: 10.0.1.3 +drainer_servers: + - host: 10.0.1.12 + config: + syncer.db-type: "tidb" + syncer.to.host: "10.0.1.12" + syncer.to.user: "root" + syncer.to.password: "" + syncer.to.port: 4000 + +monitoring_servers: + - host: 10.0.1.10 + +grafana_servers: + - host: 10.0.1.10 + +alertmanager_servers: + - host: 10.0.1.10 \ No newline at end of file diff --git a/config-templates/simple-tiflash.yaml b/config-templates/simple-tiflash.yaml new file mode 100644 index 0000000000000..1e20244408541 --- /dev/null +++ b/config-templates/simple-tiflash.yaml @@ -0,0 +1,40 @@ +# # Global variables are applied to all deployments and used as the default value of +# # the deployments if a specific deployment value is missing. +global: + user: "tidb" + ssh_port: 22 + deploy_dir: "/tidb-deploy" + data_dir: "/tidb-data" + +server_configs: + pd: + replication.enable-placement-rules: true + +pd_servers: + - host: 10.0.1.4 + - host: 10.0.1.5 + - host: 10.0.1.6 + +tidb_servers: + - host: 10.0.1.7 + - host: 10.0.1.8 + - host: 10.0.1.9 + +tikv_servers: + - host: 10.0.1.1 + - host: 10.0.1.2 + - host: 10.0.1.3 + +tiflash_servers: + - host: 10.0.1.11 + data_dir: /tidb-data/tiflash-9000 + deploy_dir: /tidb-deploy/tiflash-9000 + +monitoring_servers: + - host: 10.0.1.10 + +grafana_servers: + - host: 10.0.1.10 + +alertmanager_servers: + - host: 10.0.1.10 \ No newline at end of file diff --git a/configure-placement-rules.md b/configure-placement-rules.md index b43f81f28b5a2..060a9627b7df9 100644 --- a/configure-placement-rules.md +++ b/configure-placement-rules.md @@ -7,6 +7,10 @@ aliases: ['/docs/dev/how-to/configure/placement-rules/'] # Placement Rules +> **Warning:** +> +> In the scenario of using TiFlash, the Placement Rules feature has been extensively tested and can be used in the production environment. Except for the scenario where TiFlash is used, using Placement Rules alone has not been extensively tested, so it is not recommended to enable this feature separately in the production environment. + Placement Rules is an experimental feature of the Placement Driver (PD) introduced in v4.0. It is a replica rule system that guides PD to generate corresponding schedules for different types of data. By combining different scheduling rules, you can finely control the attributes of any continuous data range, such as the number of replicas, the storage location, the host type, whether to participate in Raft election, and whether to act as the Raft leader. ## Rule system @@ -86,6 +90,10 @@ pd-ctl config placement-rules enable PD also generates default rules based on the `max-replicas` and `location-labels` configurations. +> **Note:** +> +> After enabling Placement Rules, the previously configured `max-replicas` and `location-labels` no longer take effect. To adjust the replica policy, use the interface related to Placement Rules. + ### Disable Placement Rules You can use pd-ctl to disable the Placement Rules feature and switch to the previous scheduling strategy. diff --git a/connectors-and-apis.md b/connectors-and-apis.md index c25da5816fcba..94002eca41f9b 100644 --- a/connectors-and-apis.md +++ b/connectors-and-apis.md @@ -29,6 +29,10 @@ TiDB is compatible with all Connectors and APIs of MySQL (5.6, 5.7), including: Oracle develops the following APIs and TiDB is compatible with all of them: +> **Note:** +> +> + To connect to TiDB using a MySQL Connector from MySQL 8.0, you must explicitly specify `default-auth=mysql_native_password`, because `mysql_native_password` is [no longer the default plugin](https://dev.mysql.com/doc/refman/8.0/en/upgrading-from-previous-series.html#upgrade-caching-sha2-password). + - [MySQL Connector/C++](https://dev.mysql.com/doc/refman/5.7/en/connector-cpp-info.html)：to enable C++ applications to connect to MySQL - [MySQL Connector/J](https://dev.mysql.com/doc/refman/5.7/en/connector-j-info.html)：to enable Java applications to connect to MySQL using the standard JDBC API - [MySQL Connector/Net](https://dev.mysql.com/doc/refman/5.7/en/connector-net-info.html)：to enable .Net applications to connect to MySQL; [MySQL for Visual Studio](https://dev.mysql.com/doc/visual-studio/en/) uses this; support Microsoft Visual Studio 2012, 2013, 2015 and 2017 versions diff --git a/coprocessor-cache.md b/coprocessor-cache.md new file mode 100644 index 0000000000000..a6bfb10d15cac --- /dev/null +++ b/coprocessor-cache.md @@ -0,0 +1,39 @@ +--- +title: Coprocessor Cache +summary: Learn the features of Coprocessor Cache. +category: reference +--- + +# Coprocessor Cache + +Starting from v4.0, the TiDB instance supports caching the results of the calculation that is pushed down to TiKV (the Coprocessor Cache feature), which can accelerate the calculation process in some scenarios. + +## Configuration + +You can configure Coprocessor Cache via the `tikv-client.copr-cache` configuration items in the TiDB configuration file. For details about how to enable and configure Coprocessor Cache, see [TiDB Configuration File](/tidb-configuration-file.md#tikv-clientcopr-cache-new-in-v400). + +## Feature description + ++ When a SQL statement is executed on a single TiDB instance for the first time, the execution result is not cached. ++ Calculation results are cached in the memory of TiDB. If the TiDB instance is restarted, the cache becomes invalid. ++ The cache is not shared among TiDB instances. ++ Only push-down calculation result is cached. Even if cache is hit, TiDB still need to perform subsequent calculation. ++ The cache is in the unit of Region. Writing data to a Region causes the Region cache to be invalid. For this reason, the Coprocessor Cache feature mainly takes effect on the data that rarely changes. ++ When push-down calculation requests are the same, the cache is hit. Usually in the following scenarios, the push-down calculation requests are the same or partially the same: + - The SQL statements are the same. For example, the same SQL statement is executed repeatedly. + + In this scenario, all the push-down calculation requests are consistent, and all requests can use the push-down calculation cache. + + - The SQL statements contain a changing condition, and the other parts are consistent. The changing condition is the primary key of the table or the partition. + + In this scenario, some of the push-down calculation requests are the same with some previous requests, and these calculation requests can use the cached (previous) push-down calculation result. + + - The SQL statements contain multiple changing conditions and the other parts are consistent. The changing conditions exactly match a compound index column. + + In this scenario, some of the push-down calculation requests are the same with some previous requests, and these calculation requests can use the cached (previous) push-down calculation result. + ++ This feature is transparent to users. Enabling or disabling this feature does not affect the calculation result and only affects the SQL execution time. + +## Check the cache effect + +Currently (TiDB v4.0.0-rc.2), Coprocessor cache is still an experimental feature. Users cannot learn how many push-down requests hit the cache in a SQL statement, or understand the overall cache hits. The corresponding monitoring and checking method will be introduced in later TiDB versions. diff --git a/daily-check.md b/daily-check.md new file mode 100644 index 0000000000000..c118ebd015515 --- /dev/null +++ b/daily-check.md @@ -0,0 +1,82 @@ +--- +title: Daily Check +summary: Learn about performance indicators of the TiDB cluster. +category: reference +--- + +# Daily Check + +As a distributed database, TiDB is more complicated than the stand-alone database in terms of the mechanism, and monitoring items. To help operate and maintain TiDB in a more convenient way, this document introduces some key performance indicators. + +## Key indicators of Dashboard + +Starting from v4.0, TiDB provides a new operation and maintenance management tool, TiDB Dashboard. This tool is integrated into the PD component. You can access TiDB Dashboard at the default address `http://${pd-ip}:${pd_port}/dashboard`. + +TiDB Dashboard simplifies the operation and maintenance of the TiDB database. You can view the running status of the entire TiDB cluster through one interface. The following are descriptions of some performance indicators. + +### Instance panel + +![Instance panel](/media/instance-status-panel.png) + ++ **Status**: This indicator is used to check whether the status is normal. For an online node, this can be ignored. ++ **Up Time**: The key indicator. If you find that the `Up Time` is changed, you need to locate the reason why the component is restarted. ++ **Version**, **Deployment Directory**, **Git Hash**: These indicators need to be checked to avoid inconsistent or even incorrect version/deployment directory. + +### Host panel + +![Host panel](/media/host-panel.png) + +You can view the usage of CPU, memory, and disk. When the usage of any resource exceeds 80%, it is recommended to scale out the capacity accordingly. + +### SQL analysis panel + +![SQL analysis panel](/media/sql-analysis-panel.png) + +You can locate the slow SQL statement executed in the cluster. Then you can optimize the specific SQL statement. + +### Region panel + +![Region panel](/media/region-panel.png) + ++ `miss-peer-region-count`: The number of Regions without enough replicas. This value is not always greater than `0`. ++ `extra-peer-region-count`: The number of Regions with extra replicas. These Regions are generated during the scheduling process. ++ `empty-region-count`: The number of empty Regions, generated by executing the `TRUNCATE TABLE`/`DROP TABLE` statement. If this number is large, you can consider enabling `Region Merge` to merge Regions across tables. ++ `pending-peer-region-count`: The number of Regions with outdated Raft logs. It is normal that a few pending peers are generated in the scheduling process. However, it is not normal if this value is large for a period of time. ++ `down-peer-region-count`: The number of Regions with an unresponsive peer reported by the Raft leader. ++ `offline-peer-region-count`: The number of Regions during the offline process. + +Generally, it is normal that these values are not `0`. However, it is not normal that they are not `0` for quite a long time. + +### KV Request Duration + +![TiKV request duration](/media/kv-duration-panel.png) + +The KV request duration 99 in TiKV. If you find nodes with a long duration, check whether there are hot spots, or whether there are nodes with poor performance. + +### PD TSO Wait Duration + +![TiDB TSO Wait Duration](/media/pd-duration-panel.png) + +The time it takes for TiDB to obtain TSO from PD. The following are reasons for the long wait duration: + ++ High network latency from TiDB to PD. You can manually execute the ping command to test the network latency. ++ High load for the TiDB server. ++ High load for the PD server. + +### Overview panel + +![Overview panel](/media/overview-panel.png) + +You can view the load, memory available, network traffic, and I/O utilities. When a bottleneck is found, it is recommended to scale out the capacity, or to optimize the cluster topology, SQL, cluster parameters, etc. + +### Exceptions + +![Exceptions](/media/failed-query-panel.png) + +You can view the errors triggered by the execution of SQL statements on each TiDB instance. These include syntax error, primary key conflicts, etc. + +### GC status + +![GC status](/media/garbage-collation-panel.png) + +You can check whether the GC (Garbage Collection) status is normal by viewing the time when the last GC happens. If the GC is abnormal, it might lead to excessive historical data, thereby decreasing the access efficiency. diff --git a/dashboard/dashboard-access.md b/dashboard/dashboard-access.md new file mode 100644 index 0000000000000..cf1c904226103 --- /dev/null +++ b/dashboard/dashboard-access.md @@ -0,0 +1,59 @@ +--- +title: Access TiDB Dashboard +summary: Learn how to access TiDB Dashboard. +category: how-to +--- + +# Access TiDB Dashboard + +To access TiDB Dashboard, visit via your browser. Replace `127.0.0.1:2379` with the actual PD instance address and port. + +If multiple PD instances are deployed in your cluster and you can directly access **every** PD instance and port, you can simply replace `127.0.0.1:2379` in the address with **any** PD instance address and port. + +> **Note:** +> +> If a firewall or reverse proxy is configured and you cannot directly access every PD instance, you might not be able to access TiDB Dashboard. Usually, this is because the firewall or reverse proxy is not correctly configured. See [Use TiDB Dashboard behind Reverse Proxy](/dashboard/dashboard-ops-reverse-proxy.md) and [Secure TiDB Dashboard](/dashboard/dashboard-ops-security.md) to learn correctly configure the firewall or reverse proxy when multiple PD instances are deployed. + +## Browser compatibility + +You can use TiDB Dashboard in the following common desktop browsers of a relatively newer version: + +- Chrome >= 77 +- Firefox >= 68 +- Edge >= 17 + +> **Note:** +> +> If you use the browsers above of earlier versions or other browsers to access TiDB Dashboard, some functions might not work properly. + +## Sign in + +For the first-time access, TiDB Dashboard displays the user sign in interface, as shown in the image below. You can sign in using the TiDB `root` account. + +![Login interface](/media/dashboard/dashboard-access-login.png) + +If one of the following situations exists, the login might fail: + +- TiDB `root` user does not exist. +- PD is not started or cannot be accessed. +- TiDB is not started or cannot be accessed. +- Wrong `root` password. + +Once you have signed in, the session remains valid within the next 24 hours. To learn how to sign out, refer to the [Logout](#logout) section. + +## Switch language + +The following languages are supported in TiDB Dashboard: + +- English +- Chinese (simplified) + +In the **SQL User Sign In** page, you can click the **Switch Language** drop-down list to switch the interface language. + +![Switch language](/media/dashboard/dashboard-access-switch-language.png) + +## Logout + +Once you have logged in, click the login user name in the left navigation bar to switch to the user page. Click the **Logout** button on the user page to log out the current user. After logging out, you need to re-enter your username and password. + +![Logout](/media/dashboard/dashboard-access-logout.png) diff --git a/dashboard/dashboard-cluster-info.md b/dashboard/dashboard-cluster-info.md new file mode 100644 index 0000000000000..8b8b06d9cae22 --- /dev/null +++ b/dashboard/dashboard-cluster-info.md @@ -0,0 +1,70 @@ +--- +title: TiDB Dashboard Cluster Information Page +summary: View the running status of TiDB, TiKV, PD, TiFlash components in the entire cluster and the running status of the host on which these components are located. +category: how-to +--- + +# TiDB Dashboard Cluster Information Page + +On the cluster information page, you can view the running status of TiDB, TiKV, PD, TiFlash components in the entire cluster and the running status of the host on which these components are located. + +## Access the page + +You can use one of the following two methods to access the cluster information page: + +- After logging into TiDB Dashboard, click **Cluster Info** on the left navigation menu: + + ![Access cluster information page](/media/dashboard/dashboard-cluster-info-access.png) + +- Visit in your browser. Replace `127.0.0.1:2379` with the actual PD instance address and port. + +## Instance list + +Click **Instances** to view the list of instances: + +![Instance list](/media/dashboard/dashboard-cluster-info-instances.png) + +This instance list shows the overview information of all instances of TiDB, TiKV, PD, and TiFlash components in the cluster. + +The list includes the following information: + +- Address: The instance address. +- Status: The running status of the instance. +- Up Time: The start time of the instance. +- Version: The instance version number. +- Deployment directory: The directory in which the instance binary file is located. +- Git Hash: The Git Hash value corresponding to the instance binary file. + +An instance has the following running status: + +- Up: The instance is running properly. +- Down or Unreachable: The instance is not started or a network problem exists on the corresponding host. +- Tombstone: The data on the instance has been completely migrated out and the scaling-in is complete. This status exists only on TiKV or TiFlash instances. +- Leaving: The data on the instance is being migrated out and the scaling-in is in process. This status exists only on TiKV or TiFlash instances. +- Unknown: The running state of the instance is unknown. + +> **Note:** +> +> Some columns in the table can be displayed only when the instance is up. + +## Host list + +Click **Hosts** to view the list of hosts: + +![Host list](/media/dashboard/dashboard-cluster-info-hosts.png) + +This host list shows the running status of hosts that correspond to all instances of TiDB, TiKV, PD, and TiFlash components in the cluster. + +The list includes the following information: + +- Address: The Host IP address. +- CPU: The number of logical cores of the host CPU. +- CPU Usage: The user-mode and kernel-mode CPU usage in the current 1 second. +- Memory: The total physical memory size of the host. +- Memory Usage: The current memory usage of the host. +- Disk: The file system of the disk on the host on which the instance is running and the mounting path of this disk. +- Disk Usage: The space usage of the disk on the host on which the instance is running. + +> **Note:** +> +> The host list information is provided by each instance process, so when all instances on the host are down, the host information is not displayed. diff --git a/dashboard/dashboard-diagnostics-access.md b/dashboard/dashboard-diagnostics-access.md new file mode 100644 index 0000000000000..4fc616ed0d5f8 --- /dev/null +++ b/dashboard/dashboard-diagnostics-access.md @@ -0,0 +1,62 @@ +--- +title: TiDB Dashboard Cluster Diagnostic Page +summary: Learn how to use the cluster diagnostic page. +category: how-to +--- + +# TiDB Dashboard Cluster Diagnostics Page + +The cluster diagnostics feature in TiDB Dashboard diagnoses the problems that might exist in a cluster within a specified time range, and summarizes the diagnostic results and the cluster-related load monitoring information into a diagnostic report. This diagnostic report is in the form of a web page. You can browse the page offline and circulate this page link after saving the page from a browser. + +> **Note:** +> +> The cluster diagnostics feature depends on Prometheus deployed in the cluster. For details about how to deploy this monitoring component, see [TiUP](/tiup/tiup-overview.md) or [TiDB Ansible](/online-deployment-using-ansible.md) deployment document. If no monitoring component is deployed in the cluster, the generated diagnostic report will indicate a failure. + +## Access the page + +You can use one of the following methods to access the cluster diagnostics page: + +* After logging into TiDB Dashboard, click **Cluster Diagnostics** on the left navigation menu: + + ![Access Cluster Diagnostics page](/media/dashboard/dashboard-diagnostics-access.png) + +* Visit `http://127.0.0.1:2379/dashboard/#/diagnose` in your browser. Replace `127.0.0.1:2379` with the actual PD address and port number. + +## Generate diagnostic report + +To diagnose a cluster within a specified time range and check the cluster load, you can take the following steps to generate a diagnostic report: + +1. Set the **Range Start Time**, such as `2020-05-21 14:40:00`. +2. Set the **Range Duration**, such as `10 min`. +3. Click **Start**. + +![Generate diagnostic report](/media/dashboard/dashboard-diagnostics-gen-report.png) + +> **Note:** +> +> It is recommended that the **Range Duration** of the report is between 1 minute and 60 minutes. This **Range Duration** cannot exceed 60 minutes. + +The steps above generate a diagnostic report for the time range from `2020-05-21 14:40:00` to `2020-05-21 14:50:00`. After clicking **Start**, you can see the interface below. **Progress** is the progress bar of the diagnostic report. After the report is generated, click **View Full Report**. + +![Report progress](/media/dashboard/dashboard-diagnostics-gen-process.png) + +## Generate comparison report + +If a system exception occurs at a certain point, for example, QPS jitter or higher latency, a diagnostic report can be generated. Particularly, this report compares the system in the abnormal time range with the system in the normal time range. For example: + +* Abnormal time range: `2020-05-21 14:40:00`-`2020-05-21 14:45:00`. Within this time range, the system is abnormal. +* Normal time range: `2020-05-21 14:30:00` - `2020-05-21 14:35:00`. Within this time range, the system is normal. + +You can take the following steps to generate a comparison report for the two time ranges above: + +1. Set the **Range Start Time**, which is the start time of the range in which the system becomes abnormal, such as `2020-05-21 14:40:00`. +2. Set the **Range Duration**. Generally, this duration is the duration of system anomalies, such as 5 minutes. +3. Enable **Compare by baseline**. +4. Set the **Baseline Range Start Time**, which is the start time of the range (to be compared with) in which the system is normal, such as `2020-05-21 14:30:00`. +5. Click **Start**. + +![Generate comparison report](/media/dashboard/dashboard-diagnostics-gen-compare-report.png) + +Then wait for the report to be generated and click **View Full Report**. + +In addition, the historical diagnostic report is displayed in the list on the main page of the diagnostic report. You can click to view these historical reports directly. diff --git a/dashboard/dashboard-diagnostics-report.md b/dashboard/dashboard-diagnostics-report.md new file mode 100644 index 0000000000000..5997ac2510896 --- /dev/null +++ b/dashboard/dashboard-diagnostics-report.md @@ -0,0 +1,369 @@ +--- +title: TiDB Dashboard Diagnostic Report +summary: Learn the TiDB Dashboard diagnostic report. +category: how-to +--- + +# TiDB Dashboard Diagnostic Report + +This document introduces the content of the diagnostic report and viewing tips. To access the cluster diagnostic page and generate reports, see [TiDB Dashboard Cluster Diagnostics Page](/dashboard/dashboard-diagnostics-access.md). + +## View report + +The diagnostic report consists of the following parts: + +* Basic information: Includes the time range of the diagnostic report, hardware information of the cluster, the version information of cluster topology. +* Diagnostic information: Shows the results of automatic diagnostics. +* Load information: Includes CPU, memory and other load information of the server, TiDB, PD, or TiKV. +* Overview information: Includes the consumed time and error information of each TiDB, PD, or TiKV module. +* TiDB/PD/TiKV monitoring information: Includes monitoring information of each component. +* Configuration information: Includes configuration information of each component. + +An example of the diagnostic report is as follows: + +![Sample report](/media/dashboard/dashboard-diagnostics-example-table.png) + +In the image above, **Total Time Consume** in the top blue box is the report name. The information in the red box below explains the content of this report and the meaning of each field in the report. + +In this report, some small buttons are described as follows: + +* **i** icon: You can move your mouse to the **i** icon to see the explanatory note of the row. +* **expand**: Click **expand** to see details about this monitoring metric. For example, the detailed information of `tidb_get_token` in the image above includes the monitoring information of each TiDB instance's latency. +* **collapse**: Contrary to **expand**, the button is used to fold detailed monitoring information. + +All monitoring metrics basically correspond to those on the TiDB Grafna monitoring dashboard. After a module is found to be abnormal, you can view more monitoring information on the TiDB Grafna. + +In addition, the `TOTAL_TIME` and `TOTAL_COUNT` metrics in this report are monitoring data read from Prometheus, so calculation inaccuracy might exits in their statistics. + +Each part of this report is introduced as follows. + +### Basic information + +#### Diagnostics Time Range + +The time range for generating the diagnostics report includes the start time and end time. + +![Report time range](/media/dashboard/dashboard-diagnostics-report-time-range.png) + +#### Cluster Hardware Info + +Cluster Hardware Info includes information such as CPU, memory, and disk of each server in the cluster. + +![Cluster hardware report](/media/dashboard/dashboard-diagnostics-cluster-hardware.png) + +The fields in the table above are described as follows: + +* `HOST`: The IP address of the server. +* `INSTANCE`: The number of instances deployed on the server. For example, `pd * 1` means that this server has 1 PD instance deployed; `tidb * 2 pd * 1` means that this server has 2 TiDB instances and 1 PD instance deployed. +* `CPU_CORES`: Indicates the number of CPU cores (physical cores or logical cores) of the server. +* `MEMORY`: Indicates the memory size of the server. The unit is GB. +* `DISK`: Indicates the server disk size. The unit is GB. +* `UPTIME`: The uptime of the server. The unit is day. + +#### Cluster Topology Info + +The `Cluster Info` table shows the cluster topology information. The information in this table are from TiDB [information_schema.cluster_info](/system-tables/system-table-cluster-info.md) system table. + +![Cluster info](/media/dashboard/dashboard-diagnostics-cluster-info.png) + +The fields in the table above are described as follows: + +* `TYPE`: The node type. +* `INSTANCE`: The instance address(es), which is a string in the `IP:PORT` format. +* `STATUS_ADDRESS`: The HTTP API service address. +* `VERSION`: The semantic version number of the corresponding node. +* `GIT_HASH`: Git Commit Hash when compiling the node version, which is used to identify whether the two nodes are absolutely the consistent version. +* `START_TIME`: The start time of the corresponding node. +* `UPTIME`: The uptime of the corresponding node. + +### Diagnostic information + +TiDB has built-in automatic diagnostic results. For the description of each field, see [information_schema.inspection-result](/system-tables/system-table-inspection-result.md) system table. + +### Load Info + +#### Node Load Info + +The `Node Load Info` table shows the load information of the server node, including the average value (AVG), maximum value (MAX), minimum value (MIN) of the following metrics of the server within the time range: + +* CPU usage (the maximum value is `100%`) +* Memory usage +* Disk I/O usage +* Disk write latency +* Disk read latency +* Disk read bytes per second +* Disk write bytes per second +* The number of bytes received by the node network per minute +* The number of bytes sent from the node network per minute +* The number of TCP connections in use by the node +* The number of all TCP connections of the node + +![Server Load Info report](/media/dashboard/dashboard-diagnostics-node-load-info.png) + +#### Instance CPU Usage + +The `Instance CPU Usage` table shows the average value (AVG), maximum value (MAX), and minimum value (MIN) of the CPU usage of each TiDB/PD/TiKV process. The maximum CPU usage of the process is `100% * the number of CPU logical cores`. + +![Instance CPU Usage report](/media/dashboard/dashboard-diagnostics-process-cpu-usage.png) + +#### Instance Memory Usage + +The `Instance Memory Usage` table shows the average value (AVG), maximum value (MAX), and minimum value (MIN) of memory bytes occupied by each TiDB/PD/TiKV process. + +![Instance memory usage report](/media/dashboard/dashboard-diagnostics-process-memory-usage.png) + +#### TiKV Thread CPU Usage + +The `TiKV Thread CPU Usage` table shows the average value (AVG), maximum value (MAX) and minimum value (MIN) of CPU usage of each module thread in TiKV. The maximum CPU usage of the process is `100% * the thread count of the corresponding configuration`. + +![TiKV Thread CPU Usage report](/media/dashboard/dashboard-diagnostics-thread-cpu-usage.png) + +In the table above, + +* `CONFIG_KEY`: The relevant thread configuration of the corresponding module. +* `CURRENT_CONFIG_VALUE`: The current value of the configuration when the report is generated. + +> **Note:** +> +> `CURRENT_CONFIG_VALUE` is the value when the report is generated, not the value within the time range of this report. Currently, some configuration values ​​of historical time cannot be obtained. + +#### `TiDB/PD Goroutines Count` + +The `TiDB/PD Goroutines Count` table shows the average value (AVG), maximum value (MAX), and minimum value (MIN) of the number of TiDB or PD goroutines. If the number of goroutines exceeds 2,000, the concurrency of the process is too high, which affects the overall request latency. + +![TiDB/PD goroutines count report](/media/dashboard/dashboard-diagnostics-goroutines-count.png) + +### Overview information + +#### Time Consumed by Each Component + +The `Time Consumed by Each Component` table shows the monitored consumed time and the time ratio of TiDB, PD, TiKV modules in the cluster. The default time unit is seconds. You can use this table to quickly locate which modules consume more time. + +![Time Consume report](/media/dashboard/dashboard-diagnostics-total-time-consume.png) + +The fields in columns of the table above are described as follows: + +* `METRIC_NAME`: The name of the monitoring metric. +* `Label`: The label information for the monitoring metric. Click **expand** to view more detailed monitoring information of each label for a metric. +* `TIME_RATIO`: The ratio of the total time consumed by this monitoring metric to the total time of the monitoring row where `TIME_RATIO` is `1`. For example, the total consumed time by `kv_request` is `1.65` (namely, `38325.58`/`23223.86`) times that of `tidb_query`. Because KV requests are executed concurrently, the total time of all KV requests might exceed the total query (`tidb_query`) execution time. +* `TOTAL_TIME`: The total time consumed by this monitoring metric. +* `TOTAL_COUNT`: The total number of times this monitoring metric is executed. +* `P999`: The maximum P999 time of this monitoring metric. +* `P99`: The maximum P99 time of this monitoring metric. +* `P90`: The maximum P90 time of this monitoring metric. +* `P80`: The maximum P80 time of this monitoring metric. + +The following image shows the relationship of time consumption of the related modules in the monitoring metrics above. + +![Time-consumption relationship of each module](/media/dashboard/dashboard-diagnostics-time-relation.png) + +In the image above, yellow boxes are the TiDB-related monitoring metrics.Blue boxes are TiKV-related monitoring metrics, and gray boxes temporarily do not correspond to specific monitoring metrics. + +In the image above, the time consumption of `tidb_query` includes the following four parts: + +* `get_token` +* `parse` +* `compile` +* `execute` + +The `execute` time includes the following parts: + +* `wait_start_tso` +* The execution time at the TiDB layer, which is currently not monitored +* KV request time +* `KV_backoff` time, which is the time for backoff after the KV request fails + +Among the parts above, the KV request time includes the following parts: + +* The time consumed by the network sending and receiving of requests. Currently, there is no monitoring metric for this item. You can subtract the time of `tikv_grpc_message` from the KV request time to roughly estimate this item. +* `tikv_grpc_message` time consumption. + +Among the parts above, `tikv_grpc_message` time consumption includes the following parts: + +* Coprocessor request time consumption, which refers to processing the COP type requests. This time consumption includes the following parts: + * `tikv_cop_wait`: The time consumed by request queue. + * `Coprocessor handle`: The time consumed to process Coprocessor requests. + +* `tikv_scheduler_command` time consumption, which includes the following parts: + * `tikv_scheduler_processing_read`: The time consumed to process read requests. + * The time consumed to get snapshot in `tikv_storage_async_request` (snapshot is the label for this monitoring metric). + * Time consumed to process write requests. This time consumption includes the following parts: + * `tikv_scheduler_latch_wait`: The time consumed to wait for latch. + * The time consumption of writes in `tikv_storage_async_request` (write is the label for this monitoring metric). + +Among the above metrics, The time consumption of writes in `tikv_storage_async_request` refers to the time consumption of writing Raft KVs, including the following parts: + +* `tikv_raft_propose_wait` +* `tikv_raft_process`, which mainly includes `tikv_raft_append_log` +* `tikv_raft_commit_log` +* `tikv_raft_apply_wait` +* `tikv_raft_apply_log` + +You can use `TOTAL_TIME`, the P999 time, and the P99 time to determine which modules consume longer time according to the relationship between the time consumptions described above, and then look at the related monitoring metrics. + +> **Note:** +> +> Because the Raft KVs writes might be processed in one batch, using `TOTAL_TIME` to measure the time consumed by each module is inapplicable to monitoring metrics related to Raft KV writes, specifically, `tikv_raft_process`, `tikv_raft_append_log`, `tikv_raft_commit_log`, `tikv_raft_apply_wait`, and `tikv_raft_apply_log`. In this situation, it is more reasonable to compare the time consumption of each module with the time of P999 and P99. +> +> The reason is that if there are 10 asynchronous write requests, Raft KVs internally pack 10 requests into a batch execution, and the execution time is 1 second. Therefore, the execution time of each request is 1 second, and the total time of 10 requests is 10 seconds, but the total time for Raft KV processing is 1 second. If you use `TOTAL_TIME` to measure the consumed time, you might not understand where the remaining 9 seconds are spent. You can also see the difference between the monitoring metric of Raft KV and other previous monitoring metrics from the total number of requests (`TOTAL_COUNT`). + +#### Errors Occurred in Each Component + +The `Errors Occurred in Each Component` table shows the total number of errors in TiDB and TiKV, such as the failure to write binlog, `tikv server is busy`, `TiKV channel full`, `tikv write stall`. You can see the row comments for the specific meaning of each error. + +![Errors Occurred in Each Component report](/media/dashboard/dashboard-diagnostics-error.png) + +#### Specific TiDB/PD/TiKV monitoring information + +This part includes more specific monitoring information of TiDB, PD, or TiKV. + +#### TiDB-related monitoring information + +##### Time Consumed by TiDB Component + +This table shows the time consumed by each TiDB module and the ratio of each time consumption, which is similar to the `time consume` table in the overview, but the label information of this table are more detailed. + +##### TiDB Server Connections + +This table shows the number of client connections for each TiDB instance. + +##### TiDB Transaction + +This table shows transaction-related monitoring metrics. + +![Transaction report](/media/dashboard/dashboard-diagnostics-tidb-txn.png) + +* `TOTAL_VALUE`: The sum of all values ​​(SUM) during the report time range. +* `TOTAL_COUNT`: The total number of occurrences of this monitoring metric. +* `P999`: The maximum P999 value of this monitoring metric. +* `P99`: The maximum P99 value of this monitoring metric. +* `P90`: The maximum P90 value of this monitoring metric. +* `P80`: The maximum P80 value of this monitoring metric. + +Example: + +In the table above, within the report time range, `tidb_txn_kv_write_size`: a total of about 181,296 transactions of KV writes, and the total KV write size is 266.772 MB, of which the maximum P999, P99, P90, P80 values for a single transaction of KV writes ​​are 116.913 KB, 1.996 KB, 1.905 KB, and 1.805 KB. + +##### DDL Owner + +![TiDB DDL Owner Report](/media/dashboard/dashboard-diagnostics-tidb-ddl.png) + +The table above shows that from `2020-05-21 14:40:00`, the cluster's `DDL OWNER` is at the `10.0.1.13:10080` node. If the owner changes, multiple rows of data exist in the table above, where the `Min_Time` column indicates the minimum time of the corresponding known owner. + +> **Note:** +> +> If the owner information is empty, it does not mean that no owner exists in this period of time. Because in this situation, the DDL owner is determined based on the monitoring information of `ddl_worker`, it might be that `ddl_worker` has not done any DDL job in this period of time, causing the owner information to be empty. + +Other monitoring tables in TiDB are as follows: + +* Statistics Info: Shows related monitoring metrics of TiDB statistical information. +* Top 10 Slow Query: Shows the Top 10 slow query information in the report time range. +* Top 10 Slow Query Group By Digest: Shows the Top 10 slow query information in the report time range, which is aggregated according to the SQL fingerprint. +* Slow Query With Diff Plan: The SQL statement whose execution plan changes within the report time range. + +#### PD related monitoring information + +The tables related to the monitoring information of PD modules are as follows: + +* `Time Consumed by PD Component`: The time consumed by the monitoring metrics of related modules in PD. +* `Blance Leader/Region`: The monitoring information of `balance-region` and `balance leader` occurred in the cluster within the report time range, such as the number of leaders that are scheduled out from `tikv_note_1` or the number of leaders that are scheduled in. +* `Cluster Status`: The cluster status information, including total number of TiKV nodes, total cluster storage capacity, the number of Regions, and the number of offline TiKV nodes. +* `Store Status`: Record the status information of each TiKV node, including Region score, leader score, and the number of Regions/leaders. +* `Etcd Status`: etcd related information in PD. + +#### TiKV related monitoring information + +The tables related to the monitoring information of TiKV modules are as follows: + +* `Time Consumed by TiKV Component`: The time consumed by related modules in TiKV. +* `Time Consumed by RocksDB`: The time consumed by RocksDB in TiKV. +* `TiKV Error`: The error information related to each module in TiKV. +* `TiKV Engine Size`: The size of stored data of column families on each node in TiKV. +* `Coprocessor Info`: Monitoring information related to the Coprocessor module in TiKV. +* `Raft Info`: Monitoring information of the Raft module in TiKV. +* `Snapshot Info`: Snapshot related monitoring information in TiKV. +* `GC Info`: Garbage Collection (GC) related monitoring information in TiKV. +* `Cache Hit`: The hit rate information of each cache of RocksDB in TiKV. + +### Configuration information + +In the configuration information, the configuration values of some modules are shown within the report time range. But the historical values of some other configurations of these modules cannot be obtained, so the shown values of these configurations are the current (when the report is generated) values . + +Within the report time range, the following tables include items whose values are configured at the start time of the report time range: + +* `Scheduler Initial Config`: The initial value of PD scheduling-related configuration at the report's start time. +* `TiDB GC Initial Config`: The initial value of TiDB GC related-configuration at the report's start time +* `TiKV RocksDB Initial Config`: The initial value of TiKV RocksDB-related configuration at the report's start time +* `TiKV RaftStore Initial Config`: The initial value of TiKV RaftStore-related configuration at the report's start time + +Within the report time range, if some configurations have been modified, the following tables include records of some configurations that have been modified: + +* `Scheduler Config Change History` +* `TiDB GC Config Change History` +* `TiKV RocksDB Config Change History` +* `TiKV RaftStore Config Change History` + +Example: + +![Scheduler Config Change History report](/media/dashboard/dashboard-diagnostics-config-change.png) + +The table above shows that the `leader-schedule-limit` configuration parameter has been modified within the report time range: + +* `2020-05-22T20:00:00+08:00`: At the start time of the report, the configuration value of `leader-schedule-limit` is `4`, which does not mean that the configuration has been modified, but that at the start time in the report time range, its configuration value is `4`. +* `2020-05-22T20:07:00+08:00`: The `leader-schedule-limit` configuration value is `8`, which indicates that the value of this configuration has been modified around `2020-05-22T20:07:00+08:00`. + +The following tables show the current configuration of TiDB, PD, and TiKV at the time when the report is generated: + +* `TiDB's Current Config` +* `PD's Current Config` +* `TiKV's Current Config` + +## Comparison report + +You can generate a comparison report for two time ranges. The report content is the same as the report for a single time range, except that a comparison column is added to show the difference between the two time ranges. The following sections introduce some unique tables in the comparison report and how to view the comparison report. + +First, the `Compare Report Time Range` report in the basic information shows the two time ranges for comparison: + +![Compare Report Time Range report](/media/dashboard/dashboard-diagnostics-compare-time.png) + +In the table above, `t1` is the normal time range, or the reference time range. `t2` is the abnormal time range. + +Tables related to slow queries are shown as follows: + +* `Slow Queries In Time Range t2`: Shows slow queries that only appear in `t2` but not during `t1`. +* `Top 10 slow query in time range t1`: The Top 10 slow queries during `t1`. +* `Top 10 slow query in time range t2`: The Top 10 slow queries during `t2`. + +### DIFF_RATIO introduction + +This section introduces `DIFF_RATIO` using the `Instance CPU Usage` table as an example. + +![Compare Instance CPU Usage report](/media/dashboard/dashboard-diagnostics-compare-instance-cpu-usage.png) + +* `t1.AVG`, `t1.MAX`, `t1.Min` are the average value, maximum value, and minimum value of CPU usage in the `t1`. +* `t2.AVG`, `t2.MAX`, and `t2.Min` are the average value, maximum value, and minimum value ​​of CPU usage during `t2`. +* `AVG_DIFF_RATIO` is `DIFF_RATIO` of the average values during `t1` and `t2`. +* `MAX_DIFF_RATIO` is `DIFF_RATIO` of the maximum values during `t1` and `t2`. +* `MIN_DIFF_RATIO` is `DIFF_RATIO` of the minimum values during `t1` and `t2`. + +`DIFF_RATIO`: Indicates the difference value between the two time ranges. It has the following values: + +* If the monitoring metric has a value only within `t2` and has no value within `t1`, the value of `DIFF_RATIO` is `1`. +* If the monitoring metric has a value only within `t1`, and has no value within `t2` time range, the value of `DIFF_RATIO` is `-1`. +* If the value of `t2` is greater than that of `t1`, then `DIFF_RATIO` = `(t2.value / t1.value)-1` +* If the value of `t2` is smaller than that of `t1`, then `DIFF_RATIO` = `1-(t1.value / t2.value)` + +For example, in the table above, the average CPU usage of the `tidb` node in `t2` is 2.02 times higher than that in `t1`, which is `2.02` = `1240/410-1`. + +### Maximum Different Item table + +The `Maximum Different Item` table compares the monitoring metrics of two time ranges, and sorts them according to the difference of the monitoring metrics. Using this table, you can quickly find out which monitoring metric has the biggest difference in the two time ranges. See the following example: + +![Maximum Different Item table](/media/dashboard/dashboard-diagnostics-maximum-different-item.png) + +* `Table`: Indicates this monitoring metric comes from which table in the comparison report. For example, `TiKV, coprocessor_info` indicates the `coprocessor_info` table in the TiKV component. +* `METRIC_NAME`: The monitoring metric name. Click `expand` to view the comparison of different labels of metrics. +* `LABEL`: The label corresponding to the monitoring metric. For example, the monitoring metric of `TiKV Coprocessor scan` has 2 labels, namely `instance`, `req`, `tag`, `sql_type`, which are the TiKV address, request type, operation type and operation column family. +* `MAX_DIFF`: Difference value, which is the `DIFF_RATIO` calculation of `t1.VALUE` and `t2.VALUE`. + +From the table above, you can see the `t2` time range has much more Coprocessor requests than the `t1` time range, and the SQL parsing time of TiDB in `t2` is much longer. diff --git a/dashboard/dashboard-diagnostics-usage.md b/dashboard/dashboard-diagnostics-usage.md new file mode 100644 index 0000000000000..e8ebbe835e3f8 --- /dev/null +++ b/dashboard/dashboard-diagnostics-usage.md @@ -0,0 +1,114 @@ +--- +title: Locate Problems Using Diagnostic Report of TiDB Dashboard +summary: Learn how to locate problems using diagnostic report of TiDB Dashboard. +category: how-to +--- + +# Locate Problems Using Diagnostic Report of TiDB Dashboard + +This document introduces how to locate problems using diagnostic report of TiDB Dashboard. + +## Comparison diagnostics + +This section demonstrates how to use the comparison diagnostic feature to diagnose QPS jitter or latency increase caused by large queries or writes. + +### Example 1 + +![QPS example](/media/dashboard/dashboard-diagnostics-usage1.png) + +The result of a `go-ycsb` pressure test is shown in the image above. You can see that at `2020-03-10 13:24:30`, QPS suddenly began to decrease. After 3 minutes, QPS began to return to normal. You can use diagnostic report of TiDB Dashboard to find out the cause. + +Generate a report that compares the system in the following two time ranges: + +T1: `2020-03-10 13:21:00` to `2020-03-10 13:23:00`. In this range, the system is normal, which is called a reference range. + +T2: `2020-03-10 13:24:30` to `2020-03-10 13:27:30`. In this range, QPS began to decrease. + +Because the impact range of jitter is 3 minutes, the two time ranges above are both 3 minutes. Because some monitored average values are used for comparison during diagnostics, too long a range will cause the difference of average values to be insignificant, and then the problem cannot be accurately located. + +After the report is generated, you can view this report on the **Compare Diagnose** page. + +![Comparison diagnostics](/media/dashboard/dashboard-diagnostics-usage2.png) + +The diagnostic results above show that big queries might exist during the diagnostic time. Each **DETAIL** in the report above is described as follows: + +* `tidb_qps`: QPS decreased by 0.93 times. +* `tidb_query_duration`: P999 query latency increased by 1.54 times. +* `tidb_cop_duration`: The processing latency of P999 coprocessor requests increased by 2.48 times. +* `tidb_kv_write_num`: The number written KVs in the P999 TiDB transaction increased by 7.61 times. +* `tikv_cop_scan_keys_total_nun`: The number of keys/values scanned by the TiKV Coprocessor has greatly improved on 3 TiKV instances. +* In `pd_operator_step_finish_total_count`, the number of transferred leaders increases by 2.45 times, which means that the scheduling in the abnormal time range is higher than that in the normal time range. +* The report indicates that there might be slow queries and indicates that you can use SQL statements to query slow queries. The execution result of the SQL statements are as follows: + +```sql +SELECT * FROM (SELECT count(*), min(time), sum(query_time) AS sum_query_time, sum(Process_time) AS sum_process_time, sum(Wait_time) AS sum_wait_time, sum(Commit_time), sum(Request_count), sum(process_keys), sum(Write_keys), max(Cop_proc_max), min(query),min(prev_stmt), digest FROM information_schema.CLUSTER_SLOW_QUERY WHERE time >= '2020-03-10 13:24:30' AND time < '2020-03-10 13:27:30' AND Is_internal = false GROUP BY digest) AS t1 WHERE t1.digest NOT IN (SELECT digest FROM information_schema.CLUSTER_SLOW_QUERY WHERE time >= '2020-03-10 13:21:00' AND time < '2020-03-10 13:24:00' GROUP BY digest) ORDER BY t1.sum_query_time DESC limit 10\G +***************************[ 1. row ]*************************** +count(*) | 196 +min(time) | 2020-03-10 13:24:30.204326 +sum_query_time | 46.878509117 +sum_process_time | 265.924 +sum_wait_time | 8.308 +sum(Commit_time) | 0.926820886 +sum(Request_count) | 6035 +sum(process_keys) | 201453000 +sum(Write_keys) | 274500 +max(Cop_proc_max) | 0.263 +min(query) | delete from test.tcs2 limit 5000; +min(prev_stmt) | +digest | 24bd6d8a9b238086c9b8c3d240ad4ef32f79ce94cf5a468c0b8fe1eb5f8d03df +``` + +From the result above, you can see that from `13:24:30`, there is a large write of batch deletion, which has been executed 196 times in total, each time deleting 5,000 rows of data, in a total duration of 46.8 seconds. + +### Example 2 + +If a large query has not been executed, the query is not recorded in the slow log. In this situation, this large query can still be diagnosed. See the following example: + +![QPS results](/media/dashboard/dashboard-diagnostics-usage3.png) + +The result of another `go-ycsb` pressure test is shown in the image above. You can see that at `2020-03-08 01:46:30`, QPS suddenly began to drop and did not recover. + +Generate a report that compares the system in the following two time ranges: + +T1: `2020-03-08 01:36:00` to `2020-03-08 01:41:00`. In this range, the system is normal, which is called a reference range. +T2: `2020-03-08 01:46:30` to `2020-03-08 01:51:30`. In this range, QPS began to decrease. + +After the report is generated, you can view this report on the **Compare Diagnose** page. + +![Comparison diagnostics](/media/dashboard/dashboard-diagnostics-usage4.png) + +The diagnostic result is similar to that of example 1. The last row of the image above indicates that there might be slow queries and indicate that you can use SQL statements to query the expensive queries in the TiDB log. The execution result of the SQL statements are as follows. + +```sql +> SELECT * FROM information_schema.cluster_log WHERE type='tidb' AND time >= '2020-03-08 01:46:30' AND time < '2020-03-08 01:51:30' AND level = 'warn' AND message LIKE '%expensive_query%'\G +TIME | 2020/03/08 01:47:35.846 +TYPE | tidb +INSTANCE | 172.16.5.40:4009 +LEVEL | WARN +MESSAGE | [expensivequery.go:167] [expensive_query] [cost_time=60.085949605s] [process_time=2.52s] [wait_time=2.52s] [request_count=9] [total_keys=996009] [process_keys=996000] [num_cop_tasks=9] [process_avg_time=0.28s] [process_p90_time=0.344s] [process_max_time=0.344s] [process_max_addr=172.16.5.40:20150] [wait_avg_time=0.000777777s] [wait_p90_time=0.003s] [wait_max_time=0.003s] [wait_max_addr=172.16.5.40:20150] [stats=t_wide:pseudo] [conn_id=19717] [user=root] [database=test] [table_ids="[80,80]"] [txn_start_ts=415132076148785201] [mem_max="23583169 Bytes (22.490662574768066 MB)"] [sql="select count(*) from t_wide as t1 join t_wide as t2 where t1.c0>t2.c1 and t1.c2>0"] +``` + +The query result above shows that on this `172.16.5.40:4009` TiDB instance, at `2020/03/08 01:47:35.846` there is an expensive query that has been executed for 60 seconds, but the execution has not yet been finished. This query is a join of Cartesian products. + +## Locate problems using comparison diagnostic report + +Because the diagnostics might be wrong, using a comparison report might help DBAs locate problems more quickly. See the following example. + +![QPS results](/media/dashboard/dashboard-diagnostics-usage5.png) + +The result of a `go-ycsb` pressure test is shown in the image above. You can see that at `2020-05-22 22:14:00`, QPS suddenly began to decrease. After 3 minutes, QPS began to return to normal. You can use the comparison diagnostic report of TiDB Dashboard to find out the cause. + +Generate a report that compares the system in the following two time ranges: + +T1: `2020-05-22 22:11:00` to `2020-05-22 22:14:00`. In this range, the system is normal, which is called a reference range. +T2: `2020-05-22 22:14:00` `2020-05-22 22:17:00`. In this range, QPS began to decrease. + +After generating the comparison report, check the **Max diff item** report. This report compares the monitoring items of the two time ranges above and sorts them according to the difference of the monitoring items. The result of this table is as follows: + +![Comparison results](/media/dashboard/dashboard-diagnostics-usage6.png) + +From the result above, you can see that the Coprocessor requests in T2 are much more than those in T1. It might be that some large queries appear in T2 that bring more load. + +In fact, during the entire time range from T1 to T2, the `go-ycsb` pressure test is running. Then 20 `tpch` queries are running during T2. So it is the `tpch` queries that cause many Coprocessor requests. + +If such a large query whose execution time exceeds the threshold of slow log is recorded in the slow log. You can check the `Slow Queries In Time Range t2` report to see whether there is any slow query. However, some queries in T1 might become slow queries in T2, because in T2, other loads cause their executions to slow down. diff --git a/dashboard/dashboard-faq.md b/dashboard/dashboard-faq.md new file mode 100644 index 0000000000000..6fc7f3c76362b --- /dev/null +++ b/dashboard/dashboard-faq.md @@ -0,0 +1,60 @@ +--- +title: TiDB Dashboard FAQ +summary: Learn about the frequently asked questions (FAQs) and answers about TiDB Dashboard. +category: how-to +--- + +# TiDB Dashboard FAQ + +This document summarizes the frequently asked questions (FAQs) and answers about TiDB Dashboard. + +## Access-related FAQ + +### When the firewall or reverse proxy is configured, I am redirected to an internal address other than TiDB Dashboard + +When multiple Placement Driver (PD) instances are deployed in a cluster, only one of the PD instances actually runs the TiDB Dashboard service. If you access other PD instances instead of this one, your browser redirects you to another address. If the firewall or reverse proxy is not properly configured for accessing TiDB Dashboard, when you visit the Dashboard, you might be redirected to an internal address that is protected by the firewall or reverse proxy. + +- See [TiDB Dashboard Multi-PD Instance Deployment](/dashboard/dashboard-ops-deploy.md#) to learn the working principle of TiDB Dashboard with multiple PD instances. +- See [Use TiDB Dashboard through a Reverse Proxy](/dashboard/dashboard-ops-reverse-proxy.md) to learn how to correctly configure a reverse proxy. +- See [Secure TiDB Dashboard](/dashboard/dashboard-ops-security.md) to learn how to correctly configure the firewall. + +### When TiDB Dashboard is deployed with dual network interface cards (NICs), TiDB Dashboard cannot be accessed using another NIC + +For security reasons, TiDB Dashboard on PD only monitors the IP addresses specified during deployment (that is, it only listens on one NIC), not on `0.0.0.0`. Therefore, when multiple NICs are installed on the host, you cannot access TiDB Dashboard using another NIC. + +If you have deployed TiDB using the `tiup cluster` or `tiup playground` command, currently this problem cannot be solved. It is recommended that you use a reverse proxy to safely expose TiDB Dashboard to another NIC. For details, see [Use TiDB Dashboard behind a Reverse Proxy](/dashboard/dashboard-ops-reverse-proxy.md). + +## UI-related FAQ + +### A `prometheus_not_found` error is shown in **QPS** and **Latency** sections on the Overview page + +The **QPS** and **Latency** sections on the **Overview** page require a cluster with Prometheus deployed. Otherwise, the error is shown. You can solve this problem by deploying a Prometheus instance in the cluster. + +If you still encounter this problem when the Prometheus instance has been deployed, the possible reason is that your deployment tool is out of date (TiUP, TiDB Operator, or TiDB Ansible), and your tool does not automatically report metrics addresses, which makes TiDB Dashboard unable to query metrics. You can upgrade you deployment tool to the latest version and try again. + +If your deployment tool is TiUP, take the following steps to solve this problem. For other deployment tools, refer to the corresponding documents of those tools. + +1. Upgrade TiUP and TiUP Cluster: + + {{< copyable "shell-regular" >}} + + ```bash + tiup update --self + tiup update cluster --force + ``` + +2. After the upgrade, when a new cluster is deployed with Prometheus instances, the metrics can be displayed normally. + +3. For an existing cluster, you can restart this cluster to report the metrics addresses. Replace `CLUSTER_NAME` with the actual cluster name: + + {{< copyable "shell-regular" >}} + + ```bash + tiup cluster start CLUSTER_NAME + ``` + + Even if the cluster has been started, still execute this command. This command does not affect the normal application in the cluster, but refreshes and reports the metrics addresses, so that the monitoring metrics can be displayed normally in TiDB Dashboard. + +### An `invalid connection` error is shown in **Top SQL Statements** and **Recent Slow Queries** on the Overview page + +The possible reason is that you have enabled the `prepared-plan-cache` feature of TiDB. As an experimental feature, when enabled, `prepared-plan-cache` might not function properly in specific TiDB versions, which could cause this problem in TiDB Dashboard (and other applications). You can disable `prepared-plan-cache` by updating [TiDB Configuration file](/tidb-configuration-file.md#prepared-plan-cache) to solve this problem. diff --git a/dashboard/dashboard-intro.md b/dashboard/dashboard-intro.md new file mode 100644 index 0000000000000..0b2eb3ba5a532 --- /dev/null +++ b/dashboard/dashboard-intro.md @@ -0,0 +1,67 @@ +--- +title: TiDB Dashboard Introduction +summary: Introduce TiDB Dashboard. +category: how-to +--- + +# TiDB Dashboard Introduction + +> **Warning:** +> +> TiDB Dashboard is still an experimental feature. It is **NOT** recommended that you use it in the production environment. + +TiDB Dashboard is a Web UI for monitoring, diagnosing, and managing the TiDB cluster, which is available since v4.0. It is built into the PD component and does not require an independent deployment. + +![TiDB Dashboard interface](/media/dashboard/dashboard-intro.gif) + +TiDB Dashboard is open-sourced on [GitHub](https://github.com/pingcap-incubator/tidb-dashboard). + +This document introduces the main features of TiDB Dashboard. You can click links in the following sections to learn more details. + +## Show the overall running status of the TiDB cluster + +You can use TiDB Dashboard to learn the TiDB cluster's queries per second (QPS), execution time, the types of SQL statements that consume the most resources, and other overview information. + +See [TiDB Dashboard Overview](/dashboard/dashboard-overview.md) for details. + +## Show the running status of components and hosts + +You can use TiDB Dashboard to view the running status of TiDB, TiKV, PD, TiFlash components in the entire cluster and the running status of the host on which these components are located. + +See [TiDB Dashboard Cluster Information Page](/dashboard/dashboard-cluster-info.md) for details. + +## Show distribution and trends of read and write traffic + +The Key Visualizer feature of TiDB Dashboard visually shows the change of read and write traffic over time in the entire cluster in the form of heatmap. You can use this feature to timely discover changes of application modes or locate hotspot issues with uneven performance. + +See [Key Visualizer Page](/dashboard/dashboard-key-visualizer.md) for details. + +## Show a list of execution information of all SQL statements + +The execution information of all SQL statements is listed on the SQL Statements page. You can use this page to learn the execution time and total executions at all stages, which helps you analyze and locate the SQL queries that consume the most resources and improve the overall cluster performance. + +See [SQL Statements Page of TiDB Dashboard](/dashboard/dashboard-statement-list.md) for details. + +## Learn the detailed execution information of slow queries + +The Slow Queries page of TiDB Dashboard shows a list of all SQL statements that take a long time to execute, including the SQL texts and execution information. This page helps you locate the cause of slow queries or performance jitter. + +See [Slow Queries Page](/dashboard/dashboard-slow-query.md) for details. + +## Diagnose common cluster problems and generate reports + +The diagnostic feature of TiDB Dashboard automatically determines whether some common risks (such as inconsistent configurations) or problems exist in the cluster, generates reports, and gives operation suggestions, or compares the status of each cluster metric in different time ranges for you to analyze possible problems. + +See [TiDB Dashboard Cluster Diagnostics Page](/dashboard/dashboard-diagnostics-access.md) for details. + +## Query logs of all components + +On the Search Logs page of TiDB Dashboard, you can quickly search logs of all running instances in the cluster by keywords, time range, and other conditions, package these logs, and download them to your local machine. + +See [Search Logs Page](/dashboard/dashboard-log-search.md) for details. + +## Collect profiling data for each instance + +This is an advanced debugging feature that lets you profile each instance online and analyze various internal operations an instance performed during the profiling data collection period and the proportion of the operation execution time in this period without third-party tools. + +See [Profile Instances Page](/dashboard/dashboard-profiling.md) for details. diff --git a/dashboard/dashboard-key-visualizer.md b/dashboard/dashboard-key-visualizer.md new file mode 100644 index 0000000000000..8f683860ae205 --- /dev/null +++ b/dashboard/dashboard-key-visualizer.md @@ -0,0 +1,182 @@ +--- +title: Key Visualizer Page +summary: Learn how to use Key Visualizer to monitor traffic. +category: how-to +aliases: ['/docs/dev/key-visualizer-monitoring-tool/'] +--- + +# Key Visualizer Page + +The Key Visualizer page of TiDB Dashboard is used to analyze the usage of TiDB and troubleshoot traffic hotspots. This page visually shows the traffic of the TiDB cluster over a period of time. + +## Access Key Visualizer page + +You can use one of the following two methods to access the Key Visualizer page: + +* After logging into TiDB Dashboard, click **Key Visualizer** on the left navigation menu: + +![Access Key Visualizer](/media/dashboard/dashboard-keyviz-access.png) + +* Visit in your browser. Replace `127.0.0.1:2379` with the actual PD instance address and port. + +## Interface demonstration + +The following image is a demonstration of the Key Visualizer page: + +![Key Visualizer page](/media/dashboard/dashboard-keyviz-overview.png) + +From the interface above, you can see the following objects: + ++ A large heatmap that shows changes of the overall traffic over time. ++ The detailed information of a certain coordinate point. ++ Information of tables, indexes, and so on (on the left side of the heatmap). + +## Basic concepts + +This section introduces the basic concepts that relate to Key Visualizer. + +### Region + +In a TiDB cluster, the stored data is distributed among TiKV instances. Logically, TiKV is a huge and orderly key-value map. The whole key-value space is divided into many segments and each segment consists of a series of adjacent keys. Such segment is called a `Region`. + +For detailed introduction of Region, refer to [TiDB Internal (I) - Data Storage](https://pingcap.com/blog/2017-07-11-tidbinternal1/#region). + +### Hotspot + +When you use the TiDB database, the hotspot issue is typical, where high traffic is concentrated on a small range of data. Because consecutive data ranges are often processed on the same TiKV instance, the TiKV instance on which the hotspot occurs becomes the performance bottleneck of the whole application. The hotspot issue often occurs in the following scenarios: + ++ Write adjacent data into a table with the `AUTO_INCREMENT` primary key, which causes a hotspot issue on this table. ++ Write adjacent time data into the time index of a table, which causes a hotspot issue on the table index. + +For more details about hotspot, refer to [Highly Concurrent Write Best Practices](/best-practices/high-concurrency-best-practices.md#hotspot-causes) + +### Heatmap + +The heatmap is the core part of Key Visualizer, which shows the change of a metric over time. The X-axis of the heatmap indicates the time. The Y-axis of the heatmap indicates the consecutive Regions based on key ranges that cover all schemas and tables of the TiDB cluster. + +Colder colors in the heatmap indicate lower read and write traffic of Regions in that period of time. Hotter (brighter) colors indicate higher traffic. + +### Region compression + +A TiDB cluster might have up to hundreds of thousands of Regions. It is difficult to display so many Regions on screen. Therefore, on each heatmap, these Regions are compressed into 1,500 consecutive ranges, each range called a Bucket. In the heatmap, because hotter instances need more attention, Key Visualizer tends to compress a large number of Regions with lower traffic into one Bucket, and displays the Region with higher traffic also in one Bucket. + +## Use Key Visualizer + +This section introduces how to use Key Visualizer. + +### Settings + +To use the Key Visualizer page for the first time, you need to manually enable this feature on the **Settings** page. Follow the page guide and click **Open Settings** to open the settings page: + +![Feature disabled](/media/dashboard/dashboard-keyviz-not-enabled.png) + +After this feature is enabled, you can open the settings page by clicking the **Settings** icon in the upper right corner: + +![Settings icon](/media/dashboard/dashboard-keyviz-settings-button.png) + +The settings page is shown as follows: + +![Settings page](/media/dashboard/dashboard-keyviz-settings.png) + +Set whether to start data collection through the switch, and click **Save** to take effect. After enabling the feature, you can see that the toolbar is available: + +![Toolbar](/media/dashboard/dashboard-keyviz-toolbar.png) + +After this feature is enabled, data collection is going on at the backend. You can see the heatmap shortly. + +### Observe a certain period of time or Region range + +When you open Key Visualizer, the heatmap of the entire database over the recent six hours is displayed by default. In this heatmap, the closer to the right side (current time), the shorter the time interval corresponding to each column of Buckets. If you want to observe a specific time period or a specific Region range, you can zoom in to get more details. The specific instructions are as follows: + +1. Scroll up or down in the heatmap. +2. Click and drag one of the following buttons to select the range. + + Click the **Select & Zoom** button. Then click and drag this button to select the area to zoom in. + + ![Selection box](/media/dashboard/dashboard-keyviz-select-zoom.gif) + + + Click the **Reset** button to reset the Region range to the entire database. + + Click the **time selection box** (at the position of `6 hours` on the interface above) and select the observation time period again. + + ![Select time](/media/dashboard/dashboard-keyviz-select-time.png) + +> **Note:** +> +> If you follow step 2 in the instruction above, the heatmap is redrawn, which might be very different from the original heatmap. This difference is normal because if you observe in more detail, the granularity of the Region compression has changed, or the basis of `hot` in a specific range has changed. + +### Adjust brightness + +The heatmap uses colors of different brightnesses to indicate the Bucket traffic. Colder colors in the heatmap indicate lower read and write traffic of the Region in that period of time. Hotter (brighter) colors indicate higher traffic. If the color is too cold or too hot, it is difficult to observe in details. In this situation, you can click the **Brightness** button and then use the slider to adjust the brightness of the page. + +> **Note:** +> +> When Key Visualizer displays the heatmap of an area, it defines the basis of being cold and hot according to the traffic of this area. When the traffic in the entire area is relatively even, even if the overall traffic is low in value, you might still see a large bright-colored area. Remember to include the value into your analysis. + +### Select metrics + +![Select metrics](/media/dashboard/dashboard-keyviz-select-type.png) + +You can view a metric you are interested in by selecting this metric in the **metrics selection box** (at the `Write (bytes)` position in the interface above): + +* `Read (bytes)`: Read traffic. +* `Write (bytes)`: Write traffic. +* `Read (keys)`: The number of read rows. +* `Write (keys)`: The number of written rows. +* `All`: The sum of read and write traffic. + +### Refresh and automatic refresh + +To regain a heatmap based on the current time, click the **Refresh** button. If you need to observe the traffic distribution of the database in real time, click the down arrow on the right side of the **Refresh** button and select a fixed time interval for the heatmap to automatically refresh at this interval. + +> **Note:** +> +> If you adjust the time range or Region range, the automatic refresh is disabled. + +### See Bucket details + +You can hover your mouse over the Bucket you are interested in to view the detailed information of this Region range. The image below is an example of this information: + +![Bucket details](/media/dashboard/dashboard-keyviz-tooltip.png) + +If you want to copy this information, click a Bucket. Then, the page with relevant details is temporarily pinned. Click on the information, and you have copied it to the clipboard. + +![Copy Bucket details](/media/dashboard/dashboard-keyviz-tooltip-copy.png) + +## Common heatmap types + +This section shows and interprets four common types of heatmap in Key Visualizer. + +### Evenly distributed workload + +![Balanced](/media/dashboard/dashboard-keyviz-well-dist.png) + +In the heatmap above, bright and dark colors are a fine-grained mix. This indicates that reads or writes are evenly distributed over time and among key ranges. The workload is evenly distributed to all nodes, which is ideal for a distributed database. + +### Periodically reads and writes + +![Periodically](/media/dashboard/dashboard-keyviz-period.png) + +In the heatmap above, there is an alternating brightness and darkness along the X-axis (time) but the brightness is relatively even along the Y-axis (Region). This indicates that the reads and writes change periodically, which might occur in scenarios of periodically scheduled tasks. For example, the big data platform periodically extracts data from TiDB every day. In this kind of scenarios, pay attention to whether the resources are sufficient during peak usage. + +## Concentrated reads or writes + +![Concentrated](/media/dashboard/dashboard-keyviz-continue.png) + +In the heatmap above, you can see several bright lines. Along the Y-axis, the fringes around the bright lines are dark, which indicates that the Regions corresponding to bright lines have high read and write traffic. You can observe whether the traffic distribution is expected by your application. For example, when all services are associated with the user table, the overall traffic of the user table can be high, so it is reasonable to show bright lines in the heatmap. + +In addition, the height of the bright lines (the thickness along the Y-axis) is important. Because TiKV has its own Region-based hotspot balancing mechanism, the more Regions involved in the hotspot, the better it is for balancing traffic across all TiKV instances. The thicker and more bright lines indicate that hotspots are more scattered, and TiKV is better used. The thinner and fewer bright lines indicate that hotspots are more concentrated, and the hotspot issue is more obvious in TiKV, which might requires manual intervention. + +### Sequential reads or writes + +![Sequential](/media/dashboard/dashboard-keyviz-sequential.png) + +In the heatmap above, you can see a bright line. This means that the data reads or writes are sequential. Typical scenarios of sequential data reads or writes are importing data or scanning tables and indexes. For example, you continuously write data to tables with auto-increment IDs. + +Regions in the bright areas are the hotspots of read and write traffic, which often become the performance bottleneck of the entire cluster. In this situation, you might need to readjust the primary key for the application. By doing this, you scatter Regions much as possible to spread the pressure across multiple Regions. You can also schedule application tasks during the low-peak period. + +> **Note:** +> +> In this section, only the common types of heatmap are shown. Key Visualizer actually displays the heatmap of all schemas and tables in the entire cluster, so you might see different types of heatmap in different areas, or mixed results of multiple heatmap types. Use the heatmap based on the actual situation. + +## Address hotspot issues + +TiDB has some built-in features to mitigate the common hotspot issue. Refer to [Highly Concurrent Write Best Practices](/best-practices/high-concurrency-best-practices.md) for details. diff --git a/dashboard/dashboard-log-search.md b/dashboard/dashboard-log-search.md new file mode 100644 index 0000000000000..2391f8781c906 --- /dev/null +++ b/dashboard/dashboard-log-search.md @@ -0,0 +1,72 @@ +--- +title: TiDB Dashboard Log Search Page +summary: Learn how to search logs of all nodes using the log search page of TiDB Dashboard. +category: how-to +--- + +# TiDB Dashboard Log Search Page + +On the log search page of TiDB Dashboard, you can search logs of all nodes, preview the search result and download logs. + +## Access the page + +After logging into TiDB Dashboard, you can click **Search Logs** to enter this log search homepage. + +![Log Search Page](/media/dashboard/dashboard-log-search-home.png) + +This page provides the following search parameters: + +- Time range: Specifies the time range of logs to search. The default value is the recent 30 minutes. +- Log level: Specifies the minimum log level. All logs above this log level are searched. The default value is the `INFO`. +- Keywords: The parameter is optional and its value can be any legal string. Multiple keywords are separated by a space. Regular expressions are supported (case-insensitive). +- Components: Selects the cluster components to search, which are multi-select and non-empty. By default, all components are selected. + +After clicking the **Search** button, you enter the detail page of the search results. + +## Page of search result + +The following image shows the page of the search results. + +![Search result](/media/dashboard/dashboard-log-search-result.png) + +This page consists of the following three areas: + +- Parameter options (area 1 in the image above): These options are the same as the parameter options on the search homepage. You can re-select the parameters in the boxes and start a new search. +- Progress (area 2 in the image above): The current search progress is shown on the right side of this page, including the log search status and statistics of each node. +- Search results (area 3 in the image above): + - Time: The time at which the log is generated. The time zone is the same as the time zone of the front-end user. + - Level: log level. + - Component: Shows the component name and its address. + - Log: The body part of each log record, excluding the log time and log level. Logs that are too long are automatically truncated. Click a row to expand the full content. The full log can show up to 512 characters. + +> **Note:** +> +> At most 500 search results can be previewed on this page. You can get the complete search results by downloading them. + +### Search progress + +In the search progress area, a search on a node is called a search task. A search task might have the following statuses: + +- Running: After starting the search, all tasks enter the **Running** status. +- Success: After the task is completed, it automatically enters the **Success** status. At this time, the logs have been cached in the local disk where the Dashboard backend is located, and can be provided to the frontend to download. +- Failed: When you cancel the search task, or the task exits with an error, the task enters the **Failed** status. When the task fails, the local temporary files are automatically cleaned. + +The search progress area has the following three control buttons: + +- **Download Selected**: Click this button to download logs of the selected components (only the completed ones can be selected), and you will get a tar file. Unzip this tar file to get one or more zip files (each component corresponds to a zip file). Unzip the zip file(s) to get the log text file. +- **Cancel**: Click this button to cancel all running tasks. You can click this button only when there are running tasks. +- **Retry**: Click this button to retry all failed tasks. You can click this button only when there are failed tasks and no running tasks. + +## Search history list + +Click the **View search history** link on the log search homepage to enter page of search history list: + +![Search history entry](/media/dashboard/dashboard-log-search-history-entry.png) + +![Search history list](/media/dashboard/dashboard-log-search-history.png) + +The history list shows the time range, log level, components, keywords, and search status of each search log. Click the **Detail** link in the **Action** column to see the search result details: + +You can delete the search history that you no longer need. Click **Delete All** in the upper right corner, or select the rows to be deleted and then click **Delete selected** to delete the history: + +![Delete search history](/media/dashboard/dashboard-log-search-delete-history.png) diff --git a/dashboard/dashboard-ops-deploy.md b/dashboard/dashboard-ops-deploy.md new file mode 100644 index 0000000000000..b34eac0e91fb9 --- /dev/null +++ b/dashboard/dashboard-ops-deploy.md @@ -0,0 +1,139 @@ +--- +title: Deploy TiDB Dashboard +summary: Learn how to deploy TiDB Dashboard. +category: how-to +--- + +# Deploy TiDB Dashboard + +The TiDB Dashboard UI is built into the PD component for v4.0 or higher versions, and no additional deployment is required. Simply deploy a standard TiDB cluster, and TiDB Dashboard will be there. + +See the following documents to learn how to deploy a standard TiDB cluster: + ++ [Quick Start Guide for the TiDB Database Platform](/quick-start-with-tidb.md#deploy-a-local-test-environment-using-tiup-playground) ++ [Deploy TiDB in Production Environment](/production-deployment-using-tiup.md) ++ [Kubernetes environment deployment](https://pingcap.com/docs/tidb-in-kubernetes/stable/access-dashboard/) + +> **Note:** +> +> You cannot deploy TiDB Dashboard in a TiDB cluster earlier than v4.0. + +## Deployment with multiple PD instances + +When multiple PD instances are deployed in the cluster, only one of these instances serves the TiDB Dashboard. + +When PD instances are running for the first time, they automatically negotiate with each other to choose one instance to serve the TiDB Dashboard. TiDB Dashboard will not run on other PD instances. The TiDB Dashboard service will always be provided by the chosen PD instance no matter PD instances are restarted or new PD instances are joined. However, there will be a re-negotiation when the PD instance that serves TiDB Dashboard is removed from the cluster (scaled-in). The negotiation process does not need user intervention. + +When you access a PD instance that does not serve TiDB Dashboard, the browser will be redirected automatically to guide you to access the PD instance that serves the TiDB Dashboard, so that you can access the service normally. This process is illustrated in the image below. + +![Process Schematic](/media/dashboard/dashboard-ops-multiple-pd.png) + +> **Note:** +> +> The PD instance that serves TiDB Dashboard might not be a PD leader. + +### Check the PD instance that actually serves TiDB Dashboard + +For a running cluster deployed using TiUP, you can use the `tiup cluster display` command to see which PD instance serves TiDB Dashboard. Replace `CLUSTER_NAME` with the cluster name. + +{{< copyable "shell-regular" >}} + +```bash +tiup cluster display CLUSTER_NAME --dashboard +``` + +A sample output is as follows: + +```bash +http://192.168.0.123:2379/dashboard/ +``` + +> **Note:** +> +> This feature is available only in the later version of the `tiup cluster` deployment tool (v1.0.3 or later). You can upgrade `tiup cluster` with the following commands: +> +> {{< copyable "shell-regular" >}} +> +> ```bash +> tiup update --self +> tiup update cluster --force +> ``` + +### Switch to another PD instance to serve TiDB Dashboard + +For a running cluster deployed using TiUP, you can use the `tiup ctl pd` command to change the PD instance that serves TiDB Dashboard, or re-specify a PD instance to serve TiDB Dashboard when it is disabled: + +{{< copyable "shell-regular" >}} + +```bash +tiup ctl pd -u http://127.0.0.1:2379 config set dashboard-address http://9.9.9.9:2379 +``` + +In the command above: + +- Replace `127.0.0.1:2379` with the IP and port of any PD instance. +- Replace `9.9.9.9:2379` with the IP and port of the new PD instance that you desire to run the TiDB Dashboard service. + +You can use the `tiup cluster display` command to see whether the modification is taking effect (replace `CLUSTER_NAME` with the cluster name): + +{{< copyable "shell-regular" >}} + +```bash +tiup cluster display CLUSTER_NAME --dashboard +``` + +> **Warning:** +> +> If you change the instance to run TiDB Dashboard, the local data stored in the previous TiDB Dashboard instance will be lost, including the Key Visualize history and search history. + +## Disable TiDB Dashboard + +For a running cluster deployed using TiUP, use the `tiup ctl pd` command to disable TiDB Dashboard on all PD instances (replace `127.0.0.1:2379` with the IP and port of any PD instance): + +{{< copyable "shell-regular" >}} + +```bash +tiup ctl pd -u http://127.0.0.1:2379 config set dashboard-address none +``` + +After disabling TiDB Dashboard, checking which PD instance provides the TiDB Dashboard service will fail: + +``` +Error: TiDB Dashboard is disabled +``` + +Visiting the TiDB Dashboard address of any PD instance via the browser will also fail: + +``` +Dashboard is not started. +``` + +## Re-enable TiDB Dashboard + +For a running cluster deployed using TiUP, use the `tiup ctl pd` command to request PD to renegotiate an instance to run TiDB Dashboard (replace `127.0.0.1:2379` with the IP and port of any PD instance): + +{{< copyable "shell-regular" >}} + +```bash +tiup ctl pd -u http://127.0.0.1:2379 config set dashboard-address auto +``` + +After executing the command above, you can use the `tiup cluster display` command to view the TiDB Dashboard instance address automatically negotiated by PD (replace `CLUSTER_NAME` with the cluster name): + +{{< copyable "shell-regular" >}} + +```bash +tiup cluster display CLUSTER_NAME --dashboard +``` + +You can also re-enable TiDB Dashboard by manually specifying the PD instance that serves TiDB Dashboard. See [Switch to another PD instance to serve TiDB Dashboard](#switch-to-another-pd-instance-to-serve-tidb-dashboard). + +> **Warning:** +> +> If the newly enabled TiDB Dashboard instance is different with the previous instance that served the TiDB Dashboard, the local data stored in the previous TiDB Dashboard instance will be lost, including Key Visualize history and search history. + +## What's next + +- To learn how to access and log into the TiDB Dashboard UI, see [Access TiDB Dashboard](/dashboard/dashboard-access.md). + +- To learn how to enhance the security of TiDB Dashboard, such as configuring a firewall, see [Secure TiDB Dashboard](/dashboard/dashboard-ops-security.md). diff --git a/dashboard/dashboard-ops-reverse-proxy.md b/dashboard/dashboard-ops-reverse-proxy.md new file mode 100644 index 0000000000000..769d15c5a0231 --- /dev/null +++ b/dashboard/dashboard-ops-reverse-proxy.md @@ -0,0 +1,200 @@ +--- +title: Use TiDB Dashboard behind a Reverse Proxy +category: how-to +--- + +# Use TiDB Dashboard behind a Reverse Proxy + +You can use a reverse proxy to safely expose the TiDB Dashboard service from the internal network to the external. + +## Use NGINX reverse proxy + +### Step 1: Get the actual TiDB Dashboard address + +When multiple PD instances are deployed in the cluster, only one of the PD instances actually runs TiDB Dashboard. Therefore, you need to ensure that the upstream of the reverse proxy points to the correct address. For details of this mechanism, see [Deployment with multiple PD instances](/dashboard/dashboard-ops-deploy.md#deployment-with-multiple-pd-instances). + +When you use the TiUP tool for deployment, execute the following command to get the actual TiDB Dashboard address (replace `CLUSTER_NAME` with your cluster name): + +{{< copyable "shell-regular" >}} + +```bash +tiup cluster display CLUSTER_NAME --dashboard +``` + +The output is the actual TiDB Dashboard address. A sample is as follows: + +```bash +http://192.168.0.123:2379/dashboard/ +``` + +> **Note:** +> +> This feature is available only in the later version of the `tiup cluster` deployment tool (v1.0.3 or later). You can upgrade `tiup cluster` with the following commands: +> +> {{< copyable "shell-regular" >}} +> +> ```bash +> tiup update --self +> tiup update cluster --force +> ``` + +### Step 2: Configure the NGINX reverse proxy + +When you use [NGINX](https://nginx.org/) as the reverse proxy, take the following steps: + +1. Use reverse proxy for TiDB Dashboard on the `80` port (for example). In the NGINX configuration file, add the following configuration: + + {{< copyable "" >}} + + ```nginx + server { + listen 80; + location /dashboard/ { + proxy_pass http://192.168.0.123:2379/dashboard/; + } + } + ``` + + Configure the URL filled in `proxy_pass` to be the actual address of the TiDB Dashboard obtained in [Step 1](#step-1-get-the-actual-tidb-dashboard-address). + + > **Warning:** + > + > You must keep the `/dashboard/` path in the `proxy_pass` directive to ensure that only the services under this path are reverse proxied. Otherwise, security risks will be introduced. See [Secure TiDB Dashboard](/dashboard/dashboard-ops-security.md). + +2. Restart NGINX for the configuration to take effect. + +3. Test whether the reverse proxy is effective: access the `/dashboard/` address on the `80` port of the machine where NGINX is located (such as ) to access TiDB Dashboard. + +## Customize path prefix + +TiDB Dashboard provides services by default in the `/dashboard/` path, such as `http://example.com/dashboard/`, which is the case even for reverse proxies. To configure the reverse proxy to provide the TiDB Dashboard service with a non-default path, such as `http://example.com/foo` or `http://example.com/`, take the following steps. + +### Step 1: Modify PD configuration to specify the path prefix of TiDB Dashboard service + +Modify the `public-path-prefix` configuration item in the `[dashboard]` category of the PD configuration to specify the path prefix of the TiDB Dashboard service. After this item is modified, restart the PD instance for the modification to take effect. + +For example, if the cluster is deployed using TiUP and you want the service to run on `http://example.com/foo`, you can specify the following configuration: + +{{< copyable "" >}} + +```yaml +server_configs: + pd: + dashboard.public-path-prefix: /foo +``` + +If you are deploying a new cluster, you can add the configuration above to the `topology.yaml` TiUP topology file and deploy the cluster. For specific instruction, see [TiUP deployment document](/production-deployment-using-tiup.md#step-3-edit-the-initialization-configuration-file). + +For a deployed cluster: + +1. Open the configuration file of the cluster in the edit mode (replace `CLUSTER_NAME` with the cluster name). + + {{< copyable "shell-regular" >}} + + ```bash + tiup cluster edit-config CLUSTER_NAME + ``` + +2. Modify or add configuration items under the `pd` configuration of `server_configs`. If no `server_configs` exists, add it at the top level: + + {{< copyable "" >}} + + ```yaml + monitored: + ... + server_configs: + tidb: ... + tikv: ... + pd: + dashboard.public-path-prefix: /foo + ... + ``` + + The configuration file after the modification is similar to the following file: + + {{< copyable "" >}} + + ```yaml + monitored: + ... + server_configs: + tidb: ... + tikv: ... + pd: + dashboard.public-path-prefix: /foo + ... + ``` + + Or + + {{< copyable "" >}} + + ```yaml + monitored: + ... + server_configs: + tidb: ... + tikv: ... + pd: + dashboard.public-path-prefix: /foo + ``` + +3. Perform a rolling restart to all PD instances for the modified configuration to take effect (replace `CLUSTER_NAME` with your cluster name): + + {{< copyable "shell-regular" >}} + + ```bash + tiup cluster reload CLUSTER_NAME -R pd + ``` + +See [Common TiUP Operations - Modify the configuration](/maintain-tidb-using-tiup.md#modify-the-configuration) for details. + +If you want that the TiDB Dashboard service is run in the root path (such as `http://example.com/`), use the following configuration: + +{{< copyable "" >}} + +```yaml +server_configs: + pd: + dashboard.public-path-prefix: / +``` + +> **Warning:** +> +> After the modified and customized path prefix takes effect, you cannot directly access TiDB Dashboard. You can only access TiDB Dashboard through a reverse proxy that matches the path prefix. + +### Step 2: Modify the NGINX reverse proxy configuration + +Taking `http://example.com/foo` as an example, the corresponding NGINX configuration is as follows: + +{{< copyable "" >}} + +```nginx +server { + listen 80; + location /foo/ { + proxy_pass http://192.168.0.123:2379/dashboard/; + } +} +``` + +> **Warning:** +> +> Keep the `/dashboard/` path in the `proxy_pass` directive to ensure that only the services under this path are reverse proxied. Otherwise, security risks will be introduced. See [Secure TiDB Dashboard](/dashboard/dashboard-ops-security.md). + +Change `http://192.168.0.123:2379/dashboard/` in the configuration to the actual address of the TiDB Dashboard obtained in [Step 1: Get the actual TiDB Dashboard address](#step-1-get-the-actual-tidb-dashboard-address). + +If you want that the TiDB Dashboard service is run in the root path (such as `http://example.com/`), use the following configuration: + +{{< copyable "" >}} + +```nginx +server { + listen 80; + location / { + proxy_pass http://192.168.0.123:2379/dashboard/; + } +} +``` + +Modify the configuration and restart NGINX for the modified configuration to take effect. diff --git a/dashboard/dashboard-ops-security.md b/dashboard/dashboard-ops-security.md new file mode 100644 index 0000000000000..a2d0cfbc8af25 --- /dev/null +++ b/dashboard/dashboard-ops-security.md @@ -0,0 +1,79 @@ +--- +title: Secure TiDB Dashboard +summary: Learn how to improve the security of TiDB Dashboard. +category: how-to +--- + +# Secure TiDB Dashboard + +Although you need to sign into TiDB Dashboard before accessing it, TiDB Dashboard is designed to be accessed by trusted user entities by default. When you want to provide TiDB Dashboard to external network users or untrusted users for access, take the following measures to avoid security vulnerabilities. + +## Set a strong password for TiDB `root` user + +The account system of TiDB Dashboard is consistent with that of TiDB SQL users. By default, TiDB's `root` user has no password, so accessing TiDB Dashboard does not require password authentication, which will give the malicious visitor high privileges, including executing privileged SQL statements. + +It is recommended that you set a strong password for TiDB `root` user. See [TiDB User Account Management](/user-account-management.md) for details. + +## Use a firewall to block untrusted access + +TiDB Dashboard provides services through the PD client port, which defaults to . Although TiDB Dashboard requires identity authentication, other privileged interfaces (such as ) in PD carried on the PD client port do not require identity authentication and can perform privileged operations. Therefore, exposing the PD client port directly to the external network is extremely risky. + +It is recommended that you take the following measures: + ++ Use a firewall to prohibit a component from accessing **any** client port of the PD component via the external network or untrusted network. + + > **Note:** + > + > TiDB, TiKV, and other components need to communicate with the PD component through the PD client port, so do not block access to the internal network between components. Otherwise, the cluster will become unavailable. + ++ See [Use TiDB Dashboard behind a Reverse Proxy](/dashboard/dashboard-ops-reverse-proxy.md) to learn how to configure the reverse proxy to safely provide the TiDB Dashboard service on another port to the external network. + +### How to open access to TiDB Dashboard port when deploying multiple PD instances + +> **Warning:** +> +> This section describes an unsafe access solution, which is for the test environment only. **DO NOT** use this solution in the production environment. + +In the test environment, you might need to configure the firewall to open the TiDB Dashboard port for external access. + +When multiple PD instances are deployed, only one of the PD instances actually runs TiDB Dashboard, and browser redirection occurs when you access other PD instances. Therefore, you need to ensure that the firewall is configured with the correct IP address. For details of this mechanism, see [Deployment with multiple PD instances](/dashboard/dashboard-ops-deploy.md#deployment-with-multiple-pd-instances). + +When using the TiUP deployment tool, you can view the address of the PD instance that actually runs TiDB Dashboard by running the following command (replace `CLUSTER_NAME` with the cluster name): + +{{< copyable "shell-regular" >}} + +```bash +tiup cluster display CLUSTER_NAME --dashboard +``` + +The output is the actual TiDB Dashboard address. + +> **Note:** +> +> This feature is available only in the later version of the `tiup cluster` deployment tool (v1.0.3 or later). You can upgrade `tiup cluster` with the following commands: +> +> {{< copyable "shell-regular" >}} +> +> ```bash +> tiup update --self +> tiup update cluster --force +> ``` + +The following is a sample output: + +```bash +http://192.168.0.123:2379/dashboard/ +``` + +In this example, the firewall needs to be configured with inbound access for the `2379` port of the `192.168.0.123` open IP, and the TiDB Dashboard is accessed via . + +## Reverse proxy only for TiDB Dashboard + +As mentioned in [Use a firewall to block untrusted access](#use-a-firewall-to-block-untrusted access), the services provided under the PD client port include not only TiDB Dashboard (located at ), but also other privileged interfaces in PD (such as ). Therefore, when using a reverse proxy to provide TiDB Dashboard to the external network, ensure that the services **ONLY** with the `/dashboard` prefix are provided (**NOT** all services under the port) to avoid that the external network can access the privileged interface in PD through the reverse proxy. + +It is recommended that you see [Use TiDB Dashboard behind a Reverse Proxy](/dashboard/dashboard-ops-reverse-proxy.md) to learn a safe and recommended reverse proxy configuration. + +## Other recommended safety measures + +- [Enable TLS Authentication and Encrypt the Stored Data](/enable-tls-between-components.md) +- [Enable TLS Between TiDB Clients and Servers](/enable-tls-between-clients-and-servers.md) diff --git a/dashboard/dashboard-overview.md b/dashboard/dashboard-overview.md new file mode 100644 index 0000000000000..34562755da836 --- /dev/null +++ b/dashboard/dashboard-overview.md @@ -0,0 +1,95 @@ +--- +title: Overview Page +summary: Learn the overview page of TiDB Dashboard. +category: how-to +--- + +# Overview Page + +This page shows the overview of the entire TiDB cluster, including the following information: + +- Queries per second (QPS) of the entire cluster. +- The query latency of the entire cluster. +- The SQL statements that have accumulated the longest execution time over the recent period. +- The slow queries whose execution time over the recent period exceeds the threshold. +- The node count and status of each instance. +- Monitor and alert messages. + +## Access the page + +After logging into TiDB Dashboard, the overview page is entered by default, or you can click **Overview** on the left navigation menu to enter this page: + +![Enter overview page](/media/dashboard/dashboard-overview-access.png) + +## QPS + +This area shows the number of successful and failed queries per second for the entire cluster over the recent hour: + +![QPS](/media/dashboard/dashboard-overview-qps.png) + +> **Note:** +> +> This feature is available only in the cluster where the Prometheus monitoring component is deployed. If Prometheus is not deployed, an error will be displayed. + +## Latency + +This area shows the latency of 99.9%, 99%, and 90% of queries in the entire cluster over the recent one hour: + +![Latency](/media/dashboard/dashboard-overview-latency.png) + +> **Note:** +> +> This feature is available only on the cluster where the Prometheus monitoring component is deployed. If Prometheus is not deployed, an error will be displayed. + +## Top SQL statements + +This area shows the ten types of SQL statements that have accumulated the longest execution time in the entire cluster over the recent period. SQL statements with different query parameters but of the same structure are classified into the same SQL type and displayed in the same row: + +![Top SQL](/media/dashboard/dashboard-overview-top-statements.png) + +The information shown in this area is consistent with the more detailed [SQL Statements Page](/dashboard/dashboard-statement-list.md). You can click the **Top SQL Statements** heading to view the complete list. For details of the columns in this table, see [SQL Statements Page](/dashboard/dashboard-statement-list.md). + +> **Note:** +> +> This feature is available only on the cluster where SQL Statements feature is enabled. + +## Recent slow queries + +By default, this area shows the latest 10 slow queries in the entire cluster over the recent 30 minutes: + +![Recent slow queries](/media/dashboard/dashboard-overview-slow-query.png) + +By default, the SQL query that is executed longer than 300 milliseconds is counted as a slow query and displayed on the table. You can change this threshold by modifying the [tidb_slow_log_threshold](/tidb-specific-system-variables.md#tidb_slow_log_threshold) variable or the [slow-threshold](/tidb-configuration-file.md#slow-threshold) TiDB parameter. + +The content displayed in this area is consistent with the more detailed [Slow Queries Page](/dashboard/dashboard-slow-query.md). You can click the **Recent Slow Queries** title to view the complete list. For details of the columns in this table, see this [Slow Queries Page](/dashboard/dashboard-slow-query.md). + +> **Note:** +> +> This feature is available only in the cluster with slow query logs enabled. By default, slow query logs are enabled in the cluster deployed using TiUP or TiDB Ansible. + +## Instances + +This area summarizes the total number of instances and abnormal instances of TiDB, TiKV, PD, and TiFlash in the entire cluster: + +![Instances](/media/dashboard/dashboard-overview-instances.png) + +The statuses in the image above are described as follows: + +- Up: The instance is running properly (including the offline storage instance). +- Down: The instance is running abnormally, such as network disconnection, process crash, and so on. + +Click the **Instance** title to enter the [Cluster Info Page](/dashboard/dashboard-cluster-info.md) that shows the detailed running status of each instance. + +## Monitor and alert + +This area provides links for you to view detailed monitor and alert: + +![Monitor and alert](/media/dashboard/dashboard-overview-monitor.png) + +- **View Metrics**: Click this link to jump to the Grafana dashboard where you can view detailed monitoring information of the cluster. For details of each monitoring metric in the Grafana dashboard, see [monitoring metrics](/grafana-overview-dashboard.md). +- **View Alerts**: Click this link to jump to the AlertManager page where you can view detailed alert information of the cluster. If alerts exist in the cluster, the number of alerts is directly shown in the link text. +- **Run Diagnostics**: Click this link to jump to the more detailed [cluster diagnostics page](/dashboard/dashboard-diagnostics-access.md). + +> **Note:** +> +> The **View Metrics** link is available only in the cluster where the Grafana node is deployed. The **View Alerts** link is available only in the cluster where the AlertManager node is deployed. diff --git a/dashboard/dashboard-profiling.md b/dashboard/dashboard-profiling.md new file mode 100644 index 0000000000000..5632ddd5b57fb --- /dev/null +++ b/dashboard/dashboard-profiling.md @@ -0,0 +1,55 @@ +--- +title: Instance Profiling Page +summary: Learn the instance profiling page of TiDB Dashboard. +category: how-to +--- + +# Instance Profiling Page + +The instance profiling page summarizes the internal performance data of each TiDB instance, TiKV instance, and PD instance. With this page, you can view the instance profiling without restarting these instances. + +The summarized performance data can be displayed as a flame graph or a directed acyclic graph, which visually shows the internal operations and proportions performed on the instances during the performance summary period. With this graph, you can quickly learn the main CPU resource consumption of these instances. + +## Access the page + +You can access the instance profiling page using one of the following methods: + +- After logging into TiDB Dashboard, click **Advanced Debugging** → **Profile Instances** on the left navigation bar. + + ![Access instance profiling page](/media/dashboard/dashboard-profiling-access.png) + +- Visit in your browser. Replace `127.0.0.1:2379` with the actual PD instance address and port. + +## Start Profiling + +In the instance profiling page, choose at least one target instance and click **Start Profiling** to start the instance profiling. + +![Start instance profiling](/media/dashboard/dashboard-profiling-start.png) + +You can modify the profiling duration before starting the profiling. This duration is determined by the time needed for the profiling, which is 30 seconds by default. The 30-second duration takes approximately 30 seconds to complete. + +## View profiling status + +After a profiling is started, you can view the profiling status and progress in real time. + +![Profiling detail](/media/dashboard/dashboard-profiling-view-progress.png) + +The profiling runs in the background. Refreshing or exiting the current page does not stop the profiling task that is running. + +## Download profiling result + +After the profiling of all instances is completed, you can click **Download Profiling Result** in the upper right corner to download all profiling results. + +![Download profiling result](/media/dashboard/dashboard-profiling-download.png) + +You can also click a single instance on the list to directly view its profiling result. + +![Single instance result](/media/dashboard/dashboard-profiling-view-single.png) + +## View profiling history + +The profiling history is listed on the instance profiling page. Click one row of the list and you can view the status detail. + +![View profiling history](/media/dashboard/dashboard-profiling-history.png) + +For detailed operations on the profiling status page, see [View profiling status](#view-profiling-status). diff --git a/dashboard/dashboard-slow-query.md b/dashboard/dashboard-slow-query.md new file mode 100644 index 0000000000000..88d379bb9c528 --- /dev/null +++ b/dashboard/dashboard-slow-query.md @@ -0,0 +1,61 @@ +--- +title: Slow Queries Page of TiDB Dashboard +summary: Learn the Slow Queries page of TiDB Dashboard. +category: how-to +--- + +# Slow Queries Page of TiDB Dashboard + +On the Slow Queries page of TiDB Dashboard, you can search and view all slow queries in the cluster. + +By default, SQL queries with an execution time of more than 300 milliseconds are considered as slow queries. These queries are recorded in the [slow query logs](/identify-slow-queries.md) and can be searched via TiDB Dashboard. You can adjust the threshold of slow queries through the [`tidb_slow_log_threshold`](/tidb-specific-system-variables.md#tidb_slow_log_threshold) session variable or the [`slow-threshold`](/tidb-configuration-file.md#slow-threshold) TiDB parameter. + +> **Note:** +> +> If the slow query log is disabled, this feature will be unavailable. The slow query log is enabled by default, and you can enable or disable the slow query log through the [`enable-slow-log`](/tidb-configuration-file.md#enable-slow-log) TiDB configuration item. + +## Access the page + +You can use one of the following two methods to access the slow query page: + +* After logging into TiDB Dashboard, click **Slow Queries** on the left navigation menu: + +![Access slow query page](/media/dashboard/dashboard-slow-queries-access.png) + +* Visit in your browser. Replace `127.0.0.1:2379` with the actual PD address and port. + +All data displayed on the slow query page comes from TiDB slow query system tables and slow query logs. See [slow query logs](/identify-slow-queries.md) for details. + +### Change Filters + +You can filter slow queries based on the time range, the related database, SQL keywords, SQL types, the number of slow queries to be displayed. In the image below, 100 slow queries over the recent 30 minutes are displayed by default. + +![Modify list filters](/media/dashboard/dashboard-slow-queries-list1.png) + +### Display More Columns + +Click **Columns** on the page and you can choose to see more columns. You can move your mouse to the **(i)** icon at the right side of a column name to view the description of this column: + +![Show more columns](/media/dashboard/dashboard-slow-queries-list2.png) + +### Sort by Column + +By default, the list is sorted by **Finish Time** in the descending order. Click column headings to sort by the column or switch the sorting order: + +![Modify sorting basis](/media/dashboard/dashboard-slow-queries-list3.png) + +## View execution details + +Click any item in the list to display detailed execution information of the slow query, including: + +- Query: The text of the SQL statement (see area 1 in the image below); +- Plan: The execution plan of the slow query. See [Understand the Query Execution Plan](/query-execution-plan.md) to learn how to read the execution plan (see area 2 in the image below); +- Other sorted SQL execution information (see area 3 in the image below). + +![View execution details](/media/dashboard/dashboard-slow-queries-detail1.png) + +Click the **Expand** link to show the detailed information of an item. Click the **Copy** link to copy the detailed information to the clipboard. + +Click the corresponding tab titles to switch information of different sorted SQL executions. + +![Show different sorted execution information](/media/dashboard/dashboard-slow-queries-detail2.png) diff --git a/dashboard/dashboard-statement-details.md b/dashboard/dashboard-statement-details.md new file mode 100644 index 0000000000000..0c4ac3a6435ae --- /dev/null +++ b/dashboard/dashboard-statement-details.md @@ -0,0 +1,61 @@ +--- +title: Statement Execution Details of TiDB Dashboard +summary: View the execution details of a single SQL statement in TiDB Dashboard. +category: how-to +--- + +# Statement Execution Details of TiDB Dashboard + +Click any item in the list to enter the detail page of the SQL statement to view more detailed information. This information includes the following parts: + +- The overview of SQL statements, which includes the SQL template, the SQL template ID, the current time range of displayed SQL executions, the number of execution plans and the database in which the SQL statement is executed (see area 1 in the image below). +- The execution plan list: If the SQL statement has multiple execution plans, this list is displayed. You can select different execution plans, and the details of the selected plans are displayed below the list. If there is only one execution plan, the list is not displayed (see area 2 below). +- Execution detail of plans, which displays the detailed information of the selected execution plans. See [Execution plan in details](#execution-plan-in-details) (area 3 in the image below). + +![Details](/media/dashboard/dashboard-statement-detail.png) + +## Execution details of plans + +The execution detail of plans includes the following information: + +- SQL sample: The text of a certain SQL statement that is actually executed corresponding to the plan. Any SQL statement that has been executed within the time range might be used as a SQL sample. +- Execution plan: For details of the execution plan, see [Understand the Query Execution Plan](/query-execution-plan.md). If multiple execution plans are selected, only (any) one of them is displayed. +- For basic information, execution time, Coprocessor read, transaction, and slow query of the SQL statement, you can click the corresponding tab titles to switch among different information. + +![Execution details of plans](/media/dashboard/dashboard-statement-plans-detail.png) + +### Basic Tab + +The basic information of a SQL execution includes the table names, index name, execution count, and total latency. The **Description** column provides detailed description of each field. + +![Basic information](/media/dashboard/dashboard-statement-plans-basic.png) + +### Time Tab + +Click the **Time** tab, and you can see how long each stage of the execution plan lasts. + +> **Note:** +> +> Because some operations might be performed in parallel within a single SQL statement, the cumulative duration of each stage might exceed the actual execution time of the SQL statement. + +![Execution time](/media/dashboard/dashboard-statement-plans-time.png) + +### Coprocessor Read Tab + +Click the **Coprocessor Read** tab, and you can see information related to Coprocessor read. + +![Coprocessor read](/media/dashboard/dashboard-statement-plans-cop-read.png) + +### Transaction Tab + +Click the **Transaction** tab, and you can see information related to execution plans and transactions, such as the average number of written keys or the maximum number of written keys. + +![Transaction](/media/dashboard/dashboard-statement-plans-transaction.png) + +### Slow Query Tab + +If an execution plan is executed too slowly, you can see its associated slow query records under the **Slow Query** tab. + +![Slow Query](/media/dashboard/dashboard-statement-plans-slow-queries.png) + +The information displayed in this area has the same structure with the slow query page. See [TiDB Dashboard Slow Query Page](/dashboard/dashboard-slow-query.md) for details. diff --git a/dashboard/dashboard-statement-list.md b/dashboard/dashboard-statement-list.md new file mode 100644 index 0000000000000..a6ba0f541bc80 --- /dev/null +++ b/dashboard/dashboard-statement-list.md @@ -0,0 +1,64 @@ +--- +title: SQL Statements Page of TiDB Dashboard +summary: View the execution status of all SQL statements in the TiDB cluster. +category: how-to +--- + +# SQL Statements Page of TiDB Dashboard + +The SQL statements page shows the execution status of all SQL statements in the cluster. This page is often used to analyze the SQL statement whose total or single execution time is long. + +On this page, SQL queries with a consistent structure (even if the query parameters are inconsistent) are classified as the same SQL statement. For example, both `SELECT * FROM employee WHERE id IN (1, 2, 3)` and `select * from EMPLOYEE where ID in (4, 5)` are classified as the same `select * from employee where id in (...)` SQL statement. + +## Access the page + +You can use one of the following two methods to access the SQL statement summary page: + +- After logging into TiDB Dashboard, click **SQL Statements** on the left navigation menu: + + ![Access SQL statement summary page](/media/dashboard/dashboard-statement-access.png) + +- Visit in your browser. Replace `127.0.0.1:2379` with the actual PD instance address and port. + +All the data shown on the SQL statement summary page are from the TiDB statement summary tables. For more details about the tables, see [TiDB Statement Summary Tables](/statement-summary-tables.md). + +### Change Filters + +On the top of the SQL statement summary page, you can modify the time range of SQL executions to be displayed. You can also filter the list by database in which SQL statements are executed, or by SQL types. The following image shows all SQL executions over the recent data collection cycle (recent 30 minutes by default). + +![Modify filters](/media/dashboard/dashboard-statement-filter-options.png) + +### Display More Columns + +Click **Columns** on the page and you can choose to see more columns. You can move your mouse to the **(i)** icon at the right side of a column name to view the description of this column: + +![Choose columns](/media/dashboard/dashboard-statement-columns-selector.png) + +### Sort by Column + +By default, the list is sorted by **Total Latency** from high to low. Click on different column headings to modify the sorting basis or switch the sorting order: + +![Modify list sorting](/media/dashboard/dashboard-statement-change-order.png) + +### Change Settings + +On the list page, click the **Settings** button on the top right to change the settings of the SQL statements feature: + +![Settings entry](/media/dashboard/dashboard-statement-setting-entry.png) + +After clicking the **Settings** button, you can see the following setting dialog box: + +![Settings](/media/dashboard/dashboard-statement-settings.png) + +On the setting page, you can disable or enable the SQL statements feature. When the SQL statements feature is enabled, you can modify the following settings: + +- Collect interval: The length of period for each SQL statement analysis, which is 30 minutes by default. The SQL statements feature summarizes and counts all SQL statements within a period of time. If the period is too long, the granularity of the summary is coarse, which is not good for locating problems; if the period is too short, the granularity of the statistics is fine, which is good for locating problems, but this will result in more records and more memory usage within the same data retention duration. Therefore, you need to adjust this value based on the actual situation, and properly lower this value when locating problems. +- Data retain duration: The retention duration of summary information, which is 1 day by default. Data retained longer than this duration will be deleted from system tables. + +See [Configurations of Statement Summary Tables](/statement-summary-tables.md#configurations) for details. + +> **Note:** +> +> + Because the statement system table is only stored in memory, after the SQL Statements feature is disabled, the data in the system table will be cleared. +> +> + The values of `Collect interval` and `retain duration` affect the memory usage, so it is recommended to adjust these values according to the actual situation. The value of `retain duration` should not be set too large. diff --git a/data-type-json.md b/data-type-json.md index 2aa1a6c83653e..f562acd5bb4c3 100644 --- a/data-type-json.md +++ b/data-type-json.md @@ -7,6 +7,10 @@ aliases: ['/docs/dev/reference/sql/data-types/json/'] # JSON Type +> **Warning:** +> +> This is still an experimental feature. It is **NOT** recommended that you use it in the production environment. + TiDB supports the `JSON` (JavaScript Object Notation) data type, which is useful for storing semi-structured data. The `JSON` data type provides the following advantages over storing `JSON`-format strings in a string column: - Use the Binary format for serialization. The internal format permits quick read access to `JSON` document elements. diff --git a/monitor-a-tidb-cluster.md b/deploy-monitoring-services.md similarity index 62% rename from monitor-a-tidb-cluster.md rename to deploy-monitoring-services.md index 85bc9d0a31bbc..72b6703d716fd 100644 --- a/monitor-a-tidb-cluster.md +++ b/deploy-monitoring-services.md @@ -1,88 +1,17 @@ --- -title: Monitor a TiDB Cluster -summary: Learn how to monitor the state of a TiDB cluster. +title: Deploy Monitoring Services for the TiDB Cluster +summary: Learn how to deploy monitoring services for the TiDB cluster. category: how-to -aliases: ['/docs/dev/how-to/monitor/monitor-a-cluster/'] +aliases: ['/docs/dev/how-to/monitor/monitor-a-cluster/','/docs/dev/monitor-a-tidb-cluster/'] --- -# Monitor a TiDB Cluster +# Deploy Monitoring Services for the TiDB Cluster -You can use the following two types of interfaces to monitor the TiDB cluster state: +This document is intended for users who want to manually deploy TiDB monitoring and alert services. -- [The state interface](#use-the-state-interface): this interface uses the HTTP interface to get the component information. -- [The metrics interface](#use-the-metrics-interface): this interface uses Prometheus to record the detailed information of the various operations in components and views these metrics using Grafana. +If you deploy the TiDB cluster using TiUP, the monitoring and alert services are automatically deployed, and no manual deployment is needed. -## Use the state interface - -The state interface monitors the basic information of a specific component in the TiDB cluster. It can also act as the monitor interface for Keepalive messages. In addition, the state interface for the Placement Driver (PD) can get the details of the entire TiKV cluster. - -### TiDB server - -- TiDB API address: `http://${host}:${port}` -- Default port: `10080` -- Details about API names: see [TiDB HTTP API](https://github.com/pingcap/tidb/blob/master/docs/tidb_http_api.md) - -The following example uses `http://${host}:${port}/status` to get the current state of the TiDB server and to determine whether the server is alive. The result is returned in JSON format. - -```bash -curl http://127.0.0.1:10080/status -{ - connections: 0, # The current number of clients connected to the TiDB server. - version: "5.7.25-TiDB-v3.0.0-beta-250-g778c3f4a5", # The TiDB version number. - git_hash: "778c3f4a5a716880bcd1d71b257c8165685f0d70" # The Git Hash of the current TiDB code. -} -``` - -### PD server - -- PD API address: `http://${host}:${port}/pd/api/v1/${api_name}` -- Default port: `2379` -- Details about API names: see [PD API doc](https://download.pingcap.com/pd-api-v1.html) - -The PD interface provides the state of all the TiKV servers and the information about load balancing. See the following example for the information about a single-node TiKV cluster: - -```bash -curl http://127.0.0.1:2379/pd/api/v1/stores -{ - "count": 1, # The number of TiKV nodes. - "stores": [ # The list of TiKV nodes. - # The details about the single TiKV node. - { - "store": { - "id": 1, - "address": "127.0.0.1:20160", - "version": "3.0.0-beta", - "state_name": "Up" - }, - "status": { - "capacity": "20 GiB", # The total capacity. - "available": "16 GiB", # The available capacity. - "leader_count": 17, - "leader_weight": 1, - "leader_score": 17, - "leader_size": 17, - "region_count": 17, - "region_weight": 1, - "region_score": 17, - "region_size": 17, - "start_ts": "2019-03-21T14:09:32+08:00", # The starting timestamp. - "last_heartbeat_ts": "2019-03-21T14:14:22.961171958+08:00", # The timestamp of the last heartbeat. - "uptime": "4m50.961171958s" - } - } - ] -``` - -## Use the metrics interface - -The metrics interface monitors the state and performance of the entire TiDB cluster. - -- If you use TiDB Ansible to deploy the TiDB cluster, the monitoring system (Prometheus and Grafana) is deployed at the same time. -- If you use other deployment ways, [deploy Prometheus and Grafana](#deploy-prometheus-and-grafana) before using this interface. - -After Prometheus and Grafana are successfully deployed, [configure Grafana](#configure-grafana). - -### Deploy Prometheus and Grafana +## Deploy Prometheus and Grafana Assume that the TiDB cluster topology is as follows: @@ -95,7 +24,7 @@ Assume that the TiDB cluster topology is as follows: | Node5 | 192.168.199.117| TiKV2, node_export | | Node6 | 192.168.199.118| TiKV3, node_export | -#### Step 1: Download the binary package +### Step 1: Download the binary package {{< copyable "shell-regular" >}} @@ -115,7 +44,7 @@ tar -xzf node_exporter-0.17.0.linux-amd64.tar.gz tar -xzf grafana-6.1.6.linux-amd64.tar.gz ``` -#### Step 2: Start `node_exporter` on Node1, Node2, Node3, and Node4 +### Step 2: Start `node_exporter` on Node1, Node2, Node3, and Node4 {{< copyable "shell-regular" >}} @@ -127,7 +56,7 @@ $ ./node_exporter --web.listen-address=":9100" \ --log.level="info" & ``` -#### Step 3: Start Prometheus on Node1 +### Step 3: Start Prometheus on Node1 Edit the Prometheus configuration file: @@ -200,7 +129,7 @@ $ ./prometheus \ --storage.tsdb.retention="15d" & ``` -#### Step 4: Start Grafana on Node1 +### Step 4: Start Grafana on Node1 Edit the Grafana configuration file: @@ -259,11 +188,11 @@ $ ./bin/grafana-server \ --config="./conf/grafana.ini" & ``` -### Configure Grafana +## Configure Grafana This section describes how to configure Grafana. -#### Step 1: Add a Prometheus data source +### Step 1: Add a Prometheus data source 1. Log in to the Grafana Web interface. @@ -288,7 +217,7 @@ This section describes how to configure Grafana. 5. Click **Add** to save the new data source. -#### Step 2: Import a Grafana dashboard +### Step 2: Import a Grafana dashboard To import a Grafana dashboard for the PD server, the TiKV server, and the TiDB server, take the following steps respectively: @@ -308,7 +237,7 @@ To import a Grafana dashboard for the PD server, the TiKV server, and the TiDB s 6. Click **Import**. A Prometheus dashboard is imported. -### View component metrics +## View component metrics Click **New dashboard** in the top menu and choose the dashboard you want to view. diff --git a/deploy-tidb-from-dbdeployer.md b/deploy-tidb-from-dbdeployer.md index 1432e6d0de2dc..0821103544996 100644 --- a/deploy-tidb-from-dbdeployer.md +++ b/deploy-tidb-from-dbdeployer.md @@ -16,12 +16,12 @@ Similar to [Homebrew](/deploy-tidb-from-homebrew.md), the DBdeployer installatio > Internally this installation uses goleveldb as the storage engine. It is much slower than TiKV, and any benchmarks will be unreliable.
- + - +