diff --git a/docs/cn/release-notes/databend.md b/docs/cn/release-notes/databend.md index 125240d031..30e03fc2d6 100644 --- a/docs/cn/release-notes/databend.md +++ b/docs/cn/release-notes/databend.md @@ -1,6 +1,6 @@ --- -sidebar_label: Databend 版本发布 -title: Databend 版本发布 +sidebar_label: Databend Releases +title: Databend Releases sidebar_position: 1 slug: / --- @@ -8,7 +8,9 @@ slug: / import StepsWrap from '@site/src/components/StepsWrap'; import StepContent from '@site/src/components/Steps/step-content'; -本页面提供 Databend 最新功能、增强与错误修复的相关信息。 +This page provides information about recent features, enhancements, and bug fixes for Databend. + + import MD1 from '@site/docs/release-stable/2025-04-21_v1.2.725.md'; import MD2 from '@site/docs/release-stable/2025-02-14_v1.2.697.md'; @@ -35,7 +37,7 @@ import MD17 from '@site/docs/release-stable/2023-12-13_v1.2.233.md'; -## 2025 年 4 月 21 日（v1.2.725） +## Apr 21, 2025 (v1.2.725) @@ -43,7 +45,7 @@ import MD17 from '@site/docs/release-stable/2023-12-13_v1.2.233.md'; -## 2025 年 2 月 14 日（v1.2.697） +## Feb 14, 2025 (v1.2.697) @@ -51,7 +53,7 @@ import MD17 from '@site/docs/release-stable/2023-12-13_v1.2.233.md'; -## 2025 年 1 月 2 日（v1.2.680） +## Jan 2, 2025 (v1.2.680) @@ -59,7 +61,7 @@ import MD17 from '@site/docs/release-stable/2023-12-13_v1.2.233.md'; -## 2024 年 8 月 19 日（v1.2.615） +## Aug 19, 2024 (v1.2.615) @@ -67,7 +69,7 @@ import MD17 from '@site/docs/release-stable/2023-12-13_v1.2.233.md'; -## 2024 年 8 月 13 日（v1.2.609） +## Aug 13, 2024 (v1.2.609) @@ -75,7 +77,7 @@ import MD17 from '@site/docs/release-stable/2023-12-13_v1.2.233.md'; -## 2024 年 6 月 16 日（v1.2.530） +## Jun 16, 2024 (v1.2.530) @@ -83,7 +85,7 @@ import MD17 from '@site/docs/release-stable/2023-12-13_v1.2.233.md'; -## 2024 年 5 月 6 日（v1.2.452） +## May 6, 2024 (v1.2.452) @@ -91,7 +93,7 @@ import MD17 from '@site/docs/release-stable/2023-12-13_v1.2.233.md'; -## 2024 年 4 月 8 日（v1.2.410） +## Apr 8, 2024 (v1.2.410) @@ -99,7 +101,7 @@ import MD17 from '@site/docs/release-stable/2023-12-13_v1.2.233.md'; -## 2024 年 3 月 11 日（v1.2.371） +## Mar 11, 2024 (v1.2.371) @@ -107,7 +109,7 @@ import MD17 from '@site/docs/release-stable/2023-12-13_v1.2.233.md'; -## 2024 年 2 月 22 日（v1.2.344） +## Feb 22, 2024 (v1.2.344) @@ -115,7 +117,7 @@ import MD17 from '@site/docs/release-stable/2023-12-13_v1.2.233.md'; -## 2024 年 1 月 25 日（v1.2.307） +## Jan 25, 2024 (v1.2.307) @@ -123,7 +125,7 @@ import MD17 from '@site/docs/release-stable/2023-12-13_v1.2.233.md'; -## 2024 年 1 月 11 日（v1.2.292） +## Jan 11, 2024 (v1.2.292) @@ -131,7 +133,7 @@ import MD17 from '@site/docs/release-stable/2023-12-13_v1.2.233.md'; -## 2024 年 1 月 10 日（v1.2.290） +## Jan 10, 2024 (v1.2.290) @@ -139,7 +141,7 @@ import MD17 from '@site/docs/release-stable/2023-12-13_v1.2.233.md'; -## 2024 年 1 月 2 日（v1.2.279） +## Jan 2, 2024 (v1.2.279) @@ -147,7 +149,7 @@ import MD17 from '@site/docs/release-stable/2023-12-13_v1.2.233.md'; -## 2023 年 12 月 30 日（v1.2.275） +## Dec 30, 2023 (v1.2.275) @@ -155,7 +157,7 @@ import MD17 from '@site/docs/release-stable/2023-12-13_v1.2.233.md'; -## 2023 年 12 月 20 日（v1.2.262） +## Dec 20, 2023 (v1.2.262) @@ -163,10 +165,10 @@ import MD17 from '@site/docs/release-stable/2023-12-13_v1.2.233.md'; -## 2023 年 12 月 13 日（v1.2.233） +## Dec 13, 2023 (v1.2.233) - \ No newline at end of file + diff --git a/docs/en/guides/00-products/01-dee/10-enterprise-features.md b/docs/en/guides/00-products/01-dee/10-enterprise-features.md index 9789b9df58..b93e6bff5b 100644 --- a/docs/en/guides/00-products/01-dee/10-enterprise-features.md +++ b/docs/en/guides/00-products/01-dee/10-enterprise-features.md @@ -20,7 +20,7 @@ This page provides an updated list of available enterprise features. To access t | [Ngram Index](/guides/performance/ngram-index) | Query Performance | Accelerate LIKE pattern matching queries with wildcard searches. | | [Virtual Column](/sql/sql-commands/ddl/virtual-column) | Query Performance | Automatically accelerate JSON queries with zero-configuration performance optimization for VARIANT data. | | [Dynamic Column](/sql/sql-commands/ddl/table/ddl-create-table#computed-columns) | Query Performance | Generate columns automatically from scalar expressions with stored or virtual calculation modes. | -| [Python UDF](/guides/query/udf#python-requires-databend-enterprise) | Advanced Analytics | Execute Python code within SQL queries using built-in handler. | +| [Python UDF](/guides/query/advanced/udf#python-requires-databend-enterprise) | Advanced Analytics | Execute Python code within SQL queries using built-in handler. | | [ATTACH TABLE](/sql/sql-commands/ddl/table/attach-table) | Data Sharing | Create read-only links to existing table data with zero-copy access across environments. | | [Stream](/sql/sql-commands/ddl/stream) | Change Data Capture | Track and capture table changes for incremental data processing. | | [Vacuum Temp Files](/sql/sql-commands/administration-cmds/vacuum-temp-files) | Storage Management | Clean up temporary files (join, aggregate, sort spills) to free storage space. | diff --git a/docs/en/guides/20-cloud/20-manage/01-monitor.md b/docs/en/guides/20-cloud/20-manage/01-monitor.md index c4405c3ec3..8a7806e989 100644 --- a/docs/en/guides/20-cloud/20-manage/01-monitor.md +++ b/docs/en/guides/20-cloud/20-manage/01-monitor.md @@ -27,7 +27,7 @@ The **SQL History** tab displays a list of SQL statements that have been execute Clicking a record on the **SQL History** page reveals detailed information on how Databend Cloud executed the SQL statement, providing access to the following tabs: - **Query Details**: Includes Query State (success or failure), Rows Scanned, Warehouse, Bytes Scanned, Start Time, End Time, and Handler Type. -- **Query Profile**: Illustrates how the SQL statement was executed. For more information, see [Query Profile](/guides/query/query-profile). +- **Query Profile**: Illustrates how the SQL statement was executed. ## Task History diff --git a/docs/en/guides/51-ai-functions/01-external-functions.md b/docs/en/guides/51-ai-functions/01-external-functions.md index 7a8218ed26..0ec8bbba73 100644 --- a/docs/en/guides/51-ai-functions/01-external-functions.md +++ b/docs/en/guides/51-ai-functions/01-external-functions.md @@ -78,5 +78,5 @@ LIMIT 5; ## Learn More -- **[External Functions Guide](/guides/query/external-function)** - Complete setup and deployment instructions +- **[External Functions Guide](/guides/query/advanced/external-function)** - Complete setup and deployment instructions - **[Databend Cloud](https://databend.com)** - Try external functions with a free account \ No newline at end of file diff --git a/docs/en/guides/54-query/00-basics/_category_.json b/docs/en/guides/54-query/00-basics/_category_.json new file mode 100644 index 0000000000..78ffea89df --- /dev/null +++ b/docs/en/guides/54-query/00-basics/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Basic Queries", + "position": 1 +} \ No newline at end of file diff --git a/docs/en/guides/54-query/00-basics/aggregating-data.md b/docs/en/guides/54-query/00-basics/aggregating-data.md new file mode 100644 index 0000000000..ad209f2ac4 --- /dev/null +++ b/docs/en/guides/54-query/00-basics/aggregating-data.md @@ -0,0 +1,90 @@ +--- +title: Aggregating Data +--- + +Learn to summarize and analyze data using GROUP BY, aggregate functions, and advanced grouping techniques. + +## Basic Aggregation + +### Common Aggregate Functions +```sql +-- Count rows +SELECT COUNT(*) FROM employees; + +-- Statistical functions +SELECT + AVG(salary) as avg_salary, + MIN(salary) as min_salary, + MAX(salary) as max_salary, + SUM(salary) as total_salary +FROM employees; +``` + +## GROUP BY Fundamentals + +### Single Column Grouping +```sql +-- Count employees by department +SELECT department, COUNT(*) as emp_count +FROM employees +GROUP BY department; + +-- Average salary by department +SELECT department, AVG(salary) as avg_salary +FROM employees +GROUP BY department +ORDER BY avg_salary DESC; +``` + +### Multiple Column Grouping +```sql +-- Group by department and hire year +SELECT + department, + EXTRACT(YEAR FROM hire_date) as hire_year, + COUNT(*) as count, + AVG(salary) as avg_salary +FROM employees +GROUP BY department, EXTRACT(YEAR FROM hire_date) +ORDER BY department, hire_year; +``` + +### GROUP BY with HAVING +```sql +-- Find departments with more than 5 employees +SELECT department, COUNT(*) as emp_count +FROM employees +GROUP BY department +HAVING COUNT(*) > 5; + +-- Departments with average salary > 70000 +SELECT department, AVG(salary) as avg_salary +FROM employees +GROUP BY department +HAVING AVG(salary) > 70000; +``` + +## Advanced Grouping + +### GROUP BY ALL +```sql +-- Automatically group by all non-aggregate columns +SELECT department, job_title, COUNT(*) as count +FROM employees +GROUP BY ALL; +``` + +## Advanced Grouping Extensions + +Databend supports SQL:2003 standard grouping extensions: + +- **[ROLLUP](./groupby/group-by-rollup.md)** - Hierarchical subtotals +- **[CUBE](./groupby/group-by-cube.md)** - All possible combinations +- **[GROUPING SETS](./groupby/group-by-grouping-sets.md)** - Custom combinations + +## Best Practices + +1. **Use appropriate aggregates** - COUNT(*) vs COUNT(column) +2. **Filter before grouping** - Use WHERE before GROUP BY +3. **Use HAVING for aggregate conditions** - Filter groups after aggregation +4. **Consider indexes** - GROUP BY columns should be indexed \ No newline at end of file diff --git a/docs/en/guides/54-query/00-basics/filtering-selection.md b/docs/en/guides/54-query/00-basics/filtering-selection.md new file mode 100644 index 0000000000..172023ed36 --- /dev/null +++ b/docs/en/guides/54-query/00-basics/filtering-selection.md @@ -0,0 +1,103 @@ +--- +title: Filtering & Selection +--- + +Learn the fundamentals of querying data in Databend with SELECT, WHERE, and basic operations. + +## Basic SELECT Queries + +### Selecting Columns +```sql +-- Select specific columns +SELECT name, salary FROM employees; + +-- Select all columns +SELECT * FROM employees; + +-- Select with column aliases +SELECT name AS employee_name, salary AS monthly_pay +FROM employees; +``` + +### Filtering with WHERE +```sql +-- Simple condition +SELECT * FROM employees WHERE department = 'Engineering'; + +-- Multiple conditions +SELECT * FROM employees +WHERE salary > 70000 AND department = 'Engineering'; + +-- Using OR +SELECT * FROM employees +WHERE department = 'Engineering' OR department = 'Marketing'; + +-- Range conditions +SELECT * FROM employees +WHERE salary BETWEEN 60000 AND 80000; + +-- Pattern matching +SELECT * FROM employees +WHERE name LIKE 'A%'; -- Names starting with 'A' +``` + +## Sorting Results + +### ORDER BY Clause +```sql +-- Sort by single column +SELECT * FROM employees ORDER BY salary DESC; + +-- Sort by multiple columns +SELECT * FROM employees +ORDER BY department ASC, salary DESC; + +-- Sort by column position +SELECT name, salary FROM employees ORDER BY 2 DESC; +``` + +## Limiting Results + +### LIMIT and OFFSET +```sql +-- Get top 5 highest paid employees +SELECT * FROM employees +ORDER BY salary DESC +LIMIT 5; + +-- Pagination - skip first 10, get next 5 +SELECT * FROM employees +ORDER BY salary DESC +LIMIT 5 OFFSET 10; +``` + +## Common Operators + +### Comparison Operators +- `=` Equal to +- `!=` or `<>` Not equal to +- `>` Greater than +- `<` Less than +- `>=` Greater than or equal +- `<=` Less than or equal + +### Logical Operators +- `AND` Both conditions must be true +- `OR` Either condition can be true +- `NOT` Negates a condition + +### NULL Handling +```sql +-- Check for NULL values +SELECT * FROM employees WHERE manager_id IS NULL; + +-- Check for non-NULL values +SELECT * FROM employees WHERE manager_id IS NOT NULL; +``` + +## Best Practices + +1. **Be specific with columns** - Avoid `SELECT *` in production +2. **Use indexes** - WHERE conditions on indexed columns are faster +3. **Limit large results** - Always use LIMIT for exploratory queries +4. **Filter early** - Apply WHERE conditions before JOINs when possible \ No newline at end of file diff --git a/docs/en/guides/54-query/01-groupby/_category_.json b/docs/en/guides/54-query/00-basics/groupby/_category_.json similarity index 100% rename from docs/en/guides/54-query/01-groupby/_category_.json rename to docs/en/guides/54-query/00-basics/groupby/_category_.json diff --git a/docs/en/guides/54-query/01-groupby/group-by-cube.md b/docs/en/guides/54-query/00-basics/groupby/group-by-cube.md similarity index 100% rename from docs/en/guides/54-query/01-groupby/group-by-cube.md rename to docs/en/guides/54-query/00-basics/groupby/group-by-cube.md diff --git a/docs/en/guides/54-query/01-groupby/group-by-grouping-sets.md b/docs/en/guides/54-query/00-basics/groupby/group-by-grouping-sets.md similarity index 100% rename from docs/en/guides/54-query/01-groupby/group-by-grouping-sets.md rename to docs/en/guides/54-query/00-basics/groupby/group-by-grouping-sets.md diff --git a/docs/en/guides/54-query/01-groupby/group-by-rollup.md b/docs/en/guides/54-query/00-basics/groupby/group-by-rollup.md similarity index 100% rename from docs/en/guides/54-query/01-groupby/group-by-rollup.md rename to docs/en/guides/54-query/00-basics/groupby/group-by-rollup.md diff --git a/docs/en/guides/54-query/01-groupby/index.md b/docs/en/guides/54-query/00-basics/groupby/index.md similarity index 100% rename from docs/en/guides/54-query/01-groupby/index.md rename to docs/en/guides/54-query/00-basics/groupby/index.md diff --git a/docs/en/guides/54-query/00-basics/index.md b/docs/en/guides/54-query/00-basics/index.md new file mode 100644 index 0000000000..9a48389174 --- /dev/null +++ b/docs/en/guides/54-query/00-basics/index.md @@ -0,0 +1,63 @@ +--- +title: Basic Queries +--- + +Master the fundamentals of querying data in Databend. Start here if you're new to SQL or need a refresher on core concepts. + +## What You'll Learn + +- Select and filter data effectively +- Sort and limit query results +- Group data and calculate aggregates +- Use advanced grouping techniques + +## Query Essentials + +### [Filtering & Selection](./filtering-selection.md) +Learn the basics: SELECT, WHERE, ORDER BY, and LIMIT +```sql +SELECT name, salary FROM employees +WHERE department = 'Engineering' +ORDER BY salary DESC; +``` + +### [Aggregating Data](./aggregating-data.md) +Summarize data with GROUP BY and aggregate functions +```sql +SELECT department, AVG(salary) as avg_salary +FROM employees +GROUP BY department; +``` + +### [Advanced Grouping](./groupby/index.md) +Multi-dimensional analysis with CUBE, ROLLUP, and GROUPING SETS +```sql +-- Generate all possible grouping combinations +SELECT department, job_level, COUNT(*) +FROM employees +GROUP BY CUBE(department, job_level); +``` + +## Quick Reference + +### Most Common Patterns +```sql +-- Top N query +SELECT * FROM table ORDER BY column DESC LIMIT 10; + +-- Count by category +SELECT category, COUNT(*) FROM table GROUP BY category; + +-- Filter and aggregate +SELECT region, AVG(sales) +FROM orders +WHERE order_date >= '2023-01-01' +GROUP BY region +HAVING AVG(sales) > 1000; +``` + +## Next Steps + +Once you're comfortable with basic queries: +- [Combining Data](../01-combining-data/index.md) - JOIN tables and use CTEs +- [Advanced Features](../02-advanced/index.md) - Custom functions and procedures \ No newline at end of file diff --git a/docs/en/guides/54-query/00-cte.md b/docs/en/guides/54-query/00-cte.md deleted file mode 100644 index 2ebc39b171..0000000000 --- a/docs/en/guides/54-query/00-cte.md +++ /dev/null @@ -1,322 +0,0 @@ ---- -title: Common Table Expressions (CTE) ---- -import FunctionDescription from '@site/src/components/FunctionDescription'; - - - -Databend supports common table expressions (CTEs) with a WITH clause, allowing you to define one or multiple named temporary result sets used by the following query. The term "temporary" implies that the result sets are not permanently stored in the database schema. They act as temporary views only accessible to the following query. - -When a query with a WITH clause is executed, the CTEs within the WITH clause are evaluated and executed first. This produces one or multiple temporary result sets. Then the query is executed using the temporary result sets that were produced by the WITH clause. - -This is a simple demonstration that helps you understand how CTEs work in a query: The WITH clause defines a CTE and produces a result set that holds all customers who are from the Québec province. The main query filters the customers who live in the Montréal region from the ones in the Québec province. - -```sql -WITH customers_in_quebec - AS (SELECT customername, - city - FROM customers - WHERE province = 'Québec') -SELECT customername -FROM customers_in_quebec -WHERE city = 'Montréal' -ORDER BY customername; -``` - -CTEs simplify complex queries that use subqueries and make your code easier to read and maintain. The preceding example would be like this without using a CTE: - -```sql -SELECT customername -FROM (SELECT customername, - city - FROM customers - WHERE province = 'Québec') -WHERE city = 'Montréal' -ORDER BY customername; -``` - -## Inline or Materialized? - -When using a CTE in a query, you can control whether the CTE is inline or materialized by using the MATERIALIZED keyword. Inlining means the CTE's definition is directly embedded within the main query, while materializing the CTE means calculating its result once and storing it in memory, reducing repetitive CTE execution. - -Suppose we have a table called *orders*, storing customer order information, including order number, customer ID, and order date. - -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; - - - - -In this query, the CTE *customer_orders* will be inlined during query execution. Databend will directly embed the definition of *customer_orders* within the main query. - -```sql -WITH customer_orders AS ( - SELECT customer_id, COUNT(*) AS order_count - FROM orders - GROUP BY customer_id -) -SELECT co1.customer_id, co1.order_count, co2.order_count AS other_order_count -FROM customer_orders co1 -JOIN customer_orders co2 ON co1.customer_id = co2.customer_id -WHERE co1.order_count > 2 - AND co2.order_count > 5; -``` - - - -In this case, we use the MATERIALIZED keyword, which means the CTE *customer_orders* will not be inlined. Instead, the CTE's result will be calculated and stored in memory during the CTE definition's execution. When executing both instances of the CTE within the main query, Databend will directly retrieve the result from memory, avoiding redundant calculations and potentially improving performance. - -```sql -WITH customer_orders AS MATERIALIZED ( - SELECT customer_id, COUNT(*) AS order_count - FROM orders - GROUP BY customer_id -) -SELECT co1.customer_id, co1.order_count, co2.order_count AS other_order_count -FROM customer_orders co1 -JOIN customer_orders co2 ON co1.customer_id = co2.customer_id -WHERE co1.order_count > 2 - AND co2.order_count > 5; -``` -This can significantly improve performance for situations where the CTE's result is used multiple times. However, as the CTE is no longer inlined, the query optimizer may find it difficult to push the CTE's conditions into the main query or optimize the join order, potentially leading to decreased overall query performance. - - - - - -## Syntax - -```sql -WITH - [ ( ) ] AS [MATERIALIZED] ( SELECT ... ) - [ , [ ( ) ] AS [MATERIALIZED] ( SELECT ... ) ] - [ , [ ( ) ] AS [MATERIALIZED] ( SELECT ... ) ] -SELECT ... | UPDATE ... | DELETE FROM ... -``` - -| Parameter | Description | -|------------------------- |--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| WITH | Initiates the WITH clause. | -| cte_name1 ... cte_nameN | The CTE names. When you have multiple CTEs, separate them with commas. | -| cte_column_list | The names of the columns in the CTE. A CTE can refer to any CTEs in the same WITH clause that are defined before. | -| MATERIALIZED | `Materialized` is an optional keyword used to indicate whether the CTE should be materialized. | - -## Recursive CTEs - -A recursive CTE is a temporary result set that references itself to perform recursive operations, allowing the processing of hierarchical or recursive data structures. - -### Syntax - -```sql -WITH RECURSIVE AS ( - - UNION ALL - ) -SELECT ... -``` - -| Parameter | Description | -|-------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `cte_name` | The CTE name. | -| `initial_query` | The initial query that is executed once at the start of the recursion. It typically returns a set of rows. | -| `recursive_query` | The query that references the CTE itself and is executed repeatedly until an empty result set is returned. It must include a reference to the CTE's name. The query must NOT include aggregate functions (e.g., MAX, MIN, SUM, AVG, COUNT), window functions, GROUP BY clause, ORDER BY clause, LIMIT clause, or DISTINCT. | - -### How it Works - -The following describes the detailed execution order of a recursive CTE: - -1. **Initial Query Execution**: This query forms the base result set, denoted as R0. This result set provides the starting point for the recursion. - -2. **Recursive Query Execution**: This query uses the result set from the previous iteration (starting with R0) as input and produces a new result set (Ri+1). - -3. **Iteration and Combination**: The recursive execution continues iteratively. Each new result set (Ri) from the recursive query becomes the input for the next iteration. This process repeats until the recursive query returns an empty result set, indicating that the termination condition has been met. - -4. **Final Result Set Formation**: Using the `UNION ALL` operator, the result sets from each iteration (R0 through Rn) are combined into a single result set. The `UNION ALL` operator ensures that all rows from each result set are included in the final combined result. - -5. **Final Selection**: The final `SELECT ...` statement retrieves the combined result set from the CTE. This statement can apply additional filtering, sorting, or other operations on the combined result set to produce the final output. - -## Usage Examples - -### Non-Recursive CTEs - -Imagine you manage several bookstores located in different regions of the GTA area, and use a table to hold their store IDs, regions, and the trading volume for the last month. - -```sql -CREATE TABLE sales - ( - storeid INTEGER, - region TEXT, - amount INTEGER - ); - -INSERT INTO sales VALUES (1, 'North York', 12800); -INSERT INTO sales VALUES (2, 'Downtown', 28400); -INSERT INTO sales VALUES (3, 'Markham', 6720); -INSERT INTO sales VALUES (4, 'Mississauga', 4990); -INSERT INTO sales VALUES (5, 'Downtown', 5670); -INSERT INTO sales VALUES (6, 'Markham', 4350); -INSERT INTO sales VALUES (7, 'North York', 2490); -``` - -The following code returns the stores with a trading volume lower than the average: - -```sql --- Define a WITH clause including one CTE -WITH avg_all - AS (SELECT Avg(amount) AVG_SALES - FROM sales) -SELECT * -FROM sales, - avg_all -WHERE sales.amount < avg_sales; -``` - -Output: - -```text -┌──────────────────────────────────────────────────────────────────────────┐ -│ storeid │ region │ amount │ avg_sales │ -├─────────────────┼──────────────────┼─────────────────┼───────────────────┤ -│ 5 │ Downtown │ 5670 │ 9345.714285714286 │ -│ 4 │ Mississauga │ 4990 │ 9345.714285714286 │ -│ 7 │ North York │ 2490 │ 9345.714285714286 │ -│ 3 │ Markham │ 6720 │ 9345.714285714286 │ -│ 6 │ Markham │ 4350 │ 9345.714285714286 │ -└──────────────────────────────────────────────────────────────────────────┘ -``` - -The following code returns the average and total volume of each region: - -```sql --- Define a WITH clause including two CTEs -WITH avg_by_region - AS (SELECT region, - Avg (amount) avg_by_region_value - FROM sales - GROUP BY region), - sum_by_region - AS (SELECT region, - Sum(amount) sum_by_region_value - FROM sales - GROUP BY region) -SELECT avg_by_region.region, - avg_by_region_value, - sum_by_region_value -FROM avg_by_region, - sum_by_region -WHERE avg_by_region.region = sum_by_region.region; -``` - -Output: - -```text -┌──────────────────────────────────────────────────────────────┐ -│ region │ avg_by_region_value │ sum_by_region_value │ -├──────────────────┼─────────────────────┼─────────────────────┤ -│ North York │ 7645 │ 15290 │ -│ Downtown │ 17035 │ 34070 │ -│ Markham │ 5535 │ 11070 │ -│ Mississauga │ 4990 │ 4990 │ -└──────────────────────────────────────────────────────────────┘ -``` - -The following code updates the sales amount to 0 for stores where the sales amount is below the average sales of their respective regions: - -```sql -WITH region_avg_sales_cte AS ( - SELECT region, AVG(amount) AS avg_sales - FROM sales - GROUP BY region -) -UPDATE sales -SET amount = 0 -WHERE amount < ( - SELECT avg_sales - FROM region_avg_sales_cte AS cte - WHERE cte.region = sales.region -); -``` - -Let's assume we have another table called "store_details" that contains additional information about each store, such as the store's opening date and owner. - -```sql -CREATE TABLE store_details ( - storeid INTEGER, - store_name TEXT, - opening_date DATE, - owner TEXT -); - -INSERT INTO store_details VALUES (1, 'North York Store', '2022-01-01', 'John Doe'); -INSERT INTO store_details VALUES (12, 'Downtown Store', '2022-02-15', 'Jane Smith'); -INSERT INTO store_details VALUES (3, 'Markham Store', '2021-12-10', 'Michael Johnson'); -INSERT INTO store_details VALUES (9, 'Mississauga Store', '2022-03-20', 'Emma Brown'); -INSERT INTO store_details VALUES (5, 'Scarborough Store', '2022-04-05', 'David Lee'); -``` - -We want to delete all rows from the "store_details" table that correspond to stores with no sales records in the "sales" table: - -```sql -WITH stores_with_sales AS ( - SELECT DISTINCT storeid - FROM sales -) -DELETE FROM store_details -WHERE storeid NOT IN (SELECT storeid FROM stores_with_sales); -``` - -### Recursive CTEs - -First, we create a table to store employee data, including their ID, name, and manager ID. - -```sql -CREATE TABLE Employees ( - EmployeeID INT, - EmployeeName VARCHAR(100), - ManagerID INT -); -``` - -Next, we insert sample data into the table to represent a simple organizational structure. - -```sql -INSERT INTO Employees (EmployeeID, EmployeeName, ManagerID) VALUES -(1, 'Alice', NULL), -- Alice is the CEO -(2, 'Bob', 1), -- Bob reports to Alice -(3, 'Charlie', 1), -- Charlie reports to Alice -(4, 'David', 2), -- David reports to Bob -(5, 'Eve', 2), -- Eve reports to Bob -(6, 'Frank', 3); -- Frank reports to Charlie -``` - -Now, we use a recursive CTE to find the hierarchy of employees under a specific manager, say Alice (EmployeeID = 1). - -```sql -WITH RECURSIVE EmployeeHierarchy AS ( - -- Start with Alice (the CEO) - SELECT EmployeeID, EmployeeName, managerid, EmployeeName as LeaderName - FROM Employees - WHERE EmployeeID=1 - UNION ALL - -- Recursively find employees reporting to the current level - SELECT e.EmployeeID, e.EmployeeName, e.managerid, eh.EmployeeName - FROM Employees e - JOIN EmployeeHierarchy eh ON e.ManagerID = eh.EmployeeID -) -SELECT * FROM EmployeeHierarchy; -``` - -The output will list all employees in the hierarchy under Alice: - -```sql -┌─────────────────────────────────────────────────────────────────────────┐ -│ employeeid │ employeename │ managerid │ leadername │ -├─────────────────┼──────────────────┼─────────────────┼──────────────────┤ -│ 1 │ Alice │ NULL │ Alice │ -│ 2 │ Bob │ 1 │ Alice │ -│ 3 │ Charlie │ 1 │ Alice │ -│ 4 │ David │ 2 │ Bob │ -│ 5 │ Eve │ 2 │ Bob │ -│ 6 │ Frank │ 3 │ Charlie │ -└─────────────────────────────────────────────────────────────────────────┘ -``` \ No newline at end of file diff --git a/docs/en/guides/54-query/01-combining-data/_category_.json b/docs/en/guides/54-query/01-combining-data/_category_.json new file mode 100644 index 0000000000..bcf9b2c01b --- /dev/null +++ b/docs/en/guides/54-query/01-combining-data/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Combining Data", + "position": 2 +} \ No newline at end of file diff --git a/docs/en/guides/54-query/01-combining-data/cte.md b/docs/en/guides/54-query/01-combining-data/cte.md new file mode 100644 index 0000000000..7998639958 --- /dev/null +++ b/docs/en/guides/54-query/01-combining-data/cte.md @@ -0,0 +1,94 @@ +--- +title: Common Table Expressions (CTE) +--- + +CTEs break complex queries into simple, readable steps using `WITH`. + +## Quick Start + +```sql +-- Instead of complex nested queries +WITH high_earners AS ( + SELECT * FROM employees WHERE salary > 70000 +) +SELECT department, COUNT(*) +FROM high_earners +GROUP BY department; +``` + +**Result**: Clean, readable code that's easy to debug. + +## When to Use CTE + +**✅ Use CTE when:** +- Query has multiple steps +- You need the same subquery twice +- Query is hard to read + +**❌ Skip CTE when:** +- Simple one-step query +- Performance is critical + +## Three Essential Patterns + +### 1. Filter → Analyze +```sql +WITH filtered_data AS ( + SELECT * FROM sales WHERE date >= '2023-01-01' +) +SELECT product, SUM(amount) +FROM filtered_data +GROUP BY product; +``` + +### 2. Multiple Steps +```sql +WITH step1 AS ( + SELECT department, AVG(salary) as avg_sal FROM employees GROUP BY department +), +step2 AS ( + SELECT * FROM step1 WHERE avg_sal > 70000 +) +SELECT * FROM step2; +``` + +### 3. Use Same Data Twice +```sql +WITH dept_stats AS ( + SELECT department, AVG(salary) as avg_sal FROM employees GROUP BY department +) +SELECT d1.department, d1.avg_sal +FROM dept_stats d1 +JOIN dept_stats d2 ON d1.avg_sal > d2.avg_sal; +``` + +## Advanced: Recursive CTE + +Recursive CTEs solve problems where you need to repeatedly apply the same logic. Think of it like climbing stairs - you start at step 1, then keep going up one step at a time. + +```sql +-- Generate a sequence (useful for reports, testing, or filling gaps) +WITH RECURSIVE countdown AS ( + -- Base case: where we start + SELECT 10 as num, 'Starting countdown' as message + + UNION ALL + + -- Recursive case: what we do repeatedly + SELECT num - 1, CONCAT('Count: ', CAST(num - 1 AS VARCHAR)) + FROM countdown + WHERE num > 1 -- Stop condition: when to stop +) +SELECT num, message FROM countdown; +``` + +**Result**: Numbers from 10 down to 1 with messages. + +**Real-world example**: Generate missing months for a sales report +- Start: January 2024 +- Repeat: Add next month +- Stop: When we reach December 2024 + +**Key insight**: Recursion = Start somewhere + Repeat an action + Know when to stop. + +**That's it.** Start with simple CTEs, add complexity only when needed. \ No newline at end of file diff --git a/docs/en/guides/54-query/01-combining-data/index.md b/docs/en/guides/54-query/01-combining-data/index.md new file mode 100644 index 0000000000..e85749e608 --- /dev/null +++ b/docs/en/guides/54-query/01-combining-data/index.md @@ -0,0 +1,99 @@ +--- +title: Combining Data +--- + +Learn to combine data from multiple sources using JOINs, CTEs, and advanced query structures. + +## Core Concepts + +### [JOINs](./joins.md) +Connect data from multiple tables +```sql +-- Inner join (most common) +SELECT e.name, d.department_name +FROM employees e +JOIN departments d ON e.dept_id = d.id; +``` +**Covers**: Inner, Left, Right, Full Outer, Semi, Anti, and AsOf joins + +### [Common Table Expressions (CTEs)](./cte.md) +Structure complex queries with WITH clauses +```sql +-- Break complex logic into steps +WITH high_performers AS ( + SELECT * FROM employees WHERE rating > 4.0 +) +SELECT department, COUNT(*) +FROM high_performers +GROUP BY department; +``` +**Covers**: Basic CTEs, Recursive CTEs, Materialized CTEs + +## Advanced Combinations + +### Subqueries +```sql +-- Correlated subquery +SELECT name, salary, + (SELECT AVG(salary) FROM employees e2 + WHERE e2.department = e1.department) as dept_avg +FROM employees e1; + +-- EXISTS clause +SELECT * FROM customers c +WHERE EXISTS ( + SELECT 1 FROM orders o + WHERE o.customer_id = c.id +); +``` + +### Set Operations +```sql +-- Combine results from multiple queries +SELECT name FROM employees WHERE department = 'Sales' +UNION +SELECT name FROM contractors WHERE active = true; + +-- Other set operations +INTERSECT -- Common rows only +EXCEPT -- Rows in first query but not second +``` + +## Practical Patterns + +### Data Enrichment +```sql +-- Add lookup data to main table +WITH region_lookup AS ( + SELECT zip_code, region_name + FROM zip_regions +) +SELECT + c.customer_name, + c.zip_code, + r.region_name, + SUM(o.amount) as total_sales +FROM customers c +LEFT JOIN region_lookup r ON c.zip_code = r.zip_code +LEFT JOIN orders o ON c.id = o.customer_id +GROUP BY c.customer_name, c.zip_code, r.region_name; +``` + +### Hierarchical Data +```sql +-- Recursive CTE for organizational structure +WITH RECURSIVE org_chart AS ( + -- Base case: top-level managers + SELECT id, name, manager_id, 1 as level + FROM employees WHERE manager_id IS NULL + + UNION ALL + + -- Recursive case: add direct reports + SELECT e.id, e.name, e.manager_id, o.level + 1 + FROM employees e + JOIN org_chart o ON e.manager_id = o.id +) +SELECT * FROM org_chart ORDER BY level, name; +``` + diff --git a/docs/en/guides/54-query/01-combining-data/joins.md b/docs/en/guides/54-query/01-combining-data/joins.md new file mode 100644 index 0000000000..92771d9cc2 --- /dev/null +++ b/docs/en/guides/54-query/01-combining-data/joins.md @@ -0,0 +1,200 @@ +--- +title: JOINs +--- + +A JOIN combines rows from two tables to create a new combined row that can be used in the query. + +## Introduction + +JOINs are useful when data is split across related tables. For example, one table holds employee information, and another holds department details. + +Let's look at our sample data: + +**Employees Table:** +```sql +SELECT * FROM employees ORDER BY id; +``` + +| id | name | department | salary | +|----|---------|-------------|----------| +| 1 | Alice | Engineering | 75000.00 | +| 2 | Bob | Engineering | 80000.00 | +| 3 | Carol | Marketing | 65000.00 | +| 4 | David | Sales | 70000.00 | +| 5 | Eve | Marketing | 68000.00 | + +**Departments Table:** +```sql +SELECT * FROM departments ORDER BY id; +``` + +| id | name | budget | +|----|-------------|------------| +| 1 | Engineering | 1000000.00 | +| 2 | Marketing | 500000.00 | +| 3 | Sales | 750000.00 | + +The two tables share a common relationship: `employees.department` matches `departments.name`. JOINs let us combine this related data. + +## Types of JOINs + +### INNER JOIN + +An INNER JOIN pairs each row in one table with matching rows in the other table. + +```sql +-- Get employees with their department budgets +SELECT e.name, e.salary, d.name as department_name, d.budget +FROM employees e +INNER JOIN departments d ON e.department = d.name +ORDER BY e.name; +``` + +**Output:** +| name | salary | department_name | budget | +|-------|----------|----------------|------------| +| Alice | 75000.00 | Engineering | 1000000.00 | +| Bob | 80000.00 | Engineering | 1000000.00 | +| Carol | 65000.00 | Marketing | 500000.00 | +| David | 70000.00 | Sales | 750000.00 | +| Eve | 68000.00 | Marketing | 500000.00 | + +The output contains only rows where there's a match between `employees.department` and `departments.name`. + +### LEFT JOIN + +A LEFT JOIN returns all rows from the left table, with matching data from the right table. If there's no match, the right columns contain NULL. + +```sql +-- Get all employees, show department budget if available +SELECT e.name, e.department, d.budget +FROM employees e +LEFT JOIN departments d ON e.department = d.name +ORDER BY e.name; +``` + +**Output:** +| name | department | budget | +|-------|-------------|------------| +| Alice | Engineering | 500000.00 | +| Bob | Engineering | 500000.00 | +| Carol | Marketing | 200000.00 | +| David | Sales | 300000.00 | +| Eve | Marketing | 200000.00 | + +All employees are shown, even if their department doesn't exist (budget would be NULL). + +### RIGHT JOIN + +A RIGHT JOIN returns all rows from the right table, with matching data from the left table. + +```sql +-- Get all departments, show employees if any +SELECT d.name as department_name, e.name as employee_name +FROM employees e +RIGHT JOIN departments d ON e.department = d.name +ORDER BY d.name, e.name; +``` + +**Output:** +| department_name | employee_name | +|----------------|---------------| +| Engineering | Alice | +| Engineering | Bob | +| Marketing | Carol | +| Marketing | Eve | +| Sales | David | + +All departments are shown, even if they have no employees (employee_name would be NULL). + +### FULL JOIN + +A FULL JOIN returns all rows from both tables, matching where possible. + +```sql +-- Get all employees and all departments +SELECT e.name as employee_name, d.name as department_name +FROM employees e +FULL JOIN departments d ON e.department = d.name +ORDER BY e.name, d.name; +``` + +**Output:** +| employee_name | department_name | +|--------------|----------------| +| Alice | Engineering | +| Bob | Engineering | +| Carol | Marketing | +| David | Sales | +| Eve | Marketing | + +Shows all employees and departments, with NULLs where there's no match. + +### CROSS JOIN + +A CROSS JOIN creates a Cartesian product - every row from the first table combined with every row from the second table. + +**Warning**: This can create very large result sets. Use with caution. + +```sql +-- Every employee paired with every department (rarely useful) +SELECT e.name, d.name as department +FROM employees e +CROSS JOIN departments d +ORDER BY e.name, d.name; +``` + +**Output (partial - 15 rows total):** +| name | department | +|-------|-------------| +| Alice | Engineering | +| Alice | Marketing | +| Alice | Sales | +| Bob | Engineering | +| Bob | Marketing | +| Bob | Sales | +| Carol | Engineering | +| ... | ... | + +**Result**: 5 employees × 3 departments = 15 rows total. + +## JOIN Implementation + +### Using ON Clause (Recommended) + +```sql +SELECT e.name, d.budget +FROM employees e +JOIN departments d ON e.department = d.name; +``` + +### Using WHERE Clause (Legacy) + +```sql +SELECT e.name, d.budget +FROM employees e, departments d +WHERE e.department = d.name; +``` + +**Recommendation**: Use the ON clause syntax because it's clearer and handles outer joins correctly. + +## Multiple Table JOINs + +You can chain JOINs together to combine data from more than two tables: + +```sql +-- Employees with department budgets and project info (if projects table existed) +SELECT e.name, d.name as department, d.budget +FROM employees e +JOIN departments d ON e.department = d.name +JOIN projects p ON d.id = p.department_id +WHERE p.status = 'Active'; +``` + +## When to Use Each JOIN Type + +- **INNER JOIN**: When you only want matching records from both tables +- **LEFT JOIN**: When you want all records from the left table, matched where possible +- **RIGHT JOIN**: When you want all records from the right table, matched where possible +- **FULL JOIN**: When you want all records from both tables +- **CROSS JOIN**: Rarely used; only when you specifically need a Cartesian product diff --git a/docs/en/guides/54-query/01-combining-data/subqueries.md b/docs/en/guides/54-query/01-combining-data/subqueries.md new file mode 100644 index 0000000000..f55e36ca35 --- /dev/null +++ b/docs/en/guides/54-query/01-combining-data/subqueries.md @@ -0,0 +1,158 @@ +--- +title: Subqueries +--- + +A subquery is a query inside another query. Use subqueries to filter, compare, or compute values that depend on data from the main query. + +## Quick Start + +```sql +-- Find employees earning above department average +SELECT name, salary, department +FROM employees +WHERE salary > ( + SELECT AVG(salary) + FROM employees AS e2 + WHERE e2.department = employees.department +); +``` + +**Result**: Employees who earn more than their department's average salary. + +## Types of Subqueries + +### 1. Scalar Subqueries (Single Value) +```sql +-- Compare to overall average +SELECT name, salary, + (SELECT AVG(salary) FROM employees) AS company_avg +FROM employees +WHERE salary > (SELECT AVG(salary) FROM employees); +``` + +**Returns**: Single value (one row, one column). + +### 2. Table Subqueries (Multiple Rows) +```sql +-- Filter by department conditions +SELECT name, department +FROM employees +WHERE department IN ( + SELECT name + FROM departments + WHERE budget > 500000 +); +``` + +**Returns**: Multiple rows, used with IN, EXISTS, ANY, ALL. + +## Subquery Placement + +### WHERE Clause - Filtering +```sql +-- Employees in high-budget departments +SELECT name, salary +FROM employees +WHERE department IN ( + SELECT name FROM departments WHERE budget > 500000 +); +``` + +### FROM Clause - Data Source +```sql +-- Analyze high earners by department +SELECT department, AVG(salary) as avg_salary +FROM ( + SELECT * FROM employees WHERE salary > 70000 +) AS high_earners +GROUP BY department; +``` + +### SELECT Clause - Computed Columns +```sql +-- Show salary vs department average +SELECT name, salary, + (SELECT AVG(salary) + FROM employees e2 + WHERE e2.department = e1.department) as dept_avg +FROM employees e1; +``` + +## Correlated vs Uncorrelated + +### Uncorrelated - Independent +```sql +-- Same subquery runs once +SELECT name, salary +FROM employees +WHERE salary > (SELECT AVG(salary) FROM employees); +``` + +**Performance**: Subquery executes once, result reused. + +### Correlated - Dependent +```sql +-- Subquery runs for each main row +SELECT name, salary, department +FROM employees e1 +WHERE salary > ( + SELECT AVG(salary) + FROM employees e2 + WHERE e2.department = e1.department +); +``` + +**Performance**: Subquery executes for each outer row. + +## Common Patterns + +### EXISTS - Check for Related Data +```sql +-- Employees who have projects +SELECT name, department +FROM employees e +WHERE EXISTS ( + SELECT 1 FROM projects p WHERE p.employee_id = e.id +); +``` + +### NOT EXISTS - Check for Missing Data +```sql +-- Employees without projects +SELECT name, department +FROM employees e +WHERE NOT EXISTS ( + SELECT 1 FROM projects p WHERE p.employee_id = e.id +); +``` + +### ANY/ALL - Multiple Comparisons +```sql +-- Employees earning more than ANY marketing employee +SELECT name, salary +FROM employees +WHERE salary > ANY ( + SELECT salary FROM employees WHERE department = 'Marketing' +); + +-- Employees earning more than ALL marketing employees +SELECT name, salary +FROM employees +WHERE salary > ALL ( + SELECT salary FROM employees WHERE department = 'Marketing' +); +``` + +## When to Use Subqueries vs JOINs + +**✅ Use Subqueries when:** +- Filtering based on aggregate conditions +- Checking existence/non-existence +- Need computed values in SELECT +- Logic is clearer as nested steps + +**✅ Use JOINs when:** +- Need columns from multiple tables +- Better performance for large datasets +- Combining data rather than filtering + diff --git a/docs/en/guides/54-query/02-advanced/_category_.json b/docs/en/guides/54-query/02-advanced/_category_.json new file mode 100644 index 0000000000..dee56ac072 --- /dev/null +++ b/docs/en/guides/54-query/02-advanced/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Advanced Features", + "position": 3 +} \ No newline at end of file diff --git a/docs/en/guides/54-query/04-external-function.md b/docs/en/guides/54-query/02-advanced/external-function.md similarity index 99% rename from docs/en/guides/54-query/04-external-function.md rename to docs/en/guides/54-query/02-advanced/external-function.md index b30db66067..2037940e1f 100644 --- a/docs/en/guides/54-query/04-external-function.md +++ b/docs/en/guides/54-query/02-advanced/external-function.md @@ -1,6 +1,7 @@ --- title: "External Functions in Databend Cloud" sidebar_label: "External Function" +sidebar_position: 2 --- External functions in Databend allow you to define custom operations for processing data using external servers written in programming languages like Python. These functions enable you to extend Databend's capabilities by integrating custom logic, leveraging external libraries, and handling complex processing tasks. Key features of external functions include: diff --git a/docs/en/guides/54-query/02-advanced/index.md b/docs/en/guides/54-query/02-advanced/index.md new file mode 100644 index 0000000000..33e5c04b9e --- /dev/null +++ b/docs/en/guides/54-query/02-advanced/index.md @@ -0,0 +1,59 @@ +--- +title: Advanced Features +--- + +Extend Databend's capabilities with custom functions, external integrations, stored procedures, and sequences. + +## [User-Defined Functions (UDF)](./udf.md) +Create reusable custom functions in SQL, Python, or JavaScript +```sql +-- SQL function +CREATE FUNCTION calculate_bonus(salary FLOAT, rating FLOAT) +RETURNS FLOAT AS $$ salary * rating * 0.1 $$; + +-- Use it +SELECT name, calculate_bonus(salary, performance_rating) as bonus +FROM employees; +``` + +## [External Functions](./external-function.md) +Integrate with external services and APIs +```sql +-- Call external ML model +SELECT customer_id, predict_churn(age, tenure, usage) as churn_risk +FROM customers; +``` + +## [Stored Procedures](./stored-procedure.md) +Multi-step operations with control flow +```sql +-- Complex business logic +CALL monthly_report_generation('2023-12'); +``` + +## [Sequences](./sequences.md) +Generate unique identifiers and sequential values +```sql +-- Create auto-incrementing ID +CREATE SEQUENCE user_id_seq; +INSERT INTO users VALUES (NEXTVAL(user_id_seq), 'John Doe'); +``` + + +## Best Practices + +### Function Design +- **Keep functions pure** - Same input always produces same output +- **Handle NULLs** - Consider NULL input handling +- **Use appropriate types** - Match input/output types to usage +- **Document well** - Clear parameter and return descriptions + +### Performance Considerations +- **UDF overhead** - SQL functions are fastest, Python/JS slower +- **Batch operations** - Process data in batches when possible +- **Resource limits** - Monitor memory usage for complex functions + +### Security +- **Validate inputs** - Check parameters in UDF code +- **Least privilege** - External connections should have minimal permissions +- **Audit usage** - Monitor UDF and external function calls \ No newline at end of file diff --git a/docs/en/guides/54-query/02-advanced/sequences.md b/docs/en/guides/54-query/02-advanced/sequences.md new file mode 100644 index 0000000000..f2b3f8fc40 --- /dev/null +++ b/docs/en/guides/54-query/02-advanced/sequences.md @@ -0,0 +1,134 @@ +--- +title: Sequences +sidebar_position: 4 +--- + +Sequences generate unique, sequential numeric values, commonly used for primary keys and auto-incrementing identifiers. + +## What are Sequences? + +Sequences provide a thread-safe way to generate unique numbers across concurrent operations. Unlike auto-increment columns, sequences are database objects that can be used across multiple tables. + +## Creating Sequences + +### Basic Sequence +```sql +-- Create a simple sequence +CREATE SEQUENCE user_id_seq; + +-- Use the sequence +SELECT NEXTVAL(user_id_seq); -- Returns: 1 +SELECT NEXTVAL(user_id_seq); -- Returns: 2 +SELECT NEXTVAL(user_id_seq); -- Returns: 3 +``` + +### Sequence with Options +```sql +-- Create sequence with custom settings +CREATE SEQUENCE order_id_seq + START = 1000 + INCREMENT = 1 + MINVALUE = 1000 + MAXVALUE = 999999999 + CACHE = 50; +``` + +## Using Sequences + +### In INSERT Statements +```sql +-- Create table with sequence-generated IDs +CREATE TABLE users ( + id BIGINT, + name VARCHAR(100), + email VARCHAR(100) +); + +-- Insert with sequence value +INSERT INTO users VALUES + (NEXTVAL(user_id_seq), 'John Doe', 'john@example.com'), + (NEXTVAL(user_id_seq), 'Jane Smith', 'jane@example.com'); +``` + +### Current and Next Values +```sql +-- Get next value (advances sequence) +SELECT NEXTVAL(user_id_seq); + +-- Get current value (doesn't advance sequence) +SELECT CURRVAL(user_id_seq); +``` + +## Sequence Functions + +| Function | Description | Example | +|----------|-------------|---------| +| `NEXTVAL(seq)` | Get next value and advance sequence | `NEXTVAL(user_id_seq)` | +| `CURRVAL(seq)` | Get current value without advancing | `CURRVAL(user_id_seq)` | + +## Managing Sequences + +### View Sequences +```sql +-- Show all sequences +SHOW SEQUENCES; + +-- Describe specific sequence +DESC SEQUENCE user_id_seq; +``` + +### Modify Sequences +```sql +-- Reset sequence to specific value +ALTER SEQUENCE user_id_seq RESTART = 5000; + +-- Change increment value +ALTER SEQUENCE user_id_seq INCREMENT = 10; +``` + +### Drop Sequences +```sql +-- Remove sequence +DROP SEQUENCE user_id_seq; +``` + +## Best Practices + +### Performance Optimization +- **Use CACHE** - Improves performance by pre-allocating values +- **Appropriate INCREMENT** - Match your application needs +- **Monitor gaps** - Cached values may create gaps if server restarts + +### Common Patterns +```sql +-- Auto-incrementing primary key pattern +CREATE SEQUENCE pk_seq START = 1 INCREMENT = 1 CACHE = 100; + +CREATE TABLE products ( + id BIGINT DEFAULT NEXTVAL(pk_seq), + name VARCHAR(100), + price DECIMAL(10,2) +); + +-- Order number pattern (readable IDs) +CREATE SEQUENCE order_seq START = 10000 INCREMENT = 1; + +INSERT INTO orders VALUES + (NEXTVAL(order_seq), customer_id, order_date); +``` + +## vs Auto-Increment Columns + +| Feature | Sequences | Auto-Increment | +|---------|-----------|----------------| +| **Reusability** | ✅ Multiple tables | ❌ Single column | +| **Control** | ✅ Full control | ❌ Limited options | +| **Gaps** | ✅ Predictable | ❌ May have gaps | +| **Performance** | ✅ Cacheable | ✅ Optimized | + +## Common Use Cases + +1. **Primary Keys** - Unique identifiers across tables +2. **Order Numbers** - Sequential business identifiers +3. **Version Numbers** - Document or record versioning +4. **Batch IDs** - Processing batch identification \ No newline at end of file diff --git a/docs/en/guides/54-query/08-stored-procedure.md b/docs/en/guides/54-query/02-advanced/stored-procedure.md similarity index 99% rename from docs/en/guides/54-query/08-stored-procedure.md rename to docs/en/guides/54-query/02-advanced/stored-procedure.md index d9a0bf19da..2065433795 100644 --- a/docs/en/guides/54-query/08-stored-procedure.md +++ b/docs/en/guides/54-query/02-advanced/stored-procedure.md @@ -1,5 +1,6 @@ --- title: Stored Procedure +sidebar_position: 3 --- A stored procedure is a set of executable commands or logic blocks stored within the database, written in SQL or other programming languages, designed to be reusable for efficiently performing specific tasks or operations. diff --git a/docs/en/guides/54-query/03-udf.md b/docs/en/guides/54-query/02-advanced/udf.md similarity index 99% rename from docs/en/guides/54-query/03-udf.md rename to docs/en/guides/54-query/02-advanced/udf.md index fb4b5f264f..f8cbed380c 100644 --- a/docs/en/guides/54-query/03-udf.md +++ b/docs/en/guides/54-query/02-advanced/udf.md @@ -1,5 +1,6 @@ --- title: User-Defined Function +sidebar_position: 1 --- # User-Defined Functions (UDFs) in Databend diff --git a/docs/en/guides/54-query/02-join.md b/docs/en/guides/54-query/02-join.md deleted file mode 100644 index 19436f6a61..0000000000 --- a/docs/en/guides/54-query/02-join.md +++ /dev/null @@ -1,519 +0,0 @@ ---- -title: JOIN ---- - -## Supported Join Types - -The *join* combines columns from two or more tables into a single result set. Databend supports the following *join* types: - -* [Inner Join](#inner-join) -* [Natural Join](#natural-join) -* [Cross Join](#cross-join) -* [Left Join](#left-join) -* [Right Join](#right-join) -* [Full Outer Join](#full-outer-join) -* [Left / Right Semi-Join](#left--right-semi-join) -* [Left / Right Anti-Join](#left--right-anti-join) -* [AsOf Join](#asof-join) - -## Example Tables - -Unless explicitly specified, the join examples on this page are created based on the following tables: - -Table "vip_info": This table stores the VIP client information. - -| Client_ID | Region | -|---------------|-------------| -| 101 | Toronto | -| 102 | Quebec | -| 103 | Vancouver | - -Table "purchase_records": This table lists the purchase records for all the clients. - -| Client_ID | Item | QTY | -|---------------|-------------|-----------| -| 100 | Croissant | 2,000 | -| 102 | Donut | 3,000 | -| 103 | Coffee | 6,000 | -| 106 | Soda | 4,000 | - -Table "gift": This table lists the gift options for the VIP clients. - -| Gift | -|-------------| -| Croissant | -| Donut | -| Coffee | -| Soda | - -Table trades: This table records trade transactions for various stock symbols. - -| Symbol | Time | Price | -| ------ |--------|--------| -| AAPL | 100003 | 101 | -| AAPL | 100007 | 103 | -| MSFT | 100002 | 99 | -| TSLA | 100010 | 200 | - - -Table quotes: This table provides quote snapshots (bid/ask) for symbols at various timestamps. - -| Symbol | Time | Bid | Ask | -| ------ |--------| --- | --- | -| AAPL | 100000 | 99 | 102 | -| AAPL | 100005 | 100 | 104 | -| MSFT | 100001 | 98 | 101 | -| NVDA | 100006 | 150 | 155 | - - -## Inner Join - -The *inner join* returns the rows that meet the join conditions in the result set. - -### Syntax - -```sql -SELECT select_list -FROM table_a - [INNER] JOIN table_b - ON join_condition_1 - [[INNER] JOIN table_c - ON join_condition_2]... -``` - -:::tip -The keyword INNER is optional. -::: - -When you join two tables on a common column with the equal operator, you can use the keyword USING to simplify the syntax. - -```sql -SELECT select_list -FROM table_a - JOIN table_b - USING join_column_1 - [JOIN table_c - USING join_column_2]... -``` - -### Examples - -The following example returns the purchase records of the VIP clients: - -```sql -SELECT purchase_records.client_id, - purchase_records.item, - purchase_records.qty -FROM vip_info - INNER JOIN purchase_records - ON vip_info.client_id = purchase_records.client_id; -``` - -For the definitions of the tables in the example, see [Example Tables](#example-tables). - -Output: - -```sql -|102|Donut|3000 -|103|Coffee|6000 -``` - -## Natural Join - -The *natural join* joins two tables based on all columns in the two tables that have the same name. - -### Syntax - -```sql -SELECT select_list -FROM table_a - NATURAL JOIN table_b - [NATURAL JOIN table_c]... -``` - -### Examples - -The following example returns the purchase records of the VIP clients: - -```sql -SELECT purchase_records.client_id, - purchase_records.item, - purchase_records.qty -FROM vip_info - NATURAL JOIN purchase_records; -``` - -For the definitions of the tables in the example, see [Example Tables](#example-tables). - -Output: - -```sql -|102|Donut|3,000 -|103|Coffee|6,000 -``` - -## Cross Join - -The *cross join* returns a result set that includes each row from the first table joined with each row from the second table. - -### Syntax - -```sql -SELECT select_list -FROM table_a - CROSS JOIN table_b -``` - -### Examples - -The following example returns a result set that assigns each gift option to each VIP client: - -```sql -SELECT * -FROM vip_info - CROSS JOIN gift; -``` - -For the definitions of the tables in the example, see [Example Tables](#example-tables). - -Output: - -```sql -101|Toronto|Croissant -101|Toronto|Donut -101|Toronto|Coffee -101|Toronto|Soda -102|Quebec|Croissant -102|Quebec|Donut -102|Quebec|Coffee -102|Quebec|Soda -103|Vancouver|Croissant -103|Vancouver|Donut -103|Vancouver|Coffee -103|Vancouver|Soda -``` - -## Left Join - -The *left join* returns all records from the left table, and the matching records from the right table. The result is NULL records from the right side, if there is no match. - -### Syntax - -```sql -SELECT select_list -FROM table_a - LEFT [OUTER] JOIN table_b - ON join_condition -``` -:::tip -The keyword OUTER is optional. -::: - -### Examples - -The following example returns the purchase records of all VIP clients, the purchase records will be NULL if the VIP client has no purchases: - -```sql -SELECT vip_info.client_id, - purchase_records.item, - purchase_records.qty -FROM vip_info - LEFT JOIN purchase_records - ON vip_info.client_id = purchase_records.client_id; -``` - -For the definitions of the tables in the example, see [Example Tables](#example-tables). - -Output: - -```sql -|101|NULL|NULL -|102|Donut|3000 -|103|Coffee|6000 -``` - -## Right Join - -The *right join* returns all records from the right table, and the matching records from the left table. The result is NULL records from the left side, if there is no match. - -### Syntax - -```sql -SELECT select_list -FROM table_a - RIGHT [OUTER] JOIN table_b - ON join_condition -``` - -:::tip -The keyword OUTER is optional. -::: - -### Examples - -Imagine we have the following tables: - -The following example returns all vip_info of all purchase_records, the vip_info will be NULL if purchase_record does not have the corresponding vip_info. - -```sql -SELECT vip_info.client_id, - vip_info.region -FROM vip_info - RIGHT JOIN purchase_records - ON vip_info.client_id = purchase_records.client_id; -``` - -For the definitions of the tables in the example, see [Example Tables](#example-tables). - -Output: - -```sql -NULL|NULL -102|Quebec -103|Vancouver -NULL|NULL -``` - -## Full Outer Join - -The *full outer join* returns all rows from both tables, matching up the rows wherever a match can be made and placing NULLs in the places where no matching row exists. - -### Syntax - -```sql -SELECT select_list -FROM table_a - FULL OUTER JOIN table_b - ON join_condition -``` - -:::tip -The keyword OUTER is optional. -::: - -### Examples - -The following example returns all matched and unmatched rows from both tables: - -```sql -SELECT vip_info.region, - purchase_records.item -FROM vip_info - FULL OUTER JOIN purchase_records - ON vip_info.client_id = purchase_records.client_id; -``` - -For the definitions of the tables in the example, see [Example Tables](#example-tables). - -Output: - -```sql -Toronto|NULL -Quebec|Donut -Vancouver|Coffee -NULL|Croissant -NULL|Soda -``` - -## Left / Right Semi Join - -The *left semi join* returns rows from the left table that have a matching row in the right table. The *right semi join* returns rows from the right table that have a matching row in the left table. - -### Syntax - -```sql --- Left Semi Join - -SELECT select_list -FROM table_a - LEFT SEMI JOIN table_b - ON join_condition - --- Right Semi Join - -SELECT select_list -FROM table_a - RIGHT SEMI JOIN table_b - ON join_condition -``` - -### Examples - -The following example returns the VIP clients (Client_ID & Region) who have a purchase record: - -```sql -SELECT * -FROM vip_info - LEFT SEMI JOIN purchase_records - ON vip_info.client_id = purchase_records.client_id; -``` - -For the definitions of the tables in the example, see [Example Tables](#example-tables). - -Output: - -```sql -102|Quebec -103|Vancouver -``` - -The following example returns the purchase records (Client_ID, Item, and QTY) of the VIP clients: - -```sql -SELECT * -FROM vip_info - RIGHT SEMI JOIN purchase_records - ON vip_info.client_id = purchase_records.client_id; -``` - -For the definitions of the tables in the example, see [Example Tables](#example-tables). - -Output: - -```sql -|102|Donut|3000 -|103|Coffee|6000 -``` - -## Left / Right Anti Join - -The *left anti join* returns rows from the left table that have NO matching row in the right table. The *right anti join* returns rows from the right table that have NO matching row in the left table. - -### Syntax - -```sql --- Left Anti Join - -SELECT select_list -FROM table_a - LEFT ANTI JOIN table_b - ON join_condition - --- Right Anti Join - -SELECT select_list -FROM table_a - RIGHT ANTI JOIN table_b - ON join_condition -``` - -### Examples - -The following example returns the VIP clients (Client_ID & Region) who have NO purchase records: - -```sql -SELECT * -FROM vip_info - LEFT ANTI JOIN purchase_records - ON vip_info.client_id = purchase_records.client_id; -``` - -For the definitions of the tables in the example, see [Example Tables](#example-tables). - -Output: - -```sql -101|Toronto -``` - -The following example returns the purchase records (Client_ID, Item, and QTY) of non-VIP clients: - -```sql -SELECT * -FROM vip_info - RIGHT ANTI JOIN purchase_records - ON vip_info.client_id = purchase_records.client_id; -``` - -For the definitions of the tables in the example, see [Example Tables](#example-tables). - -Output: - -```sql -|100|Croissant|2000 -|106|Soda|4000 -``` - -## Asof Join - -An Asof Join (Approximate Sort-Merge Join) returns rows from the left table matched with the most recent row in the -right table whose timestamp is less than or equal to the left's. It is commonly used in time series data to attach the -latest contextual information, such as quotes, status, or sensor readings. - -Unlike typical equi-joins, the join condition is based on inequality (usually `<=`) and may optionally include additional -equality conditions. - -### Syntax - -```sql -SELECT select_list -FROM table_a ASOF - JOIN table_b - ON table_a.time >= table_b.time - [ AND table_a.key = table_b.key] -``` - -### Examples - -The following example joins a trade record with the latest quote before or at the trade time for the same symbol: - -```sql -SELECT * -FROM trades - ASOF JOIN quotes - ON trades.symbol = quotes.symbol - AND trades.time >= quotes.time; -``` - -For the definitions of the tables in the example, see [Example Tables](#example-tables). - -Output: - -```sql -│ AAPL │ 100003 │ 101 │ AAPL │ 100000 │ 99 │ 102 │ -│ AAPL │ 100007 │ 103 │ AAPL │ 100005 │ 100 │ 104 │ -│ MSFT │ 100002 │ 99 │ MSFT │ 100001 │ 98 │ 101 │ - -``` - -The following example performs an ASOF LEFT JOIN, returning all trade records along with the latest quote before or at the trade time for the same symbol, if such a quote exists. If no matching quote exists, the quote fields will be NULL. - -```sql -SELECT * -FROM trades - ASOF LEFT JOIN quotes - ON trades.symbol = quotes.symbol - AND trades.time >= quotes.time; -``` - -For the definitions of the tables in the example, see [Example Tables](#example-tables). - -Output: - -```sql -│ AAPL │ 100003 │ 101 │ AAPL │ 100000 │ 99 │ 102 │ -│ MSFT │ 100002 │ 99 │ MSFT │ 100001 │ 98 │ 101 │ -│ AAPL │ 100007 │ 103 │ AAPL │ 100005 │ 100 │ 104 │ -│ TSLA │ 100010 │ 200 │ NULL │ NULL │ NULL │ NULL │ -``` - -The following example performs an ASOF RIGHT JOIN, returning all quote records along with the latest trade after or at the quote time for the same symbol, if such a trade exists. If no matching trade exists, the trade fields will be NULL. - -```sql -SELECT * -FROM trades - ASOF RIGHT JOIN quotes - ON trades.symbol = quotes.symbol - AND trades.time >= quotes.time; -``` - -For the definitions of the tables in the example, see [Example Tables](#example-tables). - -Output: - -```sql -│ AAPL │ 100003 │ 101 │ AAPL │ 100000 │ 99 │ 102 │ -│ AAPL │ 100007 │ 103 │ AAPL │ 100005 │ 100 │ 104 │ -│ MSFT │ 100002 │ 99 │ MSFT │ 100001 │ 98 │ 101 │ -│ NULL │ NULL │ NULL │ NVDA │ 100006 │ 150 │ 155 │ -``` diff --git a/docs/en/guides/54-query/02-sequences.md b/docs/en/guides/54-query/02-sequences.md deleted file mode 100644 index e64452e022..0000000000 --- a/docs/en/guides/54-query/02-sequences.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -title: 'Using Sequence' -sidebar_label: 'Sequence' ---- - -import ComponentContent from '../../sql-reference/10-sql-commands/00-ddl/04-sequence/create-sequence.md'; - - \ No newline at end of file diff --git a/docs/en/guides/54-query/03-optimization/_category_.json b/docs/en/guides/54-query/03-optimization/_category_.json new file mode 100644 index 0000000000..8a21b6cede --- /dev/null +++ b/docs/en/guides/54-query/03-optimization/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Query Optimization", + "position": 4 +} \ No newline at end of file diff --git a/docs/en/guides/54-query/03-optimization/index.md b/docs/en/guides/54-query/03-optimization/index.md new file mode 100644 index 0000000000..bce7dc387e --- /dev/null +++ b/docs/en/guides/54-query/03-optimization/index.md @@ -0,0 +1,201 @@ +--- +title: Query Optimization +--- + +Analyze and improve query performance with profiling tools, execution plans, and optimization techniques. + +## Performance Analysis Tools + +### [Query Profile](./query-profile.md) +Visual execution plan analysis in Databend Cloud +- **Access**: Monitor → SQL History → Query Profile tab +- **Shows**: Execution nodes, timing, resource usage +- **Use for**: Identifying bottlenecks, understanding query execution + +### [Query Hash](./query-hash.md) +Unique query fingerprinting for performance tracking +```sql +-- Get query fingerprint +SELECT query_hash('SELECT * FROM table WHERE id = ?'); +``` + +## Query Optimization Fundamentals + +### Execution Plan Analysis +```sql +-- View query execution plan +EXPLAIN SELECT * FROM orders o +JOIN customers c ON o.customer_id = c.id +WHERE o.order_date >= '2023-01-01'; +``` + +**Look for:** +- **Table scans** vs **index usage** +- **Join algorithms** (hash, merge, nested loop) +- **Filter pushdown** effectiveness +- **Resource consumption** estimates + +### Index Strategy +```sql +-- Create indexes for common query patterns +CREATE INDEX idx_orders_date ON orders(order_date); +CREATE INDEX idx_orders_customer ON orders(customer_id); +``` + +**Index Guidelines:** +- Index WHERE clause columns +- Index JOIN columns on both sides +- Consider composite indexes for multi-column filters +- Monitor index usage statistics + +## Performance Optimization Techniques + +### Query Rewriting +```sql +-- ❌ Inefficient: Function on column prevents index usage +SELECT * FROM orders WHERE YEAR(order_date) = 2023; + +-- ✅ Optimized: Range condition can use index +SELECT * FROM orders +WHERE order_date >= '2023-01-01' + AND order_date < '2024-01-01'; +``` + +### Filter Pushdown +```sql +-- ❌ Filter after join +SELECT * FROM ( + SELECT o.*, c.name + FROM orders o JOIN customers c ON o.customer_id = c.id +) WHERE order_date >= '2023-01-01'; + +-- ✅ Filter before join +SELECT o.*, c.name +FROM orders o +JOIN customers c ON o.customer_id = c.id +WHERE o.order_date >= '2023-01-01'; +``` + +### Aggregation Optimization +```sql +-- Use appropriate GROUP BY extensions +SELECT + region, + product_category, + COUNT(*) as sales_count, + SUM(amount) as total_sales +FROM sales +GROUP BY CUBE(region, product_category); +``` + +## Common Performance Issues + +### Issue 1: Large Result Sets +```sql +-- ❌ Problem: No limit on large table +SELECT * FROM events ORDER BY timestamp DESC; + +-- ✅ Solution: Always limit exploratory queries +SELECT * FROM events ORDER BY timestamp DESC LIMIT 1000; +``` + +### Issue 2: Inefficient Joins +```sql +-- ❌ Problem: Cartesian product +SELECT * FROM table1, table2 WHERE condition; + +-- ✅ Solution: Explicit join with proper conditions +SELECT * FROM table1 t1 +INNER JOIN table2 t2 ON t1.id = t2.foreign_id +WHERE condition; +``` + +### Issue 3: Unnecessary Complexity +```sql +-- ❌ Problem: Nested subqueries +SELECT * FROM ( + SELECT * FROM ( + SELECT col1, col2 FROM table WHERE condition1 + ) WHERE condition2 +) WHERE condition3; + +-- ✅ Solution: Combine conditions +SELECT col1, col2 FROM table +WHERE condition1 AND condition2 AND condition3; +``` + +## Monitoring and Metrics + +### Key Performance Indicators +- **Query execution time** +- **Rows scanned vs rows returned** +- **Memory usage** +- **CPU utilization** +- **I/O operations** + +### Performance Monitoring Query +```sql +-- Find slow queries from query history +SELECT + query_text, + query_duration_ms, + scan_bytes, + result_bytes, + memory_usage +FROM system.query_log +WHERE query_duration_ms > 10000 -- Queries over 10 seconds +ORDER BY query_duration_ms DESC +LIMIT 10; +``` + +## Optimization Checklist + +### Query Design +- [ ] Use appropriate WHERE conditions +- [ ] Minimize data scanned with column selection +- [ ] Apply filters before joins +- [ ] Use proper join types +- [ ] Limit result sets appropriately + +### Indexing +- [ ] Index frequently filtered columns +- [ ] Index join columns +- [ ] Remove unused indexes +- [ ] Monitor index effectiveness + +### Schema Design +- [ ] Choose appropriate data types +- [ ] Normalize appropriately (avoid over-normalization) +- [ ] Consider partitioning for large tables +- [ ] Use clustering keys for sort optimization + +## Advanced Optimization + +### Materialized Views +```sql +-- Pre-compute expensive aggregations +CREATE MATERIALIZED VIEW daily_sales AS +SELECT + DATE(order_time) as order_date, + product_id, + COUNT(*) as order_count, + SUM(amount) as total_sales +FROM orders +GROUP BY DATE(order_time), product_id; +``` + +### Query Hints +```sql +-- Force specific join algorithm when needed +SELECT /*+ USE_HASH_JOIN */ * +FROM large_table l +JOIN small_table s ON l.id = s.foreign_id; +``` + +## Best Practices Summary + +1. **Measure first** - Use Query Profile to identify bottlenecks +2. **Index strategically** - Cover your query patterns +3. **Filter early** - Apply WHERE conditions as soon as possible +4. **Limit appropriately** - Don't fetch more data than needed +5. **Monitor continuously** - Track query performance over time \ No newline at end of file diff --git a/docs/en/guides/54-query/06-query-hash.md b/docs/en/guides/54-query/03-optimization/query-hash.md similarity index 100% rename from docs/en/guides/54-query/06-query-hash.md rename to docs/en/guides/54-query/03-optimization/query-hash.md diff --git a/docs/en/guides/54-query/05-query-profile.md b/docs/en/guides/54-query/03-optimization/query-profile.md similarity index 100% rename from docs/en/guides/54-query/05-query-profile.md rename to docs/en/guides/54-query/03-optimization/query-profile.md diff --git a/docs/en/guides/54-query/07-dictionary.md b/docs/en/guides/54-query/07-dictionary.md deleted file mode 100644 index ccc34eab5c..0000000000 --- a/docs/en/guides/54-query/07-dictionary.md +++ /dev/null @@ -1,113 +0,0 @@ ---- -title: Dictionary ---- - -Databend's dictionary feature provides an efficient way to integrate and query data from [Supported External Sources](#supported-external-sources) directly within Databend. By acting as an in-memory key-value store, the dictionary enables rapid access to external data without the need for complex data pipelines or traditional ETL processes. - -## How the Dictionary Works - -In Databend, you create a dictionary by defining its structure and specifying an external source. When queried, Databend fetches the required data from the external source. You can use the [DICT_GET](/sql/sql-functions/dictionary-functions/dict-get) function to retrieve values efficiently, ensuring that queries always reflect the latest data. - -## Supported External Sources - -Databend currently supports MySQL and Redis as external sources for dictionaries. - -## Tutorials - -- [Accessing MySQL and Redis with Dictionaries](/tutorials/integrate/access-mysql-and-redis) - - > In this tutorial, we’ll guide you through accessing MySQL and Redis data using dictionaries in Databend. You’ll learn how to create dictionaries that map to these external data sources, enabling seamless data querying and integration. - -## Usage Example - -The following example demonstrates how to integrate MySQL with Databend by using dictionaries, allowing you to query data stored in MySQL directly from Databend. This process involves creating a table in MySQL, setting up a corresponding table in Databend, creating a dictionary to map the data, and using the [DICT_GET](/sql/sql-functions/dictionary-functions/dict-get) function to retrieve values from the dictionary in your queries. - -### Step 1: Create a Table in MySQL - -First, create a table in your local MySQL database. For this example, we'll create a database named `dict` and a table named `orders`. - -```sql -CREATE DATABASE dict; -USE dict; - -CREATE TABLE orders ( - order_id INT PRIMARY KEY, - customer_name VARCHAR(100), - order_total INT); - -INSERT INTO orders (order_id, customer_name, order_total) VALUES -(1, 'John Doe', 250), -(2, 'Jane Smith', 175), -(3, 'Alice Johnson', 300); -``` - -### Step 2: Create a Table in Databend - -Next, create a corresponding table in Databend that includes the `order_id` and `customer_name`, but will query the `order_total` from the dictionary. - -```sql -CREATE TABLE orders ( - order_id INT, - customer_name VARCHAR -); - -INSERT INTO orders (order_id, customer_name) VALUES -(1, 'John Doe'), -(2, 'Jane Smith'), -(3, 'Alice Johnson'); -``` - -### Step 3: Create a Dictionary in Databend - -Now, create a dictionary in Databend that references the MySQL `orders` table. - -```sql -CREATE DICTIONARY order_dict -( - order_id INT, - customer_name STRING, - order_total INT -) -PRIMARY KEY order_id -SOURCE(MYSQL( - host='mysql' - port='3306' - username='root' - password='admin' - db='dict' - table='orders' -)); -``` - -### Step 4: Use `DICT_GET` in Queries - -Now you can use the [DICT_GET](/sql/sql-functions/dictionary-functions/dict-get) function in combination with a query on the Databend table. Here are a few examples: - -To retrieve order total for a specific order ID: - -```sql -SELECT DICT_GET(order_dict, 'order_total', 1); - --[ RECORD 1 ]----------------------------------- -dict_get(default.order_dict, 'order_total', 1): 250 -``` - -To retrieve customer name and order total for all orders: - -```sql -SELECT - order_id, - customer_name, - dict_get(order_dict, 'order_total', order_id) AS order_total -FROM - orders; - -┌──────────────────────────────────────────────────────┐ -│ order_id │ customer_name │ order_total │ -│ Nullable(Int32) │ Nullable(String) │ Nullable(Int32) │ -├─────────────────┼──────────────────┼─────────────────┤ -│ 1 │ John Doe │ 250 │ -│ 2 │ Jane Smith │ 175 │ -│ 3 │ Alice Johnson │ 300 │ -└──────────────────────────────────────────────────────┘ -``` diff --git a/docs/en/guides/54-query/index.md b/docs/en/guides/54-query/index.md index 67ea337441..d358164e51 100644 --- a/docs/en/guides/54-query/index.md +++ b/docs/en/guides/54-query/index.md @@ -2,47 +2,123 @@ title: Query Data in Databend --- -Databend supports standard SQL with ANSI SQL:1999 and SQL:2003 analytic extensions. This section covers query techniques, optimization tools, and advanced features for efficient data processing. +Databend supports standard SQL with ANSI SQL:2003 analytics extensions. This guide covers essential query techniques from basic to advanced, organized by learning path for optimal understanding. -## Core Query Features +## Learning Path -| Feature | Description | Key Benefits | -|---------|-------------|--------------| -| [**Common Table Expressions (CTE)**](00-cte.md) | Define named temporary result sets with WITH clause | Improved query readability, reusable subqueries | -| [**JOIN**](02-join.md) | Combine data from multiple tables | Support for Inner, Outer, Cross, Semi, and Anti joins | -| [**GROUP BY**](01-groupby/index.md) | Group and aggregate data with extensions | CUBE, ROLLUP, and GROUPING SETS support | -| [**Sequence**](02-sequences.md) | Generate sequential numeric values | Auto-incrementing identifiers and counters | +**📚 New to SQL?** Start with [Basic Queries](./00-basics/index.md) +**🔗 Joining data?** Go to [Combining Data](./01-combining-data/index.md) +**⚡ Need custom logic?** Check [Advanced Features](./02-advanced/index.md) +**🚀 Performance issues?** Visit [Query Optimization](./03-optimization/index.md) -## Advanced Query Capabilities +--- + +## 📚 [Basic Queries](./00-basics/index.md) + +Master fundamental SQL operations for data selection and aggregation. -| Feature | Type | Description | Use Cases | -|---------|------|-------------|-----------| -| [**User-Defined Functions**](03-udf.md) | Lambda & Embedded | Custom operations with Python, JavaScript, WebAssembly | Complex data transformations, custom business logic | -| [**External Functions**](04-external-function.md) | Cloud Feature | Custom operations using external servers | Scalable processing, external library integration | -| [**Dictionary**](07-dictionary.md) | Data Integration | In-memory key-value store for external data | Fast lookups from MySQL, Redis sources | -| [**Stored Procedures**](08-stored-procedure.md) | SQL Scripting | Reusable command sets with control flow | Multi-step operations, complex business logic | +### [Filtering & Selection](./00-basics/filtering-selection.md) +```sql +-- Select and filter data +SELECT name, salary FROM employees +WHERE department = 'Engineering' +ORDER BY salary DESC; +``` -## Query Optimization & Analysis +### [Aggregating Data](./00-basics/aggregating-data.md) +```sql +-- Group and summarize data +SELECT department, + COUNT(*) as emp_count, + AVG(salary) as avg_salary +FROM employees +GROUP BY department; +``` + +### [Advanced Grouping](./00-basics/groupby/index.md) +Multi-dimensional analysis with CUBE, ROLLUP, and GROUPING SETS + +--- -| Tool | Purpose | Access Method | Key Features | -|------|---------|---------------|--------------| -| [**Query Profile**](05-query-profile.md) | Performance analysis | Databend Cloud Monitor | Visual execution plan, performance metrics | -| [**Query Hash**](06-query-hash.md) | Query identification | SQL functions | Unique query fingerprinting, performance tracking | +## 🔗 [Combining Data](./01-combining-data/index.md) + +Connect data from multiple sources using JOINs and CTEs. + +### [JOINs](./01-combining-data/joins.md) +```sql +-- Combine related tables +SELECT e.name, d.department_name +FROM employees e +JOIN departments d ON e.department_id = d.id; +``` + +### [Common Table Expressions (CTE)](./01-combining-data/cte.md) +```sql +-- Structure complex queries +WITH high_earners AS ( + SELECT * FROM employees WHERE salary > 75000 +) +SELECT department, COUNT(*) as count +FROM high_earners GROUP BY department; +``` + +--- -## GROUP BY Extensions +## ⚡ [Advanced Features](./02-advanced/index.md) -| Extension | Description | Best For | -|-----------|-------------|----------| -| [**CUBE**](01-groupby/group-by-cube.md) | All possible combinations of grouping columns | Multi-dimensional analysis | -| [**ROLLUP**](01-groupby/group-by-rollup.md) | Hierarchical subtotals and grand totals | Hierarchical reporting | -| [**GROUPING SETS**](01-groupby/group-by-grouping-sets.md) | Custom grouping combinations | Flexible aggregation scenarios | +Extend capabilities with custom functions and external integrations. -## Quick Start Guide +### [User-Defined Functions](./02-advanced/udf.md) +```sql +-- Create reusable functions +CREATE FUNCTION calculate_bonus(salary FLOAT, rating FLOAT) +RETURNS FLOAT AS $$ salary * rating * 0.1 $$; +``` -1. **Basic Queries**: Start with [JOIN](02-join.md) and [GROUP BY](01-groupby/index.md) for fundamental data operations -2. **Advanced Logic**: Use [CTE](00-cte.md) for complex query structures -3. **Custom Functions**: Implement [UDF](03-udf.md) for specialized data processing -4. **Performance**: Leverage [Query Profile](05-query-profile.md) for optimization insights -5. **External Data**: Integrate external sources with [Dictionary](07-dictionary.md) +### More Advanced Features +- [External Functions](./02-advanced/external-function.md) - Cloud ML integration +- [Stored Procedures](./02-advanced/stored-procedure.md) - Multi-step operations +- [Sequences](./02-advanced/sequences.md) - Unique ID generation --- + +## 🚀 [Query Optimization](./03-optimization/index.md) + +Analyze and improve query performance with profiling tools. + +### [Query Profile](./03-optimization/query-profile.md) +Visual execution plan analysis (Databend Cloud: Monitor → SQL History) + +### [Performance Analysis](./03-optimization/query-hash.md) +```sql +-- Analyze query execution +EXPLAIN SELECT * FROM orders o +JOIN customers c ON o.customer_id = c.id +WHERE o.order_date >= '2023-01-01'; +``` + +--- + +## Quick Reference + +### Most Common Patterns +```sql +-- Top N query +SELECT * FROM employees ORDER BY salary DESC LIMIT 10; + +-- Filter and aggregate +SELECT department, AVG(salary) +FROM employees +WHERE hire_date >= '2023-01-01' +GROUP BY department +HAVING AVG(salary) > 70000; + +-- Join with CTE +WITH recent_orders AS ( + SELECT * FROM orders WHERE order_date >= '2023-01-01' +) +SELECT c.name, COUNT(*) as order_count +FROM customers c +JOIN recent_orders o ON c.id = o.customer_id +GROUP BY c.name; +``` diff --git a/docs/en/sql-reference/00-sql-reference/98-sql-scripting.md b/docs/en/sql-reference/00-sql-reference/98-sql-scripting.md index 2d0885079b..a2241f4779 100644 --- a/docs/en/sql-reference/00-sql-reference/98-sql-scripting.md +++ b/docs/en/sql-reference/00-sql-reference/98-sql-scripting.md @@ -4,7 +4,7 @@ title: SQL Scripting This page outlines the SQL scripting options available in Databend. You can use SQL scripting with either of the following query methods: -- [Stored Procedure](/guides/query/stored-procedure) +- [Stored Procedure](/sql/sql-commands/ddl/procedure/create-procedure) - [EXECUTE IMMEDIATE](/sql/sql-commands/administration-cmds/execute-immediate) ### Variable Declaration diff --git a/docs/en/sql-reference/10-sql-commands/00-ddl/07-aggregating-index/create-aggregating-index.md b/docs/en/sql-reference/10-sql-commands/00-ddl/07-aggregating-index/create-aggregating-index.md index f535d56c8c..4c5060a220 100644 --- a/docs/en/sql-reference/10-sql-commands/00-ddl/07-aggregating-index/create-aggregating-index.md +++ b/docs/en/sql-reference/10-sql-commands/00-ddl/07-aggregating-index/create-aggregating-index.md @@ -21,7 +21,7 @@ CREATE [ OR REPLACE ] [ ASYNC ] AGGREGATING INDEX AS SELECT ... - `ASYNC` Option: Adding ASYNC is optional. It allows the index to be created asynchronously. This means the index isn't built right away. To build it later, use the [REFRESH AGGREGATING INDEX](refresh-aggregating-index.md) command. -- When creating aggregating indexes, limit their usage to standard [Aggregate Functions](../../../20-sql-functions/07-aggregate-functions/index.md) (e.g., AVG, SUM, MIN, MAX, COUNT and GROUP BY), while keeping in mind that [GROUPING SETS](/guides/query/groupby/group-by-grouping-sets), [Window Functions](../../../20-sql-functions/08-window-functions/index.md), [LIMIT](../../20-query-syntax/01-query-select.md#limit-clause), and [ORDER BY](../../20-query-syntax/01-query-select.md#order-by-clause) are not accepted, or you will get an error: `Currently create aggregating index just support simple query, like: SELECT ... FROM ... WHERE ... GROUP BY ...`. +- When creating aggregating indexes, limit their usage to standard [Aggregate Functions](../../../20-sql-functions/07-aggregate-functions/index.md) (e.g., AVG, SUM, MIN, MAX, COUNT and GROUP BY), while keeping in mind that GROUPING SETS, [Window Functions](../../../20-sql-functions/08-window-functions/index.md), [LIMIT](../../20-query-syntax/01-query-select.md#limit-clause), and [ORDER BY](../../20-query-syntax/01-query-select.md#order-by-clause) are not accepted, or you will get an error: `Currently create aggregating index just support simple query, like: SELECT ... FROM ... WHERE ... GROUP BY ...`. - The query filter scope defined when creating aggregating indexes should either match or encompass the scope of your actual queries. diff --git a/docs/en/sql-reference/10-sql-commands/00-ddl/11-external-function/ddl-create-function.md b/docs/en/sql-reference/10-sql-commands/00-ddl/11-external-function/ddl-create-function.md index 86152f5680..9d8d8e0a47 100644 --- a/docs/en/sql-reference/10-sql-commands/00-ddl/11-external-function/ddl-create-function.md +++ b/docs/en/sql-reference/10-sql-commands/00-ddl/11-external-function/ddl-create-function.md @@ -31,4 +31,12 @@ CREATE [ OR REPLACE ] FUNCTION [ IF NOT EXISTS ] ## Examples -See [Usage Examples](/guides/query/external-function#usage-examples) for more information. \ No newline at end of file +This example creates an external function that calculates the greatest common divisor (GCD) of two integers: + +```sql +CREATE FUNCTION gcd AS (INT, INT) + RETURNS INT + LANGUAGE python + HANDLER = 'gcd' + ADDRESS = 'http://localhost:8815'; +``` \ No newline at end of file diff --git a/docs/en/sql-reference/10-sql-commands/00-ddl/11-external-function/index.md b/docs/en/sql-reference/10-sql-commands/00-ddl/11-external-function/index.md index f8baf8c72d..96dca77e0d 100644 --- a/docs/en/sql-reference/10-sql-commands/00-ddl/11-external-function/index.md +++ b/docs/en/sql-reference/10-sql-commands/00-ddl/11-external-function/index.md @@ -12,10 +12,6 @@ This page provides a comprehensive overview of External Function operations in D | [ALTER EXTERNAL FUNCTION](ddl-alter-function.md) | Modifies an existing external function | | [DROP EXTERNAL FUNCTION](ddl-drop-function.md) | Removes an external function | -## Related Topics - -- [External Function](/guides/query/external-function) - :::note External Functions in Databend allow you to extend functionality by integrating with external services through HTTP/HTTPS endpoints, enabling you to leverage external processing capabilities. ::: \ No newline at end of file diff --git a/docs/en/sql-reference/10-sql-commands/00-ddl/18-procedure/create-procedure.md b/docs/en/sql-reference/10-sql-commands/00-ddl/18-procedure/create-procedure.md index 04ef5750e0..0102687657 100644 --- a/docs/en/sql-reference/10-sql-commands/00-ddl/18-procedure/create-procedure.md +++ b/docs/en/sql-reference/10-sql-commands/00-ddl/18-procedure/create-procedure.md @@ -29,7 +29,7 @@ $$; | `` | Name of the procedure. | | ` ` | Input parameters (optional), each with a specified data type. Multiple parameters can be defined and separated by commas. | | `RETURNS [NOT NULL]` | Specifies the data type of the return value. `NOT NULL` ensures the returned value cannot be NULL. | -| `LANGUAGE` | Specifies the language in which the procedure body is written. Currently, only `SQL` is supported. For details, see [SQL Scripting](/guides/query/stored-procedure#sql-scripting). | +| `LANGUAGE` | Specifies the language in which the procedure body is written. Currently, only `SQL` is supported. | | `COMMENT` | Optional text describing the procedure. | | `AS ...` | Encloses the procedure body, which contains SQL statements, variable declarations, loops, and a RETURN statement. | diff --git a/docs/en/sql-reference/10-sql-commands/20-query-syntax/04-query-join.md b/docs/en/sql-reference/10-sql-commands/20-query-syntax/04-query-join.md index 9fe17a1c7a..3e8120ba52 100644 --- a/docs/en/sql-reference/10-sql-commands/20-query-syntax/04-query-join.md +++ b/docs/en/sql-reference/10-sql-commands/20-query-syntax/04-query-join.md @@ -2,4 +2,4 @@ title: JOIN --- -Databend supports a variety of JOINs. See [JOINs](/guides/query/join) for details. \ No newline at end of file +Databend supports a variety of JOINs. \ No newline at end of file diff --git a/docs/en/sql-reference/10-sql-commands/20-query-syntax/06-query-group-by.md b/docs/en/sql-reference/10-sql-commands/20-query-syntax/06-query-group-by.md index 24bf9ef648..1e95d1ec69 100644 --- a/docs/en/sql-reference/10-sql-commands/20-query-syntax/06-query-group-by.md +++ b/docs/en/sql-reference/10-sql-commands/20-query-syntax/06-query-group-by.md @@ -2,4 +2,4 @@ title: GROUP BY --- -Databend supports GROUP BY with a variety of extensions. For more information, see [GROUP BY](/guides/query/groupby/). \ No newline at end of file +Databend supports GROUP BY with a variety of extensions. \ No newline at end of file diff --git a/docs/en/sql-reference/10-sql-commands/20-query-syntax/with-clause.md b/docs/en/sql-reference/10-sql-commands/20-query-syntax/with-clause.md index 1927a2c36c..364c7d27d1 100644 --- a/docs/en/sql-reference/10-sql-commands/20-query-syntax/with-clause.md +++ b/docs/en/sql-reference/10-sql-commands/20-query-syntax/with-clause.md @@ -2,4 +2,4 @@ title: WITH Clause --- -Databend uses the WITH clause for common table expressions (CTEs). For details. see [CTEs](/guides/query/cte). \ No newline at end of file +Databend uses the WITH clause for common table expressions (CTEs). \ No newline at end of file diff --git a/docs/en/sql-reference/10-sql-commands/40-explain-cmds/explain.md b/docs/en/sql-reference/10-sql-commands/40-explain-cmds/explain.md index 5941c5b392..f988102b2d 100644 --- a/docs/en/sql-reference/10-sql-commands/40-explain-cmds/explain.md +++ b/docs/en/sql-reference/10-sql-commands/40-explain-cmds/explain.md @@ -4,7 +4,7 @@ title: EXPLAIN Shows the execution plan of a SQL statement. An execution plan is shown as a tree consisting of different operators where you can see how Databend will execute the SQL statement. An operator usually includes one or more fields describing the actions Databend will perform or the objects related to the query. -For example, the following execution plan returned by the EXPLAIN command includes an operator named *TableScan* with several fields. For a list of common operators and fields, see [Common Operators & Fields](/guides/query/query-profile#common-operators--fields). +For example, the following execution plan returned by the EXPLAIN command includes an operator named *TableScan* with several fields. ```sql EXPLAIN SELECT * FROM allemployees; @@ -19,7 +19,7 @@ TableScan └── push downs: [filters: [], limit: NONE] ``` -If you are using Databend Cloud, you can utilize the Query Profile feature to visualize the execution plan of your SQL statements. For more information, see [Query Profile](/guides/query/query-profile). +If you are using Databend Cloud, you can utilize the Query Profile feature to visualize the execution plan of your SQL statements. ## Syntax diff --git a/docs/en/sql-reference/20-sql-functions/19-dictionary-functions/dict-get.md b/docs/en/sql-reference/20-sql-functions/19-dictionary-functions/dict-get.md index b74278949c..44348ad546 100644 --- a/docs/en/sql-reference/20-sql-functions/19-dictionary-functions/dict-get.md +++ b/docs/en/sql-reference/20-sql-functions/19-dictionary-functions/dict-get.md @@ -21,4 +21,10 @@ DICT_GET([db_name.], '', ) ## Examples -See [Usage Example](/guides/query/dictionary#usage-example). \ No newline at end of file +This example demonstrates how to use DICT_GET to retrieve values from a dictionary: + +```sql +-- Assume we have a dictionary named 'user_info' with 'user_id' as key +-- and 'name' as an attribute +SELECT DICT_GET('user_info', 'name', 12345); +``` \ No newline at end of file diff --git a/site-redirects.ts b/site-redirects.ts index 5b901b2dc2..5ca031ee4f 100644 --- a/site-redirects.ts +++ b/site-redirects.ts @@ -383,6 +383,52 @@ const siteRedirects = [ { from: '/guides/load-data/transform/querying-metadata', to: '/tutorials/load/query-metadata' + }, + // Query guide restructuring - moved from flat to hierarchical structure + { + from: '/guides/query/filtering-selection', + to: '/guides/query/basics/filtering-selection' + }, + { + from: '/guides/query/aggregating-data', + to: '/guides/query/basics/aggregating-data' + }, + { + from: '/guides/query/joins', + to: '/guides/query/combining-data/joins' + }, + { + from: '/guides/query/cte', + to: '/guides/query/combining-data/cte' + }, + { + from: '/guides/query/udf', + to: '/guides/query/advanced/udf' + }, + { + from: '/guides/query/external-function', + to: '/guides/query/advanced/external-function' + }, + { + from: '/guides/query/stored-procedure', + to: '/guides/query/advanced/stored-procedure' + }, + { + from: '/guides/query/sequences', + to: '/guides/query/advanced/sequences' + }, + { + from: '/guides/query/query-profile', + to: '/guides/query/optimization/query-profile' + }, + { + from: '/guides/query/query-hash', + to: '/guides/query/optimization/query-hash' + }, + // Dictionary page removed - redirect to advanced features index + { + from: '/guides/query/dictionary', + to: '/guides/query/advanced/' } ]; export default siteRedirects; \ No newline at end of file diff --git a/static/robots.txt b/static/robots.txt new file mode 100644 index 0000000000..906250fb2a --- /dev/null +++ b/static/robots.txt @@ -0,0 +1,4 @@ +User-agent: * +Disallow: /404 +Disallow: /tags/ +Sitemap: https://www.databend.com/sitemap.xml