From 74c647a68579167ae18b0013787810f680ed6acb Mon Sep 17 00:00:00 2001 From: Alexander Marshalov Date: Mon, 6 Oct 2025 00:06:53 +0200 Subject: [PATCH 1/5] implemented subqueries support --- lib/logsql/select.go | 60 +++++++++++++++++++++++++++++++++++---- lib/logsql/select_test.go | 39 +++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 5 deletions(-) diff --git a/lib/logsql/select.go b/lib/logsql/select.go index 774721d..26a0eda 100644 --- a/lib/logsql/select.go +++ b/lib/logsql/select.go @@ -399,6 +399,11 @@ func (v *selectTranslatorVisitor) processFrom(from ast.TableExpr) ([]string, err return nil, err } return nil, nil + case *ast.SubqueryTable: + if err := v.registerBaseSubquery(t); err != nil { + return nil, err + } + return nil, nil case *ast.JoinExpr: return v.processJoin(t) default: @@ -487,6 +492,46 @@ func (v *selectTranslatorVisitor) registerBaseTable(table *ast.TableName) error return nil } +func (v *selectTranslatorVisitor) registerBaseSubquery(table *ast.SubqueryTable) error { + if table == nil || table.Select == nil { + return &TranslationError{ + Code: http.StatusBadRequest, + Message: "translator: invalid subquery reference", + } + } + alias := strings.TrimSpace(table.Alias) + if alias == "" { + return &TranslationError{ + Code: http.StatusBadRequest, + Message: "translator: subquery requires alias", + } + } + aliasLower := strings.ToLower(alias) + if v.baseAlias != "" && v.baseAlias != aliasLower { + return &TranslationError{ + Code: http.StatusBadRequest, + Message: "translator: multiple base tables are not supported", + } + } + subQuery, err := translateSelectStatementToLogsQLWithContext(table.Select, translationContext{ + sp: v.sp, + ctes: v.availableCTEs, + }) + if err != nil { + return &TranslationError{ + Code: http.StatusBadRequest, + Message: fmt.Sprintf("translator: failed to translate subquery: %s", err), + Err: err, + } + } + v.baseAlias = aliasLower + v.baseUsesPipeline = true + v.basePipeline = subQuery + v.baseFilter = "" + v.registerBinding(aliasLower, true) + return nil +} + func (v *selectTranslatorVisitor) registerBinding(alias string, isBase bool) { key := strings.ToLower(alias) if key == "" { @@ -509,16 +554,21 @@ func (v *selectTranslatorVisitor) processJoin(join *ast.JoinExpr) ([]string, err } } - leftTable, ok := join.Left.(*ast.TableName) - if !ok { + switch left := join.Left.(type) { + case *ast.TableName: + if err := v.registerBaseTable(left); err != nil { + return nil, err + } + case *ast.SubqueryTable: + if err := v.registerBaseSubquery(left); err != nil { + return nil, err + } + default: return nil, &TranslationError{ Code: http.StatusBadRequest, Message: "translator: JOIN left side must be table reference", } } - if err := v.registerBaseTable(leftTable); err != nil { - return nil, err - } var rightAlias string var rightQuery string diff --git a/lib/logsql/select_test.go b/lib/logsql/select_test.go index 1110935..4548ebe 100644 --- a/lib/logsql/select_test.go +++ b/lib/logsql/select_test.go @@ -324,6 +324,30 @@ SELECT user FROM recent_errors WHERE service = 'api'`, sql: "SELECT LOWER(user) AS user_lower, COUNT(*) AS total FROM logs GROUP BY user_lower", expected: "* | format \"\" as group_1 | stats by (group_1) count() total | rename group_1 as user_lower", }, + { + name: "subquery as base table", + sql: `SELECT * +FROM ( + SELECT * + FROM logs + WHERE level = 'error' +) AS recent_errors`, + expected: "level:error", + }, + { + name: "subquery as base with filter", + sql: `SELECT recent.user, recent.fail_count +FROM ( + SELECT user, COUNT(*) AS fail_count + FROM logs + WHERE level = 'error' + GROUP BY user +) AS recent +WHERE recent.fail_count > 10 +ORDER BY recent.fail_count DESC +LIMIT 5`, + expected: "level:error | stats by (user) count() fail_count | filter fail_count:>10 | fields user, fail_count | sort by (fail_count desc) | limit 5", + }, { name: "join with subquery", sql: `SELECT l.user, m.fail_count @@ -379,6 +403,21 @@ func TestToLogsQLWithConfig(t *testing.T) { } }) + t.Run("join with subquery base", func(t *testing.T) { + sql := `SELECT recent.user, a.level +FROM ( + SELECT user + FROM logs + WHERE level = 'error' +) AS recent +INNER JOIN api AS a ON recent.user = a.user` + got := mustTranslateWithTables(t, sql, tables) + expected := "level:error | fields user | join by (user) (service:api) inner | fields user, level" + if got != expected { + t.Fatalf("unexpected query:\nexpected: %s\n got: %s", expected, got) + } + }) + t.Run("unknown table", func(t *testing.T) { _, err := translateWithTables(t, "SELECT * FROM missing", tables) if err == nil { From 84eae5683dd09928aac794616af022103d1cbea3 Mon Sep 17 00:00:00 2001 From: Alexander Marshalov Date: Mon, 6 Oct 2025 00:22:18 +0200 Subject: [PATCH 2/5] support subqueries without specifying alias --- lib/logsql/select.go | 58 ++++++++++++++++++++++++++++----------- lib/logsql/select_test.go | 39 ++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 16 deletions(-) diff --git a/lib/logsql/select.go b/lib/logsql/select.go index 26a0eda..c0a5c69 100644 --- a/lib/logsql/select.go +++ b/lib/logsql/select.go @@ -51,6 +51,7 @@ type selectTranslatorVisitor struct { sp *store.Provider bindings map[string]*tableBinding + autoAliasCounter int baseAlias string pendingLeftFilter []ast.Expr aggResults map[string]string @@ -172,6 +173,7 @@ func (v *selectTranslatorVisitor) translateSimpleSelect(stmt *ast.SelectStatemen } v.bindings = make(map[string]*tableBinding) + v.autoAliasCounter = 0 v.pendingLeftFilter = nil v.aggResults = nil v.baseAlias = "" @@ -501,10 +503,7 @@ func (v *selectTranslatorVisitor) registerBaseSubquery(table *ast.SubqueryTable) } alias := strings.TrimSpace(table.Alias) if alias == "" { - return &TranslationError{ - Code: http.StatusBadRequest, - Message: "translator: subquery requires alias", - } + alias = v.generateSubqueryAlias("base") } aliasLower := strings.ToLower(alias) if v.baseAlias != "" && v.baseAlias != aliasLower { @@ -540,6 +539,21 @@ func (v *selectTranslatorVisitor) registerBinding(alias string, isBase bool) { v.bindings[key] = &tableBinding{alias: key, isBase: isBase} } +func (v *selectTranslatorVisitor) generateSubqueryAlias(prefix string) string { + base := strings.TrimSpace(prefix) + if base == "" { + base = "subquery" + } + base = strings.ToLower(base) + for { + v.autoAliasCounter++ + candidate := fmt.Sprintf("__%s_%d", base, v.autoAliasCounter) + if _, exists := v.bindings[candidate]; !exists { + return candidate + } + } +} + func (v *selectTranslatorVisitor) processJoin(join *ast.JoinExpr) ([]string, error) { if join == nil { return nil, &TranslationError{ @@ -656,10 +670,7 @@ func (v *selectTranslatorVisitor) processJoin(join *ast.JoinExpr) ([]string, err case *ast.SubqueryTable: alias := strings.TrimSpace(rt.Alias) if alias == "" { - return nil, &TranslationError{ - Code: http.StatusBadRequest, - Message: "translator: JOIN subquery requires alias", - } + alias = v.generateSubqueryAlias("join") } rightAlias = strings.ToLower(alias) if _, exists := v.bindings[rightAlias]; exists { @@ -766,8 +777,8 @@ func (v *selectTranslatorVisitor) extractJoinSpec(cond ast.JoinCondition, rightA switch { case leftIsIdent && rightIsIdent: - leftQual := v.qualifierForIdentifier(leftIdent) - rightQual := v.qualifierForIdentifier(rightIdent) + leftQual := v.qualifierForIdentifierWithDefault(leftIdent, v.baseAlias) + rightQual := v.qualifierForIdentifierWithDefault(rightIdent, rightAlias) if leftQual == v.baseAlias && rightQual == rightAlias { leftField, err := v.normalizeIdentifier(leftIdent) if err != nil { @@ -807,8 +818,8 @@ func (v *selectTranslatorVisitor) extractJoinSpec(cond ast.JoinCondition, rightA } } - leftAliases := v.aliasesForExpr(bin.Left) - rightAliases := v.aliasesForExpr(bin.Right) + leftAliases := v.aliasesForExprWithDefault(bin.Left, v.baseAlias) + rightAliases := v.aliasesForExprWithDefault(bin.Right, rightAlias) if v.isAliasOnly(leftAliases, v.baseAlias) && len(rightAliases) == 0 { leftFilters = append(leftFilters, expr) @@ -857,21 +868,30 @@ func flattenAnd(expr ast.Expr) []ast.Expr { } func (v *selectTranslatorVisitor) qualifierForIdentifier(ident *ast.Identifier) string { + return v.qualifierForIdentifierWithDefault(ident, v.baseAlias) +} + +func (v *selectTranslatorVisitor) qualifierForIdentifierWithDefault(ident *ast.Identifier, fallback string) string { if ident == nil || len(ident.Parts) == 0 { - return v.baseAlias + return fallback } first := strings.ToLower(ident.Parts[0]) if _, ok := v.bindings[first]; ok { return first } - return v.baseAlias + return fallback } func (v *selectTranslatorVisitor) aliasesForExpr(expr ast.Expr) map[string]struct{} { + return v.aliasesForExprWithDefault(expr, v.baseAlias) +} + +func (v *selectTranslatorVisitor) aliasesForExprWithDefault(expr ast.Expr, fallback string) map[string]struct{} { aliases := make(map[string]struct{}) walkExpr(expr, func(e ast.Expr) { if id, ok := e.(*ast.Identifier); ok { - aliases[v.qualifierForIdentifier(id)] = struct{}{} + alias := v.qualifierForIdentifierWithDefault(id, fallback) + aliases[alias] = struct{}{} } }) delete(aliases, "") @@ -933,7 +953,13 @@ func (v *selectTranslatorVisitor) ensureBaseAliasesOnly(expr ast.Expr) error { } func (v *selectTranslatorVisitor) ensureAliases(expr ast.Expr, allowed map[string]struct{}) error { - aliases := v.aliasesForExpr(expr) + fallback := v.baseAlias + if len(allowed) == 1 { + for alias := range allowed { + fallback = alias + } + } + aliases := v.aliasesForExprWithDefault(expr, fallback) for alias := range aliases { if alias == "" { continue diff --git a/lib/logsql/select_test.go b/lib/logsql/select_test.go index 4548ebe..862e593 100644 --- a/lib/logsql/select_test.go +++ b/lib/logsql/select_test.go @@ -334,6 +334,16 @@ FROM ( ) AS recent_errors`, expected: "level:error", }, + { + name: "subquery as base table without alias", + sql: `SELECT * +FROM ( + SELECT * + FROM logs + WHERE level = 'error' +)`, + expected: "level:error", + }, { name: "subquery as base with filter", sql: `SELECT recent.user, recent.fail_count @@ -345,6 +355,20 @@ FROM ( ) AS recent WHERE recent.fail_count > 10 ORDER BY recent.fail_count DESC +LIMIT 5`, + expected: "level:error | stats by (user) count() fail_count | filter fail_count:>10 | fields user, fail_count | sort by (fail_count desc) | limit 5", + }, + { + name: "subquery as base with filter without alias", + sql: `SELECT user, fail_count +FROM ( + SELECT user, COUNT(*) AS fail_count + FROM logs + WHERE level = 'error' + GROUP BY user +) +WHERE fail_count > 10 +ORDER BY fail_count DESC LIMIT 5`, expected: "level:error | stats by (user) count() fail_count | filter fail_count:>10 | fields user, fail_count | sort by (fail_count desc) | limit 5", }, @@ -360,6 +384,21 @@ INNER JOIN ( ) AS m ON l.user = m.user WHERE l.level = 'error' ORDER BY m.fail_count DESC +LIMIT 5`, + expected: "level:error | join by (user) (level:error | stats by (user) count() fail_count) inner | fields user, fail_count | sort by (fail_count desc) | limit 5", + }, + { + name: "join with subquery without alias", + sql: `SELECT l.user, fail_count +FROM logs AS l +INNER JOIN ( + SELECT user, COUNT(*) AS fail_count + FROM logs + WHERE level = 'error' + GROUP BY user +) ON l.user = user +WHERE l.level = 'error' +ORDER BY fail_count DESC LIMIT 5`, expected: "level:error | join by (user) (level:error | stats by (user) count() fail_count) inner | fields user, fail_count | sort by (fail_count desc) | limit 5", }, From 64d75c16ae2f918567b8bfccb8e84582ea59a85c Mon Sep 17 00:00:00 2001 From: Alexander Marshalov Date: Mon, 6 Oct 2025 00:26:58 +0200 Subject: [PATCH 3/5] add example for subquery --- .../web/ui/src/components/docs/Docs.tsx | 2 +- .../web/ui/src/components/sql-editor/examples.ts | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/cmd/sql-to-logsql/web/ui/src/components/docs/Docs.tsx b/cmd/sql-to-logsql/web/ui/src/components/docs/Docs.tsx index e46e624..906f8e6 100644 --- a/cmd/sql-to-logsql/web/ui/src/components/docs/Docs.tsx +++ b/cmd/sql-to-logsql/web/ui/src/components/docs/Docs.tsx @@ -54,7 +54,7 @@ export function Docs() {

  • SELECT, DISTINCT, AS, OVER, PARTITION BY
    • -
  • FROM, WITH
    • +
  • FROM, WITH, subqueries
  • WHERE, AND, OR
  • LEFT JOIN / JOIN / INNER JOIN
  • LIKE, NOT LIKE, BETWEEN, IN, NOT IN, IS NULL, IS NOT NULL
    • diff --git a/cmd/sql-to-logsql/web/ui/src/components/sql-editor/examples.ts b/cmd/sql-to-logsql/web/ui/src/components/sql-editor/examples.ts index 7c5b3dc..75d65ac 100644 --- a/cmd/sql-to-logsql/web/ui/src/components/sql-editor/examples.ts +++ b/cmd/sql-to-logsql/web/ui/src/components/sql-editor/examples.ts @@ -86,6 +86,19 @@ ORDER BY messages_count DESC` SELECT UPPER(container), total FROM container_stats WHERE container IS NOT NULL +ORDER BY total DESC`, + }, + { + id: "subqueries", + title: "Subqueries", + sql: `SELECT UPPER(container), total +FROM ( + SELECT kubernetes.container_name AS container, COUNT(*) AS total + FROM logs + GROUP BY kubernetes.container_name + LIMIT 20 +) container_stats +WHERE container IS NOT NULL ORDER BY total DESC`, }, { From 13c960596044063a9bb6e87fe356667501123d04 Mon Sep 17 00:00:00 2001 From: Alexander Marshalov Date: Mon, 6 Oct 2025 00:28:07 +0200 Subject: [PATCH 4/5] extend readme with subqueries support --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6003802..7094ba1 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ Supported highlights: - and date helpers (`CURRENT_DATE`, `CURRENT_TIMESTAMP`). - `WHERE` with comparison operators, `BETWEEN`, `IN`, `LIKE`, `IS (NOT) NULL` - `ORDER BY`, `LIMIT`, `OFFSET`, `DISTINCT` -- Common Table Expressions (CTE) using `WITH` keyword +- Common Table Expressions (CTE) using `WITH` keyword and subqueries - `GROUP BY`, `HAVING`, `COUNT/SUM/AVG/MIN/MAX` - Window functions (`OVER (PARTITION BY ... ORDER BY ...)`) for `SUM` and `COUNT` - `JOIN` (inner/left) on equality predicates, including subqueries. From 0749bb54c88b3d2f90ab60678bf1d8989de1c6bc Mon Sep 17 00:00:00 2001 From: Alexander Marshalov Date: Mon, 6 Oct 2025 00:29:56 +0200 Subject: [PATCH 5/5] fix linter comments --- lib/logsql/select.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lib/logsql/select.go b/lib/logsql/select.go index c0a5c69..caec74b 100644 --- a/lib/logsql/select.go +++ b/lib/logsql/select.go @@ -867,10 +867,6 @@ func flattenAnd(expr ast.Expr) []ast.Expr { return []ast.Expr{expr} } -func (v *selectTranslatorVisitor) qualifierForIdentifier(ident *ast.Identifier) string { - return v.qualifierForIdentifierWithDefault(ident, v.baseAlias) -} - func (v *selectTranslatorVisitor) qualifierForIdentifierWithDefault(ident *ast.Identifier, fallback string) string { if ident == nil || len(ident.Parts) == 0 { return fallback @@ -882,10 +878,6 @@ func (v *selectTranslatorVisitor) qualifierForIdentifierWithDefault(ident *ast.I return fallback } -func (v *selectTranslatorVisitor) aliasesForExpr(expr ast.Expr) map[string]struct{} { - return v.aliasesForExprWithDefault(expr, v.baseAlias) -} - func (v *selectTranslatorVisitor) aliasesForExprWithDefault(expr ast.Expr, fallback string) map[string]struct{} { aliases := make(map[string]struct{}) walkExpr(expr, func(e ast.Expr) {