From cbd58de21fc4379dc8198ccefb3e1cf58344c06a Mon Sep 17 00:00:00 2001 From: Alexey Pozdniakov Date: Mon, 25 Aug 2025 22:26:07 +0300 Subject: [PATCH 1/4] [YQ-4573] Streaming integration tests (#23508) --- .../streaming_optimize/canondata/result.json | 192 ++++++++++++------ .../err_file.out | 10 + .../err_file.out | 8 + .../ast.txt | 0 .../plan.json | 0 .../ast.txt | 0 .../plan.json | 0 .../ast.txt | 0 .../plan.json | 0 .../ast.txt | 0 .../plan.json | 0 .../ast.txt | 0 .../plan.json | 0 .../ast.txt | 0 .../plan.json | 0 .../ast.txt | 0 .../plan.json | 0 .../ast.txt | 0 .../plan.json | 0 .../ast.txt | 54 +++++ .../plan.json | 106 ++++++++++ .../ast.txt | 54 +++++ .../plan.json | 106 ++++++++++ .../ast.txt | 52 +++++ .../plan.json | 0 .../ast.txt | 55 +++++ .../plan.json | 109 ++++++++++ .../ast.txt | 51 +++++ .../plan.json | 106 ++++++++++ .../ast.txt | 97 +++++++++ .../plan.json | 106 ++++++++++ .../ast.txt | 97 +++++++++ .../plan.json | 106 ++++++++++ .../ast.txt | 0 .../plan.json | 0 .../ast.txt | 0 .../plan.json | 0 .../ast.txt | 0 .../plan.json | 0 .../ast.txt | 0 .../plan.json | 0 .../ast.txt | 0 .../plan.json | 0 .../ast.txt | 0 .../plan.json | 0 .../ast.txt | 0 .../plan.json | 0 .../ast.txt | 0 .../plan.json | 0 .../ast.txt | 0 .../plan.json | 0 .../ast.txt | 0 .../plan.json | 0 .../ast.txt | 0 .../plan.json | 0 .../ast.txt | 0 .../plan.json | 0 .../ast.txt | 81 -------- .../ast.txt | 20 ++ .../plan.json | 93 +++++++++ .../GroupByHoppingWithDataWatermarks.sql | 27 --- .../suites/{ => hop}/GroupByHop.sql | 0 .../{ => hop}/GroupByHopByStringKey.sql | 0 .../suites/{ => hop}/GroupByHopExprKey.sql | 0 .../suites/{ => hop}/GroupByHopListKey.sql | 0 .../suites/{ => hop}/GroupByHopNoKey.sql | 0 .../suites/{ => hop}/GroupByHopPercentile.sql | 0 .../GroupByHopTimeExtractorUnusedColumns.sql | 0 .../GroupByHopWithDataWatermarks.sql | 0 .../streaming_optimize/suites/hop/default.cfg | 1 + .../hopping_window/GroupByHoppingWindow.sql | 34 ++++ .../GroupByHoppingWindowByStringKey.sql | 32 +++ .../GroupByHoppingWindowExprKey.sql | 33 +++ .../GroupByHoppingWindowListKey.sql | 34 ++++ .../GroupByHoppingWindowNoKey.sql | 32 +++ .../GroupByHoppingWindowPercentile.sql | 33 +++ ...oppingWindowTimeExtractorUnusedColumns.sql | 33 +++ .../suites/hopping_window/default.cfg | 1 + .../suites/{ => pq}/ReadTopic.sql | 0 .../suites/{ => pq}/ReadTopicWithMetadata.sql | 0 .../ReadTopicWithMetadataInsideFilter.sql | 0 .../ReadTopicWithMetadataNestedDeep.sql | 0 .../ReadTopicWithMetadataWithFilter.sql | 0 .../suites/{ => pq}/ReadTopicWithSchema.sql | 0 .../suites/{ => pq}/ReadTwoTopics.sql | 0 .../suites/{ => pq}/ReadWriteSameTopic.sql | 0 .../suites/{ => pq}/ReadWriteTopic.sql | 0 .../{ => pq}/ReadWriteTopicWithSchema.sql | 0 .../suites/{ => pq}/WriteTwoTopics.sql | 0 .../streaming_optimize/suites/pq/default.cfg | 1 + .../ReadTopicGroupWriteToSolomon.sql | 0 .../suites/{ => solomon}/default.cfg | 0 .../suites/watermarks/bad_column.sqlx | 13 ++ .../suites/watermarks/bad_pushdown.sqlx | 13 ++ .../suites/watermarks/default.cfg | 1 + .../suites/watermarks/watermarks.sql | 13 ++ .../streaming_optimize/test_sql_negative.py | 60 ++++++ .../streaming_optimize/test_sql_streaming.py | 13 +- ydb/tests/fq/streaming_optimize/ya.make | 1 + 99 files changed, 1700 insertions(+), 178 deletions(-) create mode 100644 ydb/tests/fq/streaming_optimize/canondata/test_sql_negative.test_watermarks-bad_column-default.txt_/err_file.out create mode 100644 ydb/tests/fq/streaming_optimize/canondata/test_sql_negative.test_watermarks-bad_pushdown-default.txt_/err_file.out rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-GroupByHop-default.txt_ => test_sql_streaming.test_hop-GroupByHop-default.txt_}/ast.txt (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-GroupByHop-default.txt_ => test_sql_streaming.test_hop-GroupByHop-default.txt_}/plan.json (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-GroupByHopByStringKey-default.txt_ => test_sql_streaming.test_hop-GroupByHopByStringKey-default.txt_}/ast.txt (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-GroupByHopByStringKey-default.txt_ => test_sql_streaming.test_hop-GroupByHopByStringKey-default.txt_}/plan.json (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-GroupByHopExprKey-default.txt_ => test_sql_streaming.test_hop-GroupByHopExprKey-default.txt_}/ast.txt (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-GroupByHopExprKey-default.txt_ => test_sql_streaming.test_hop-GroupByHopExprKey-default.txt_}/plan.json (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-GroupByHopListKey-default.txt_ => test_sql_streaming.test_hop-GroupByHopListKey-default.txt_}/ast.txt (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-GroupByHopListKey-default.txt_ => test_sql_streaming.test_hop-GroupByHopListKey-default.txt_}/plan.json (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-GroupByHopNoKey-default.txt_ => test_sql_streaming.test_hop-GroupByHopNoKey-default.txt_}/ast.txt (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-GroupByHopNoKey-default.txt_ => test_sql_streaming.test_hop-GroupByHopNoKey-default.txt_}/plan.json (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-GroupByHopPercentile-default.txt_ => test_sql_streaming.test_hop-GroupByHopPercentile-default.txt_}/ast.txt (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-GroupByHopPercentile-default.txt_ => test_sql_streaming.test_hop-GroupByHopPercentile-default.txt_}/plan.json (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-GroupByHopTimeExtractorUnusedColumns-default.txt_ => test_sql_streaming.test_hop-GroupByHopTimeExtractorUnusedColumns-default.txt_}/ast.txt (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-GroupByHopTimeExtractorUnusedColumns-default.txt_ => test_sql_streaming.test_hop-GroupByHopTimeExtractorUnusedColumns-default.txt_}/plan.json (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-GroupByHopWithDataWatermarks-default.txt_ => test_sql_streaming.test_hop-GroupByHopWithDataWatermarks-default.txt_}/ast.txt (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-GroupByHopWithDataWatermarks-default.txt_ => test_sql_streaming.test_hop-GroupByHopWithDataWatermarks-default.txt_}/plan.json (100%) create mode 100644 ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindow-default.txt_/ast.txt create mode 100644 ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindow-default.txt_/plan.json create mode 100644 ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowByStringKey-default.txt_/ast.txt create mode 100644 ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowByStringKey-default.txt_/plan.json create mode 100644 ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowExprKey-default.txt_/ast.txt rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-GroupByHoppingWithDataWatermarks-default.txt_ => test_sql_streaming.test_hopping_window-GroupByHoppingWindowExprKey-default.txt_}/plan.json (100%) create mode 100644 ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowListKey-default.txt_/ast.txt create mode 100644 ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowListKey-default.txt_/plan.json create mode 100644 ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowNoKey-default.txt_/ast.txt create mode 100644 ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowNoKey-default.txt_/plan.json create mode 100644 ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowPercentile-default.txt_/ast.txt create mode 100644 ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowPercentile-default.txt_/plan.json create mode 100644 ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowTimeExtractorUnusedColumns-default.txt_/ast.txt create mode 100644 ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowTimeExtractorUnusedColumns-default.txt_/plan.json rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-ReadTopic-default.txt_ => test_sql_streaming.test_pq-ReadTopic-default.txt_}/ast.txt (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-ReadTopic-default.txt_ => test_sql_streaming.test_pq-ReadTopic-default.txt_}/plan.json (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-ReadTopicWithMetadata-default.txt_ => test_sql_streaming.test_pq-ReadTopicWithMetadata-default.txt_}/ast.txt (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-ReadTopicWithMetadata-default.txt_ => test_sql_streaming.test_pq-ReadTopicWithMetadata-default.txt_}/plan.json (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-ReadTopicWithMetadataInsideFilter-default.txt_ => test_sql_streaming.test_pq-ReadTopicWithMetadataInsideFilter-default.txt_}/ast.txt (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-ReadTopicWithMetadataInsideFilter-default.txt_ => test_sql_streaming.test_pq-ReadTopicWithMetadataInsideFilter-default.txt_}/plan.json (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-ReadTopicWithMetadataNestedDeep-default.txt_ => test_sql_streaming.test_pq-ReadTopicWithMetadataNestedDeep-default.txt_}/ast.txt (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-ReadTopicWithMetadataNestedDeep-default.txt_ => test_sql_streaming.test_pq-ReadTopicWithMetadataNestedDeep-default.txt_}/plan.json (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-ReadTopicWithMetadataWithFilter-default.txt_ => test_sql_streaming.test_pq-ReadTopicWithMetadataWithFilter-default.txt_}/ast.txt (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-ReadTopicWithMetadataWithFilter-default.txt_ => test_sql_streaming.test_pq-ReadTopicWithMetadataWithFilter-default.txt_}/plan.json (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-ReadTopicWithSchema-default.txt_ => test_sql_streaming.test_pq-ReadTopicWithSchema-default.txt_}/ast.txt (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-ReadTopicWithSchema-default.txt_ => test_sql_streaming.test_pq-ReadTopicWithSchema-default.txt_}/plan.json (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-ReadTwoTopics-default.txt_ => test_sql_streaming.test_pq-ReadTwoTopics-default.txt_}/ast.txt (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-ReadTwoTopics-default.txt_ => test_sql_streaming.test_pq-ReadTwoTopics-default.txt_}/plan.json (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-ReadWriteSameTopic-default.txt_ => test_sql_streaming.test_pq-ReadWriteSameTopic-default.txt_}/ast.txt (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-ReadWriteSameTopic-default.txt_ => test_sql_streaming.test_pq-ReadWriteSameTopic-default.txt_}/plan.json (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-ReadWriteTopic-default.txt_ => test_sql_streaming.test_pq-ReadWriteTopic-default.txt_}/ast.txt (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-ReadWriteTopic-default.txt_ => test_sql_streaming.test_pq-ReadWriteTopic-default.txt_}/plan.json (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-ReadWriteTopicWithSchema-default.txt_ => test_sql_streaming.test_pq-ReadWriteTopicWithSchema-default.txt_}/ast.txt (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-ReadWriteTopicWithSchema-default.txt_ => test_sql_streaming.test_pq-ReadWriteTopicWithSchema-default.txt_}/plan.json (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-WriteTwoTopics-default.txt_ => test_sql_streaming.test_pq-WriteTwoTopics-default.txt_}/ast.txt (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-WriteTwoTopics-default.txt_ => test_sql_streaming.test_pq-WriteTwoTopics-default.txt_}/plan.json (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-ReadTopicGroupWriteToSolomon-default.txt_ => test_sql_streaming.test_solomon-ReadTopicGroupWriteToSolomon-default.txt_}/ast.txt (100%) rename ydb/tests/fq/streaming_optimize/canondata/{test_sql_streaming.test_suites-ReadTopicGroupWriteToSolomon-default.txt_ => test_sql_streaming.test_solomon-ReadTopicGroupWriteToSolomon-default.txt_}/plan.json (100%) delete mode 100644 ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHoppingWithDataWatermarks-default.txt_/ast.txt create mode 100644 ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_watermarks-watermarks-default.txt_/ast.txt create mode 100644 ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_watermarks-watermarks-default.txt_/plan.json delete mode 100644 ydb/tests/fq/streaming_optimize/suites/GroupByHoppingWithDataWatermarks.sql rename ydb/tests/fq/streaming_optimize/suites/{ => hop}/GroupByHop.sql (100%) rename ydb/tests/fq/streaming_optimize/suites/{ => hop}/GroupByHopByStringKey.sql (100%) rename ydb/tests/fq/streaming_optimize/suites/{ => hop}/GroupByHopExprKey.sql (100%) rename ydb/tests/fq/streaming_optimize/suites/{ => hop}/GroupByHopListKey.sql (100%) rename ydb/tests/fq/streaming_optimize/suites/{ => hop}/GroupByHopNoKey.sql (100%) rename ydb/tests/fq/streaming_optimize/suites/{ => hop}/GroupByHopPercentile.sql (100%) rename ydb/tests/fq/streaming_optimize/suites/{ => hop}/GroupByHopTimeExtractorUnusedColumns.sql (100%) rename ydb/tests/fq/streaming_optimize/suites/{ => hop}/GroupByHopWithDataWatermarks.sql (100%) create mode 100644 ydb/tests/fq/streaming_optimize/suites/hop/default.cfg create mode 100644 ydb/tests/fq/streaming_optimize/suites/hopping_window/GroupByHoppingWindow.sql create mode 100644 ydb/tests/fq/streaming_optimize/suites/hopping_window/GroupByHoppingWindowByStringKey.sql create mode 100644 ydb/tests/fq/streaming_optimize/suites/hopping_window/GroupByHoppingWindowExprKey.sql create mode 100644 ydb/tests/fq/streaming_optimize/suites/hopping_window/GroupByHoppingWindowListKey.sql create mode 100644 ydb/tests/fq/streaming_optimize/suites/hopping_window/GroupByHoppingWindowNoKey.sql create mode 100644 ydb/tests/fq/streaming_optimize/suites/hopping_window/GroupByHoppingWindowPercentile.sql create mode 100644 ydb/tests/fq/streaming_optimize/suites/hopping_window/GroupByHoppingWindowTimeExtractorUnusedColumns.sql create mode 100644 ydb/tests/fq/streaming_optimize/suites/hopping_window/default.cfg rename ydb/tests/fq/streaming_optimize/suites/{ => pq}/ReadTopic.sql (100%) rename ydb/tests/fq/streaming_optimize/suites/{ => pq}/ReadTopicWithMetadata.sql (100%) rename ydb/tests/fq/streaming_optimize/suites/{ => pq}/ReadTopicWithMetadataInsideFilter.sql (100%) rename ydb/tests/fq/streaming_optimize/suites/{ => pq}/ReadTopicWithMetadataNestedDeep.sql (100%) rename ydb/tests/fq/streaming_optimize/suites/{ => pq}/ReadTopicWithMetadataWithFilter.sql (100%) rename ydb/tests/fq/streaming_optimize/suites/{ => pq}/ReadTopicWithSchema.sql (100%) rename ydb/tests/fq/streaming_optimize/suites/{ => pq}/ReadTwoTopics.sql (100%) rename ydb/tests/fq/streaming_optimize/suites/{ => pq}/ReadWriteSameTopic.sql (100%) rename ydb/tests/fq/streaming_optimize/suites/{ => pq}/ReadWriteTopic.sql (100%) rename ydb/tests/fq/streaming_optimize/suites/{ => pq}/ReadWriteTopicWithSchema.sql (100%) rename ydb/tests/fq/streaming_optimize/suites/{ => pq}/WriteTwoTopics.sql (100%) create mode 100644 ydb/tests/fq/streaming_optimize/suites/pq/default.cfg rename ydb/tests/fq/streaming_optimize/suites/{ => solomon}/ReadTopicGroupWriteToSolomon.sql (100%) rename ydb/tests/fq/streaming_optimize/suites/{ => solomon}/default.cfg (100%) create mode 100644 ydb/tests/fq/streaming_optimize/suites/watermarks/bad_column.sqlx create mode 100644 ydb/tests/fq/streaming_optimize/suites/watermarks/bad_pushdown.sqlx create mode 100644 ydb/tests/fq/streaming_optimize/suites/watermarks/default.cfg create mode 100644 ydb/tests/fq/streaming_optimize/suites/watermarks/watermarks.sql create mode 100644 ydb/tests/fq/streaming_optimize/test_sql_negative.py diff --git a/ydb/tests/fq/streaming_optimize/canondata/result.json b/ydb/tests/fq/streaming_optimize/canondata/result.json index 95c2ec8e0bdd..a6e9dbfe9651 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/result.json +++ b/ydb/tests/fq/streaming_optimize/canondata/result.json @@ -1,170 +1,236 @@ { - "test_sql_streaming.test[suites-GroupByHop-default.txt]": { + "test_sql_negative.test[watermarks-bad_column-default.txt]": [ + { + "uri": "file://test_sql_negative.test_watermarks-bad_column-default.txt_/err_file.out" + } + ], + "test_sql_negative.test[watermarks-bad_pushdown-default.txt]": [ + { + "uri": "file://test_sql_negative.test_watermarks-bad_pushdown-default.txt_/err_file.out" + } + ], + "test_sql_streaming.test[hop-GroupByHop-default.txt]": { + "Ast": { + "uri": "file://test_sql_streaming.test_hop-GroupByHop-default.txt_/ast.txt" + }, + "Plan": { + "uri": "file://test_sql_streaming.test_hop-GroupByHop-default.txt_/plan.json" + } + }, + "test_sql_streaming.test[hop-GroupByHopByStringKey-default.txt]": { + "Ast": { + "uri": "file://test_sql_streaming.test_hop-GroupByHopByStringKey-default.txt_/ast.txt" + }, + "Plan": { + "uri": "file://test_sql_streaming.test_hop-GroupByHopByStringKey-default.txt_/plan.json" + } + }, + "test_sql_streaming.test[hop-GroupByHopExprKey-default.txt]": { + "Ast": { + "uri": "file://test_sql_streaming.test_hop-GroupByHopExprKey-default.txt_/ast.txt" + }, + "Plan": { + "uri": "file://test_sql_streaming.test_hop-GroupByHopExprKey-default.txt_/plan.json" + } + }, + "test_sql_streaming.test[hop-GroupByHopListKey-default.txt]": { + "Ast": { + "uri": "file://test_sql_streaming.test_hop-GroupByHopListKey-default.txt_/ast.txt" + }, + "Plan": { + "uri": "file://test_sql_streaming.test_hop-GroupByHopListKey-default.txt_/plan.json" + } + }, + "test_sql_streaming.test[hop-GroupByHopNoKey-default.txt]": { + "Ast": { + "uri": "file://test_sql_streaming.test_hop-GroupByHopNoKey-default.txt_/ast.txt" + }, + "Plan": { + "uri": "file://test_sql_streaming.test_hop-GroupByHopNoKey-default.txt_/plan.json" + } + }, + "test_sql_streaming.test[hop-GroupByHopPercentile-default.txt]": { + "Ast": { + "uri": "file://test_sql_streaming.test_hop-GroupByHopPercentile-default.txt_/ast.txt" + }, + "Plan": { + "uri": "file://test_sql_streaming.test_hop-GroupByHopPercentile-default.txt_/plan.json" + } + }, + "test_sql_streaming.test[hop-GroupByHopTimeExtractorUnusedColumns-default.txt]": { + "Ast": { + "uri": "file://test_sql_streaming.test_hop-GroupByHopTimeExtractorUnusedColumns-default.txt_/ast.txt" + }, + "Plan": { + "uri": "file://test_sql_streaming.test_hop-GroupByHopTimeExtractorUnusedColumns-default.txt_/plan.json" + } + }, + "test_sql_streaming.test[hop-GroupByHopWithDataWatermarks-default.txt]": { "Ast": { - "uri": "file://test_sql_streaming.test_suites-GroupByHop-default.txt_/ast.txt" + "uri": "file://test_sql_streaming.test_hop-GroupByHopWithDataWatermarks-default.txt_/ast.txt" }, "Plan": { - "uri": "file://test_sql_streaming.test_suites-GroupByHop-default.txt_/plan.json" + "uri": "file://test_sql_streaming.test_hop-GroupByHopWithDataWatermarks-default.txt_/plan.json" } }, - "test_sql_streaming.test[suites-GroupByHopByStringKey-default.txt]": { + "test_sql_streaming.test[hopping_window-GroupByHoppingWindow-default.txt]": { "Ast": { - "uri": "file://test_sql_streaming.test_suites-GroupByHopByStringKey-default.txt_/ast.txt" + "uri": "file://test_sql_streaming.test_hopping_window-GroupByHoppingWindow-default.txt_/ast.txt" }, "Plan": { - "uri": "file://test_sql_streaming.test_suites-GroupByHopByStringKey-default.txt_/plan.json" + "uri": "file://test_sql_streaming.test_hopping_window-GroupByHoppingWindow-default.txt_/plan.json" } }, - "test_sql_streaming.test[suites-GroupByHopExprKey-default.txt]": { + "test_sql_streaming.test[hopping_window-GroupByHoppingWindowByStringKey-default.txt]": { "Ast": { - "uri": "file://test_sql_streaming.test_suites-GroupByHopExprKey-default.txt_/ast.txt" + "uri": "file://test_sql_streaming.test_hopping_window-GroupByHoppingWindowByStringKey-default.txt_/ast.txt" }, "Plan": { - "uri": "file://test_sql_streaming.test_suites-GroupByHopExprKey-default.txt_/plan.json" + "uri": "file://test_sql_streaming.test_hopping_window-GroupByHoppingWindowByStringKey-default.txt_/plan.json" } }, - "test_sql_streaming.test[suites-GroupByHopListKey-default.txt]": { + "test_sql_streaming.test[hopping_window-GroupByHoppingWindowExprKey-default.txt]": { "Ast": { - "uri": "file://test_sql_streaming.test_suites-GroupByHopListKey-default.txt_/ast.txt" + "uri": "file://test_sql_streaming.test_hopping_window-GroupByHoppingWindowExprKey-default.txt_/ast.txt" }, "Plan": { - "uri": "file://test_sql_streaming.test_suites-GroupByHopListKey-default.txt_/plan.json" + "uri": "file://test_sql_streaming.test_hopping_window-GroupByHoppingWindowExprKey-default.txt_/plan.json" } }, - "test_sql_streaming.test[suites-GroupByHopNoKey-default.txt]": { + "test_sql_streaming.test[hopping_window-GroupByHoppingWindowListKey-default.txt]": { "Ast": { - "uri": "file://test_sql_streaming.test_suites-GroupByHopNoKey-default.txt_/ast.txt" + "uri": "file://test_sql_streaming.test_hopping_window-GroupByHoppingWindowListKey-default.txt_/ast.txt" }, "Plan": { - "uri": "file://test_sql_streaming.test_suites-GroupByHopNoKey-default.txt_/plan.json" + "uri": "file://test_sql_streaming.test_hopping_window-GroupByHoppingWindowListKey-default.txt_/plan.json" } }, - "test_sql_streaming.test[suites-GroupByHopPercentile-default.txt]": { + "test_sql_streaming.test[hopping_window-GroupByHoppingWindowNoKey-default.txt]": { "Ast": { - "uri": "file://test_sql_streaming.test_suites-GroupByHopPercentile-default.txt_/ast.txt" + "uri": "file://test_sql_streaming.test_hopping_window-GroupByHoppingWindowNoKey-default.txt_/ast.txt" }, "Plan": { - "uri": "file://test_sql_streaming.test_suites-GroupByHopPercentile-default.txt_/plan.json" + "uri": "file://test_sql_streaming.test_hopping_window-GroupByHoppingWindowNoKey-default.txt_/plan.json" } }, - "test_sql_streaming.test[suites-GroupByHopTimeExtractorUnusedColumns-default.txt]": { + "test_sql_streaming.test[hopping_window-GroupByHoppingWindowPercentile-default.txt]": { "Ast": { - "uri": "file://test_sql_streaming.test_suites-GroupByHopTimeExtractorUnusedColumns-default.txt_/ast.txt" + "uri": "file://test_sql_streaming.test_hopping_window-GroupByHoppingWindowPercentile-default.txt_/ast.txt" }, "Plan": { - "uri": "file://test_sql_streaming.test_suites-GroupByHopTimeExtractorUnusedColumns-default.txt_/plan.json" + "uri": "file://test_sql_streaming.test_hopping_window-GroupByHoppingWindowPercentile-default.txt_/plan.json" } }, - "test_sql_streaming.test[suites-GroupByHopWithDataWatermarks-default.txt]": { + "test_sql_streaming.test[hopping_window-GroupByHoppingWindowTimeExtractorUnusedColumns-default.txt]": { "Ast": { - "uri": "file://test_sql_streaming.test_suites-GroupByHopWithDataWatermarks-default.txt_/ast.txt" + "uri": "file://test_sql_streaming.test_hopping_window-GroupByHoppingWindowTimeExtractorUnusedColumns-default.txt_/ast.txt" }, "Plan": { - "uri": "file://test_sql_streaming.test_suites-GroupByHopWithDataWatermarks-default.txt_/plan.json" + "uri": "file://test_sql_streaming.test_hopping_window-GroupByHoppingWindowTimeExtractorUnusedColumns-default.txt_/plan.json" } }, - "test_sql_streaming.test[suites-GroupByHoppingWithDataWatermarks-default.txt]": { + "test_sql_streaming.test[pq-ReadTopic-default.txt]": { "Ast": { - "uri": "file://test_sql_streaming.test_suites-GroupByHoppingWithDataWatermarks-default.txt_/ast.txt" + "uri": "file://test_sql_streaming.test_pq-ReadTopic-default.txt_/ast.txt" }, "Plan": { - "uri": "file://test_sql_streaming.test_suites-GroupByHoppingWithDataWatermarks-default.txt_/plan.json" + "uri": "file://test_sql_streaming.test_pq-ReadTopic-default.txt_/plan.json" } }, - "test_sql_streaming.test[suites-ReadTopic-default.txt]": { + "test_sql_streaming.test[pq-ReadTopicWithMetadata-default.txt]": { "Ast": { - "uri": "file://test_sql_streaming.test_suites-ReadTopic-default.txt_/ast.txt" + "uri": "file://test_sql_streaming.test_pq-ReadTopicWithMetadata-default.txt_/ast.txt" }, "Plan": { - "uri": "file://test_sql_streaming.test_suites-ReadTopic-default.txt_/plan.json" + "uri": "file://test_sql_streaming.test_pq-ReadTopicWithMetadata-default.txt_/plan.json" } }, - "test_sql_streaming.test[suites-ReadTopicGroupWriteToSolomon-default.txt]": { + "test_sql_streaming.test[pq-ReadTopicWithMetadataInsideFilter-default.txt]": { "Ast": { - "uri": "file://test_sql_streaming.test_suites-ReadTopicGroupWriteToSolomon-default.txt_/ast.txt" + "uri": "file://test_sql_streaming.test_pq-ReadTopicWithMetadataInsideFilter-default.txt_/ast.txt" }, "Plan": { - "uri": "file://test_sql_streaming.test_suites-ReadTopicGroupWriteToSolomon-default.txt_/plan.json" + "uri": "file://test_sql_streaming.test_pq-ReadTopicWithMetadataInsideFilter-default.txt_/plan.json" } }, - "test_sql_streaming.test[suites-ReadTopicWithMetadata-default.txt]": { + "test_sql_streaming.test[pq-ReadTopicWithMetadataNestedDeep-default.txt]": { "Ast": { - "uri": "file://test_sql_streaming.test_suites-ReadTopicWithMetadata-default.txt_/ast.txt" + "uri": "file://test_sql_streaming.test_pq-ReadTopicWithMetadataNestedDeep-default.txt_/ast.txt" }, "Plan": { - "uri": "file://test_sql_streaming.test_suites-ReadTopicWithMetadata-default.txt_/plan.json" + "uri": "file://test_sql_streaming.test_pq-ReadTopicWithMetadataNestedDeep-default.txt_/plan.json" } }, - "test_sql_streaming.test[suites-ReadTopicWithMetadataInsideFilter-default.txt]": { + "test_sql_streaming.test[pq-ReadTopicWithMetadataWithFilter-default.txt]": { "Ast": { - "uri": "file://test_sql_streaming.test_suites-ReadTopicWithMetadataInsideFilter-default.txt_/ast.txt" + "uri": "file://test_sql_streaming.test_pq-ReadTopicWithMetadataWithFilter-default.txt_/ast.txt" }, "Plan": { - "uri": "file://test_sql_streaming.test_suites-ReadTopicWithMetadataInsideFilter-default.txt_/plan.json" + "uri": "file://test_sql_streaming.test_pq-ReadTopicWithMetadataWithFilter-default.txt_/plan.json" } }, - "test_sql_streaming.test[suites-ReadTopicWithMetadataNestedDeep-default.txt]": { + "test_sql_streaming.test[pq-ReadTopicWithSchema-default.txt]": { "Ast": { - "uri": "file://test_sql_streaming.test_suites-ReadTopicWithMetadataNestedDeep-default.txt_/ast.txt" + "uri": "file://test_sql_streaming.test_pq-ReadTopicWithSchema-default.txt_/ast.txt" }, "Plan": { - "uri": "file://test_sql_streaming.test_suites-ReadTopicWithMetadataNestedDeep-default.txt_/plan.json" + "uri": "file://test_sql_streaming.test_pq-ReadTopicWithSchema-default.txt_/plan.json" } }, - "test_sql_streaming.test[suites-ReadTopicWithMetadataWithFilter-default.txt]": { + "test_sql_streaming.test[pq-ReadTwoTopics-default.txt]": { "Ast": { - "uri": "file://test_sql_streaming.test_suites-ReadTopicWithMetadataWithFilter-default.txt_/ast.txt" + "uri": "file://test_sql_streaming.test_pq-ReadTwoTopics-default.txt_/ast.txt" }, "Plan": { - "uri": "file://test_sql_streaming.test_suites-ReadTopicWithMetadataWithFilter-default.txt_/plan.json" + "uri": "file://test_sql_streaming.test_pq-ReadTwoTopics-default.txt_/plan.json" } }, - "test_sql_streaming.test[suites-ReadTopicWithSchema-default.txt]": { + "test_sql_streaming.test[pq-ReadWriteSameTopic-default.txt]": { "Ast": { - "uri": "file://test_sql_streaming.test_suites-ReadTopicWithSchema-default.txt_/ast.txt" + "uri": "file://test_sql_streaming.test_pq-ReadWriteSameTopic-default.txt_/ast.txt" }, "Plan": { - "uri": "file://test_sql_streaming.test_suites-ReadTopicWithSchema-default.txt_/plan.json" + "uri": "file://test_sql_streaming.test_pq-ReadWriteSameTopic-default.txt_/plan.json" } }, - "test_sql_streaming.test[suites-ReadTwoTopics-default.txt]": { + "test_sql_streaming.test[pq-ReadWriteTopic-default.txt]": { "Ast": { - "uri": "file://test_sql_streaming.test_suites-ReadTwoTopics-default.txt_/ast.txt" + "uri": "file://test_sql_streaming.test_pq-ReadWriteTopic-default.txt_/ast.txt" }, "Plan": { - "uri": "file://test_sql_streaming.test_suites-ReadTwoTopics-default.txt_/plan.json" + "uri": "file://test_sql_streaming.test_pq-ReadWriteTopic-default.txt_/plan.json" } }, - "test_sql_streaming.test[suites-ReadWriteSameTopic-default.txt]": { + "test_sql_streaming.test[pq-ReadWriteTopicWithSchema-default.txt]": { "Ast": { - "uri": "file://test_sql_streaming.test_suites-ReadWriteSameTopic-default.txt_/ast.txt" + "uri": "file://test_sql_streaming.test_pq-ReadWriteTopicWithSchema-default.txt_/ast.txt" }, "Plan": { - "uri": "file://test_sql_streaming.test_suites-ReadWriteSameTopic-default.txt_/plan.json" + "uri": "file://test_sql_streaming.test_pq-ReadWriteTopicWithSchema-default.txt_/plan.json" } }, - "test_sql_streaming.test[suites-ReadWriteTopic-default.txt]": { + "test_sql_streaming.test[pq-WriteTwoTopics-default.txt]": { "Ast": { - "uri": "file://test_sql_streaming.test_suites-ReadWriteTopic-default.txt_/ast.txt" + "uri": "file://test_sql_streaming.test_pq-WriteTwoTopics-default.txt_/ast.txt" }, "Plan": { - "uri": "file://test_sql_streaming.test_suites-ReadWriteTopic-default.txt_/plan.json" + "uri": "file://test_sql_streaming.test_pq-WriteTwoTopics-default.txt_/plan.json" } }, - "test_sql_streaming.test[suites-ReadWriteTopicWithSchema-default.txt]": { + "test_sql_streaming.test[solomon-ReadTopicGroupWriteToSolomon-default.txt]": { "Ast": { - "uri": "file://test_sql_streaming.test_suites-ReadWriteTopicWithSchema-default.txt_/ast.txt" + "uri": "file://test_sql_streaming.test_solomon-ReadTopicGroupWriteToSolomon-default.txt_/ast.txt" }, "Plan": { - "uri": "file://test_sql_streaming.test_suites-ReadWriteTopicWithSchema-default.txt_/plan.json" + "uri": "file://test_sql_streaming.test_solomon-ReadTopicGroupWriteToSolomon-default.txt_/plan.json" } }, - "test_sql_streaming.test[suites-WriteTwoTopics-default.txt]": { + "test_sql_streaming.test[watermarks-watermarks-default.txt]": { "Ast": { - "uri": "file://test_sql_streaming.test_suites-WriteTwoTopics-default.txt_/ast.txt" + "uri": "file://test_sql_streaming.test_watermarks-watermarks-default.txt_/ast.txt" }, "Plan": { - "uri": "file://test_sql_streaming.test_suites-WriteTwoTopics-default.txt_/plan.json" + "uri": "file://test_sql_streaming.test_watermarks-watermarks-default.txt_/plan.json" } } } diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_negative.test_watermarks-bad_column-default.txt_/err_file.out b/ydb/tests/fq/streaming_optimize/canondata/test_sql_negative.test_watermarks-bad_column-default.txt_/err_file.out new file mode 100644 index 000000000000..981d1c4232a9 --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_negative.test_watermarks-bad_column-default.txt_/err_file.out @@ -0,0 +1,10 @@ +Failed to execute query, invalid final status FAILED, issues: +
: Error: Query failed with code ABORTED at ISOTIME +
: Error: Run query failed: Error +
: Error: Type annotation, code: 1030 +
:4:1: Error: At function: RemovePrefixMembers, At function: RemoveSystemMembers, At function: PersistableRepr, At function: SqlProject +
:12:5: Error: At lambda +
:12:19: Error: At function: Unwrap +
:12:36: Error: At function: - +
:12:26: Error: Member not found: timestamp + diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_negative.test_watermarks-bad_pushdown-default.txt_/err_file.out b/ydb/tests/fq/streaming_optimize/canondata/test_sql_negative.test_watermarks-bad_pushdown-default.txt_/err_file.out new file mode 100644 index 000000000000..84ee45fb6765 --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_negative.test_watermarks-bad_pushdown-default.txt_/err_file.out @@ -0,0 +1,8 @@ +Failed to execute query, invalid final status FAILED, issues: +
: Error: Query failed with code ABORTED at ISOTIME +
: Error: Run query failed: Error +
: Error: Type annotation, code: 1030 +
:4:1: Error: At function: RemovePrefixMembers, At function: RemoveSystemMembers, At function: PersistableRepr, At function: SqlProject +
:6:6: Error: At function: PqReadTopic! +
:12:5: Error: Bad watermark expression + diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHop-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHop-default.txt_/ast.txt similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHop-default.txt_/ast.txt rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHop-default.txt_/ast.txt diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHop-default.txt_/plan.json b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHop-default.txt_/plan.json similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHop-default.txt_/plan.json rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHop-default.txt_/plan.json diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopByStringKey-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopByStringKey-default.txt_/ast.txt similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopByStringKey-default.txt_/ast.txt rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopByStringKey-default.txt_/ast.txt diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopByStringKey-default.txt_/plan.json b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopByStringKey-default.txt_/plan.json similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopByStringKey-default.txt_/plan.json rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopByStringKey-default.txt_/plan.json diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopExprKey-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopExprKey-default.txt_/ast.txt similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopExprKey-default.txt_/ast.txt rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopExprKey-default.txt_/ast.txt diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopExprKey-default.txt_/plan.json b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopExprKey-default.txt_/plan.json similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopExprKey-default.txt_/plan.json rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopExprKey-default.txt_/plan.json diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopListKey-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopListKey-default.txt_/ast.txt similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopListKey-default.txt_/ast.txt rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopListKey-default.txt_/ast.txt diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopListKey-default.txt_/plan.json b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopListKey-default.txt_/plan.json similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopListKey-default.txt_/plan.json rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopListKey-default.txt_/plan.json diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopNoKey-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopNoKey-default.txt_/ast.txt similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopNoKey-default.txt_/ast.txt rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopNoKey-default.txt_/ast.txt diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopNoKey-default.txt_/plan.json b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopNoKey-default.txt_/plan.json similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopNoKey-default.txt_/plan.json rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopNoKey-default.txt_/plan.json diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopPercentile-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopPercentile-default.txt_/ast.txt similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopPercentile-default.txt_/ast.txt rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopPercentile-default.txt_/ast.txt diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopPercentile-default.txt_/plan.json b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopPercentile-default.txt_/plan.json similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopPercentile-default.txt_/plan.json rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopPercentile-default.txt_/plan.json diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopTimeExtractorUnusedColumns-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopTimeExtractorUnusedColumns-default.txt_/ast.txt similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopTimeExtractorUnusedColumns-default.txt_/ast.txt rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopTimeExtractorUnusedColumns-default.txt_/ast.txt diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopTimeExtractorUnusedColumns-default.txt_/plan.json b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopTimeExtractorUnusedColumns-default.txt_/plan.json similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopTimeExtractorUnusedColumns-default.txt_/plan.json rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopTimeExtractorUnusedColumns-default.txt_/plan.json diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopWithDataWatermarks-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopWithDataWatermarks-default.txt_/ast.txt similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopWithDataWatermarks-default.txt_/ast.txt rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopWithDataWatermarks-default.txt_/ast.txt diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopWithDataWatermarks-default.txt_/plan.json b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopWithDataWatermarks-default.txt_/plan.json similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopWithDataWatermarks-default.txt_/plan.json rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopWithDataWatermarks-default.txt_/plan.json diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindow-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindow-default.txt_/ast.txt new file mode 100644 index 000000000000..2f8a78368260 --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindow-default.txt_/ast.txt @@ -0,0 +1,54 @@ +( +(let $1 (Configure! world (DataSource '"config") '"DqEngine" '"force")) +(let $2 (DataSource '"dq" '"$all")) +(let $3 (Configure! $1 $2 '"Attr" '"maxtasksperstage" '"2")) +(let $4 (Configure! $3 $2 '"Attr" '"watermarksmode" '"default")) +(let $5 (Configure! $4 $2 '"Attr" '"computeactortype" '"async")) +(let $6 (Configure! $5 (DataSource '"pq" '"$all") '"Attr" '"consumer" '"test_client")) +(let $7 (DataSource '"pq" '"pq")) +(let $8 '('('"PartitionsCount" '"1"))) +(let $9 (DataType 'String)) +(let $10 '('"k" (OptionalType $9))) +(let $11 (DataType 'Uint64)) +(let $12 (OptionalType $11)) +(let $13 '('"t" $12)) +(let $14 (StructType $10 $13 '('"v" $12))) +(let $15 (PqTopic '"pq" '"local" '"test_topic_input" $8 '() $14)) +(let $16 '('"k" '"t" '"v")) +(let $17 '('"Endpoint" '"")) +(let $18 '('"SharedReading" '"1")) +(let $19 '('"UseSsl" '"1")) +(let $20 '('('"Consumer" '"test_client") $17 $18 '('"ReconnectPeriod" '"") '('"Format" '"json_each_row") '('"ReadGroup" '"fqrun") $19 '('"WatermarksEnable" '"1") '('"WatermarksGranularityUs" '"1000000") '('"WatermarksLateArrivalDelayUs" '"5000000"))) +(let $21 (SecureParam '"cluster:default_pq")) +(let $22 (DqPqTopicSource $6 $15 $16 $20 $21 '"" $14 '"")) +(let $23 (DqStage '((DqSource $7 $22)) (lambda '($27) (block '( + (let $28 '('('"format" '"json_each_row") '('"formatSettings" '('('"data.datetime.formatname" '"POSIX") '('"data.timestamp.formatname" '"POSIX"))) '('"settings" '($18)))) + (let $29 (DqSourceWideWrap $27 $7 $14 $28)) + (return (NarrowMap $29 (lambda '($30 $31 $32) (AsStruct '('"k" $30) '('"t" $31) '('"v" $32))))) +))) '('('"_logical_id" '0)))) +(let $24 (DataSink '"pq" '"pq")) +(let $25 (PqTopic '"pq" '"local" '"test_topic_output" $8 '() (StructType '('"Data" $9)))) +(let $26 (DqPqTopicSink $25 '($17 $19) $21)) +(return (Commit! (DqQuery! $6 '((DqStage '((DqCnHashShuffle (TDqOutput $23 '0) '('"k"))) (lambda '($33) (block '( + (let $34 '('"strict")) + (let $35 (lambda '($46) $46)) + (let $36 (MultiHoppingCore (FromFlow $33) (lambda '($37) (Member $37 '"k")) (lambda '($38) (FlatMap (Member (SafeCast $38 (StructType $13)) '"t") (lambda '($39) (block '( + (let $40 '($11 '"" '"1")) + (let $41 (CallableType '() '((OptionalType (DataType 'Timestamp))) $40)) + (let $42 (Udf '"DateTime2.FromMilliseconds" (Void) (VoidType) '"" $41 (VoidType) '"" '('('"blocks") $34))) + (return (Apply $42 $39)) + ))))) (Interval '"5000") (Interval '"10000") (Interval '"5000000") 'true (lambda '($43) (AsStruct '('Sum0 (Member $43 '"v")))) (lambda '($44 $45) (AsStruct '('Sum0 (AggrAdd (Member $44 '"v") (Member $45 'Sum0))))) $35 $35 (lambda '($47 $48) (AsStruct '('Sum0 (AggrAdd (Member $47 'Sum0) (Member $48 'Sum0))))) (lambda '($49 $50 $51) (AsStruct '('Sum0 (Member $50 'Sum0)) '('"group0" $51) '('"k" $49))) '"1")) + (return (FlatMap (ExtractMembers $36 '('Sum0 '"k")) (lambda '($52) (block '( + (let $53 (ResourceType '"Yson2.Node")) + (let $54 '($53 '"" '"1")) + (let $55 (CallableType '() '((DataType 'Yson)) $54)) + (let $56 '($34)) + (let $57 (Udf '"Yson2.SerializeText" (Void) (VoidType) '"" $55 (VoidType) '"" $56)) + (let $58 (StructType $10 '('"sum" $12))) + (let $59 (TupleType (TupleType $58) (StructType) (TupleType))) + (let $60 (CallableType '() '($53) '($58))) + (let $61 (Udf '"Yson2.From" (Void) $59 '"" $60 (VoidType) '"" $56)) + (return (Just (AsStruct '('"column0" (Apply $57 (Apply $61 (AsStruct '('"k" (Member $52 '"k")) '('"sum" (Member $52 'Sum0))))))))) + ))))) +))) '('('"_logical_id" '0)) '((DqSink '0 $24 $26))))) $24)) +) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindow-default.txt_/plan.json b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindow-default.txt_/plan.json new file mode 100644 index 000000000000..48462f24e7c6 --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindow-default.txt_/plan.json @@ -0,0 +1,106 @@ +{ + "Detailed" : { + "Operations" : [ + { + "Id" : 5, + "Name" : "DqStage", + "Streams" : { + "Program" : [ + { + "Name" : "DqSourceWideWrap" + }, + { + "Name" : "NarrowMap" + } + ] + } + }, + { + "Id" : 3, + "Name" : "DqStage", + "Streams" : { + "Program" : [ + { + "Name" : "FromFlow" + }, + { + "Name" : "MultiHoppingCore" + }, + { + "Name" : "ExtractMembers" + }, + { + "Name" : "FlatMap" + } + ] + }, + "DependsOn" : [ + 5 + ] + }, + { + "Id" : 2, + "Name" : "DqQuery!", + "DependsOn" : [ + 3 + ] + }, + { + "Id" : 1, + "Name" : "Commit!", + "DependsOn" : [ + 2 + ] + } + ], + "OperationRoot" : 1, + "Providers" : [ ], + "OperationStats" : { + "Commit!" : 1, + "DqQuery!" : 1, + "DqStage" : 2 + } + }, + "Basic" : { + "nodes" : [ + { + "id" : 5, + "level" : 1, + "name" : "DqStage #5", + "type" : "op" + }, + { + "id" : 3, + "level" : 2, + "name" : "DqStage #3", + "type" : "op" + }, + { + "id" : 2, + "level" : 3, + "name" : "DqQuery!", + "type" : "op" + }, + { + "id" : 1, + "level" : 4, + "name" : "Commit!", + "type" : "op" + } + ], + "links" : [ + { + "source" : 5, + "target" : 3 + }, + { + "source" : 3, + "target" : 2 + }, + { + "source" : 2, + "target" : 1 + } + ] + } +} \ No newline at end of file diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowByStringKey-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowByStringKey-default.txt_/ast.txt new file mode 100644 index 000000000000..2f8a78368260 --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowByStringKey-default.txt_/ast.txt @@ -0,0 +1,54 @@ +( +(let $1 (Configure! world (DataSource '"config") '"DqEngine" '"force")) +(let $2 (DataSource '"dq" '"$all")) +(let $3 (Configure! $1 $2 '"Attr" '"maxtasksperstage" '"2")) +(let $4 (Configure! $3 $2 '"Attr" '"watermarksmode" '"default")) +(let $5 (Configure! $4 $2 '"Attr" '"computeactortype" '"async")) +(let $6 (Configure! $5 (DataSource '"pq" '"$all") '"Attr" '"consumer" '"test_client")) +(let $7 (DataSource '"pq" '"pq")) +(let $8 '('('"PartitionsCount" '"1"))) +(let $9 (DataType 'String)) +(let $10 '('"k" (OptionalType $9))) +(let $11 (DataType 'Uint64)) +(let $12 (OptionalType $11)) +(let $13 '('"t" $12)) +(let $14 (StructType $10 $13 '('"v" $12))) +(let $15 (PqTopic '"pq" '"local" '"test_topic_input" $8 '() $14)) +(let $16 '('"k" '"t" '"v")) +(let $17 '('"Endpoint" '"")) +(let $18 '('"SharedReading" '"1")) +(let $19 '('"UseSsl" '"1")) +(let $20 '('('"Consumer" '"test_client") $17 $18 '('"ReconnectPeriod" '"") '('"Format" '"json_each_row") '('"ReadGroup" '"fqrun") $19 '('"WatermarksEnable" '"1") '('"WatermarksGranularityUs" '"1000000") '('"WatermarksLateArrivalDelayUs" '"5000000"))) +(let $21 (SecureParam '"cluster:default_pq")) +(let $22 (DqPqTopicSource $6 $15 $16 $20 $21 '"" $14 '"")) +(let $23 (DqStage '((DqSource $7 $22)) (lambda '($27) (block '( + (let $28 '('('"format" '"json_each_row") '('"formatSettings" '('('"data.datetime.formatname" '"POSIX") '('"data.timestamp.formatname" '"POSIX"))) '('"settings" '($18)))) + (let $29 (DqSourceWideWrap $27 $7 $14 $28)) + (return (NarrowMap $29 (lambda '($30 $31 $32) (AsStruct '('"k" $30) '('"t" $31) '('"v" $32))))) +))) '('('"_logical_id" '0)))) +(let $24 (DataSink '"pq" '"pq")) +(let $25 (PqTopic '"pq" '"local" '"test_topic_output" $8 '() (StructType '('"Data" $9)))) +(let $26 (DqPqTopicSink $25 '($17 $19) $21)) +(return (Commit! (DqQuery! $6 '((DqStage '((DqCnHashShuffle (TDqOutput $23 '0) '('"k"))) (lambda '($33) (block '( + (let $34 '('"strict")) + (let $35 (lambda '($46) $46)) + (let $36 (MultiHoppingCore (FromFlow $33) (lambda '($37) (Member $37 '"k")) (lambda '($38) (FlatMap (Member (SafeCast $38 (StructType $13)) '"t") (lambda '($39) (block '( + (let $40 '($11 '"" '"1")) + (let $41 (CallableType '() '((OptionalType (DataType 'Timestamp))) $40)) + (let $42 (Udf '"DateTime2.FromMilliseconds" (Void) (VoidType) '"" $41 (VoidType) '"" '('('"blocks") $34))) + (return (Apply $42 $39)) + ))))) (Interval '"5000") (Interval '"10000") (Interval '"5000000") 'true (lambda '($43) (AsStruct '('Sum0 (Member $43 '"v")))) (lambda '($44 $45) (AsStruct '('Sum0 (AggrAdd (Member $44 '"v") (Member $45 'Sum0))))) $35 $35 (lambda '($47 $48) (AsStruct '('Sum0 (AggrAdd (Member $47 'Sum0) (Member $48 'Sum0))))) (lambda '($49 $50 $51) (AsStruct '('Sum0 (Member $50 'Sum0)) '('"group0" $51) '('"k" $49))) '"1")) + (return (FlatMap (ExtractMembers $36 '('Sum0 '"k")) (lambda '($52) (block '( + (let $53 (ResourceType '"Yson2.Node")) + (let $54 '($53 '"" '"1")) + (let $55 (CallableType '() '((DataType 'Yson)) $54)) + (let $56 '($34)) + (let $57 (Udf '"Yson2.SerializeText" (Void) (VoidType) '"" $55 (VoidType) '"" $56)) + (let $58 (StructType $10 '('"sum" $12))) + (let $59 (TupleType (TupleType $58) (StructType) (TupleType))) + (let $60 (CallableType '() '($53) '($58))) + (let $61 (Udf '"Yson2.From" (Void) $59 '"" $60 (VoidType) '"" $56)) + (return (Just (AsStruct '('"column0" (Apply $57 (Apply $61 (AsStruct '('"k" (Member $52 '"k")) '('"sum" (Member $52 'Sum0))))))))) + ))))) +))) '('('"_logical_id" '0)) '((DqSink '0 $24 $26))))) $24)) +) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowByStringKey-default.txt_/plan.json b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowByStringKey-default.txt_/plan.json new file mode 100644 index 000000000000..48462f24e7c6 --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowByStringKey-default.txt_/plan.json @@ -0,0 +1,106 @@ +{ + "Detailed" : { + "Operations" : [ + { + "Id" : 5, + "Name" : "DqStage", + "Streams" : { + "Program" : [ + { + "Name" : "DqSourceWideWrap" + }, + { + "Name" : "NarrowMap" + } + ] + } + }, + { + "Id" : 3, + "Name" : "DqStage", + "Streams" : { + "Program" : [ + { + "Name" : "FromFlow" + }, + { + "Name" : "MultiHoppingCore" + }, + { + "Name" : "ExtractMembers" + }, + { + "Name" : "FlatMap" + } + ] + }, + "DependsOn" : [ + 5 + ] + }, + { + "Id" : 2, + "Name" : "DqQuery!", + "DependsOn" : [ + 3 + ] + }, + { + "Id" : 1, + "Name" : "Commit!", + "DependsOn" : [ + 2 + ] + } + ], + "OperationRoot" : 1, + "Providers" : [ ], + "OperationStats" : { + "Commit!" : 1, + "DqQuery!" : 1, + "DqStage" : 2 + } + }, + "Basic" : { + "nodes" : [ + { + "id" : 5, + "level" : 1, + "name" : "DqStage #5", + "type" : "op" + }, + { + "id" : 3, + "level" : 2, + "name" : "DqStage #3", + "type" : "op" + }, + { + "id" : 2, + "level" : 3, + "name" : "DqQuery!", + "type" : "op" + }, + { + "id" : 1, + "level" : 4, + "name" : "Commit!", + "type" : "op" + } + ], + "links" : [ + { + "source" : 5, + "target" : 3 + }, + { + "source" : 3, + "target" : 2 + }, + { + "source" : 2, + "target" : 1 + } + ] + } +} \ No newline at end of file diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowExprKey-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowExprKey-default.txt_/ast.txt new file mode 100644 index 000000000000..73c0a61b6b0b --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowExprKey-default.txt_/ast.txt @@ -0,0 +1,52 @@ +( +(let $1 (Configure! world (DataSource '"config") '"DqEngine" '"force")) +(let $2 (DataSource '"dq" '"$all")) +(let $3 (Configure! $1 $2 '"Attr" '"maxtasksperstage" '2)) +(let $4 (Configure! $3 $2 '"Attr" '"watermarksmode" '"default")) +(let $5 (Configure! $4 $2 '"Attr" '"computeactortype" '"async")) +(let $6 (Configure! $5 (DataSource '"pq" '"$all") '"Attr" '"consumer" '"test_client")) +(let $7 (DataSource '"pq" '"pq")) +(let $8 '('('"PartitionsCount" '"1"))) +(let $9 (DataType 'Uint64)) +(let $10 (OptionalType $9)) +(let $11 '('"t" $10)) +(let $12 (StructType '('"k" $10) $11 '('"v" $10))) +(let $13 (PqTopic '"pq" '"local" '"test_topic_input" $8 '() $12)) +(let $14 '('"k" '"t" '"v")) +(let $15 '('"Endpoint" '"")) +(let $16 '('"SharedReading" '"1")) +(let $17 '('"UseSsl" '"1")) +(let $18 '('('"Consumer" '"test_client") $15 $16 '('"ReconnectPeriod" '"") '('"Format" '"json_each_row") '('"ReadGroup" '"fqrun") $17 '('"WatermarksEnable" '"1") '('"WatermarksGranularityUs" '"1000000") '('"WatermarksLateArrivalDelayUs" '"5000000"))) +(let $19 (SecureParam '"cluster:default_pq")) +(let $20 (DqPqTopicSource $6 $13 $14 $18 $19 '"" $12 '"")) +(let $21 (DqStage '((DqSource $7 $20)) (lambda '($25) (block '( + (let $26 '('('"format" '"json_each_row") '('"formatSettings" '('('"data.datetime.formatname" '"POSIX") '('"data.timestamp.formatname" '"POSIX"))) '('"settings" '($16)))) + (let $27 (DqSourceWideWrap $25 $7 $12 $26)) + (return (FlatMap (NarrowMap $27 (lambda '($28 $29 $30) (AsStruct '('"k" $28) '('"t" $29) '('"v" $30)))) (lambda '($31) (Just (AsStruct '('group0 (Min (Member $31 '"k") (Just (Uint64 '2)))) '('"t" (Member $31 '"t")) '('"v" (Member $31 '"v"))))))) +))) '('('"_logical_id" '0)))) +(let $22 (DataSink '"pq" '"pq")) +(let $23 (PqTopic '"pq" '"local" '"test_topic_output" $8 '() (StructType '('"Data" (DataType 'String))))) +(let $24 (DqPqTopicSink $23 '($15 $17) $19)) +(return (Commit! (DqQuery! $6 '((DqStage '((DqCnHashShuffle (TDqOutput $21 '0) '('group0))) (lambda '($32) (block '( + (let $33 '('"strict")) + (let $34 (lambda '($45) $45)) + (let $35 (MultiHoppingCore (FromFlow $32) (lambda '($36) (Member $36 'group0)) (lambda '($37) (FlatMap (Member (SafeCast $37 (StructType $11)) '"t") (lambda '($38) (block '( + (let $39 '($9 '"" '"1")) + (let $40 (CallableType '() '((OptionalType (DataType 'Timestamp))) $39)) + (let $41 (Udf '"DateTime2.FromMilliseconds" (Void) (VoidType) '"" $40 (VoidType) '"" '('('"blocks") $33))) + (return (Apply $41 $38)) + ))))) (Interval '"5000") (Interval '"10000") (Interval '"5000000") 'true (lambda '($42) (AsStruct '('Sum0 (Member $42 '"v")))) (lambda '($43 $44) (AsStruct '('Sum0 (AggrAdd (Member $43 '"v") (Member $44 'Sum0))))) $34 $34 (lambda '($46 $47) (AsStruct '('Sum0 (AggrAdd (Member $46 'Sum0) (Member $47 'Sum0))))) (lambda '($48 $49 $50) (AsStruct '('Sum0 (Member $49 'Sum0)) '('group0 $48) '('"group1" $50))) '"1")) + (return (FlatMap (ExtractMembers $35 '('Sum0)) (lambda '($51) (block '( + (let $52 (ResourceType '"Yson2.Node")) + (let $53 '($52 '"" '"1")) + (let $54 (CallableType '() '((DataType 'Yson)) $53)) + (let $55 '($33)) + (let $56 (Udf '"Yson2.SerializeText" (Void) (VoidType) '"" $54 (VoidType) '"" $55)) + (let $57 (StructType '('"sum" $10))) + (let $58 (TupleType (TupleType $57) (StructType) (TupleType))) + (let $59 (CallableType '() '($52) '($57))) + (let $60 (Udf '"Yson2.From" (Void) $58 '"" $59 (VoidType) '"" $55)) + (return (Just (AsStruct '('"column0" (Apply $56 (Apply $60 (AsStruct '('"sum" (Member $51 'Sum0))))))))) + ))))) +))) '('('"_logical_id" '0)) '((DqSink '0 $22 $24))))) $22)) +) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHoppingWithDataWatermarks-default.txt_/plan.json b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowExprKey-default.txt_/plan.json similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHoppingWithDataWatermarks-default.txt_/plan.json rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowExprKey-default.txt_/plan.json diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowListKey-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowListKey-default.txt_/ast.txt new file mode 100644 index 000000000000..48f9cd9fa2c1 --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowListKey-default.txt_/ast.txt @@ -0,0 +1,55 @@ +( +(let $1 (Configure! world (DataSource '"config") '"DqEngine" '"force")) +(let $2 (DataSource '"dq" '"$all")) +(let $3 (Configure! $1 $2 '"Attr" '"maxtasksperstage" '"2")) +(let $4 (Configure! $3 $2 '"Attr" '"watermarksmode" '"default")) +(let $5 (Configure! $4 $2 '"Attr" '"computeactortype" '"async")) +(let $6 (Configure! $5 (DataSource '"pq" '"$all") '"Attr" '"consumer" '"test_client")) +(let $7 (DataSource '"pq" '"pq")) +(let $8 '('('"PartitionsCount" '1))) +(let $9 (DataType 'Uint64)) +(let $10 (OptionalType $9)) +(let $11 '('"t" $10)) +(let $12 (StructType '('"k" $10) $11 '('"v" $10))) +(let $13 (PqTopic '"pq" '"local" '"test_topic_input" $8 '() $12)) +(let $14 '('"k" '"t" '"v")) +(let $15 '('"Endpoint" '"")) +(let $16 '('"SharedReading" '1)) +(let $17 '('"UseSsl" '1)) +(let $18 '('('"Consumer" '"test_client") $15 $16 '('"ReconnectPeriod" '"") '('"Format" '"json_each_row") '('"ReadGroup" '"fqrun") $17 '('"WatermarksEnable" '1) '('"WatermarksGranularityUs" '"1000000") '('"WatermarksLateArrivalDelayUs" '"5000000"))) +(let $19 (SecureParam '"cluster:default_pq")) +(let $20 (DqPqTopicSource $6 $13 $14 $18 $19 '"" $12 '"")) +(let $21 (DqStage '((DqSource $7 $20)) (lambda '($25) (block '( + (let $26 '('('"format" '"json_each_row") '('"formatSettings" '('('"data.datetime.formatname" '"POSIX") '('"data.timestamp.formatname" '"POSIX"))) '('"settings" '($16)))) + (let $27 (DqSourceWideWrap $25 $7 $12 $26)) + (return (FlatMap (NarrowMap $27 (lambda '($28 $29 $30) (AsStruct '('"k" $28) '('"t" $29) '('"v" $30)))) (lambda '($31) (block '( + (let $32 (Member $31 '"k")) + (return (Just (AsStruct '('group0 (StablePickle (AsList (Just (Uint64 '1)) $32))) '('"k" $32) '('"t" (Member $31 '"t")) '('"v" (Member $31 '"v"))))) + ))))) +))) '('('"_logical_id" '0)))) +(let $22 (DataSink '"pq" '"pq")) +(let $23 (PqTopic '"pq" '"local" '"test_topic_output" $8 '() (StructType '('"Data" (DataType 'String))))) +(let $24 (DqPqTopicSink $23 '($15 $17) $19)) +(return (Commit! (DqQuery! $6 '((DqStage '((DqCnHashShuffle (TDqOutput $21 '0) '('group0 '"k"))) (lambda '($33) (block '( + (let $34 '('"strict")) + (let $35 (lambda '($46) $46)) + (let $36 (MultiHoppingCore (FromFlow $33) (lambda '($37) '((Member $37 'group0) (Member $37 '"k"))) (lambda '($38) (FlatMap (Member (SafeCast $38 (StructType $11)) '"t") (lambda '($39) (block '( + (let $40 '($9 '"" '1)) + (let $41 (CallableType '() '((OptionalType (DataType 'Timestamp))) $40)) + (let $42 (Udf '"DateTime2.FromMilliseconds" (Void) (VoidType) '"" $41 (VoidType) '"" '('('"blocks") $34))) + (return (Apply $42 $39)) + ))))) (Interval '"5000") (Interval '"10000") (Interval '"5000000") 'true (lambda '($43) (AsStruct '('Sum0 (Member $43 '"v")))) (lambda '($44 $45) (AsStruct '('Sum0 (AggrAdd (Member $44 '"v") (Member $45 'Sum0))))) $35 $35 (lambda '($47 $48) (AsStruct '('Sum0 (AggrAdd (Member $47 'Sum0) (Member $48 'Sum0))))) (lambda '($49 $50 $51) (AsStruct '('Sum0 (Member $50 'Sum0)) '('group0 (Nth $49 '0)) '('"group1" $51) '('"k" (Nth $49 '1)))) '1)) + (return (FlatMap (ExtractMembers $36 '('Sum0)) (lambda '($52) (block '( + (let $53 (ResourceType '"Yson2.Node")) + (let $54 '($53 '"" '1)) + (let $55 (CallableType '() '((DataType 'Yson)) $54)) + (let $56 '($34)) + (let $57 (Udf '"Yson2.SerializeText" (Void) (VoidType) '"" $55 (VoidType) '"" $56)) + (let $58 (StructType '('"sum" $10))) + (let $59 (TupleType (TupleType $58) (StructType) (TupleType))) + (let $60 (CallableType '() '($53) '($58))) + (let $61 (Udf '"Yson2.From" (Void) $59 '"" $60 (VoidType) '"" $56)) + (return (Just (AsStruct '('"column0" (Apply $57 (Apply $61 (AsStruct '('"sum" (Member $52 'Sum0))))))))) + ))))) +))) '('('"_logical_id" '0)) '((DqSink '0 $22 $24))))) $22)) +) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowListKey-default.txt_/plan.json b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowListKey-default.txt_/plan.json new file mode 100644 index 000000000000..1749e7dc0f33 --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowListKey-default.txt_/plan.json @@ -0,0 +1,109 @@ +{ + "Detailed" : { + "Operations" : [ + { + "Id" : 5, + "Name" : "DqStage", + "Streams" : { + "Program" : [ + { + "Name" : "DqSourceWideWrap" + }, + { + "Name" : "NarrowMap" + }, + { + "Name" : "FlatMap" + } + ] + } + }, + { + "Id" : 3, + "Name" : "DqStage", + "Streams" : { + "Program" : [ + { + "Name" : "FromFlow" + }, + { + "Name" : "MultiHoppingCore" + }, + { + "Name" : "ExtractMembers" + }, + { + "Name" : "FlatMap" + } + ] + }, + "DependsOn" : [ + 5 + ] + }, + { + "Id" : 2, + "Name" : "DqQuery!", + "DependsOn" : [ + 3 + ] + }, + { + "Id" : 1, + "Name" : "Commit!", + "DependsOn" : [ + 2 + ] + } + ], + "OperationRoot" : 1, + "Providers" : [ ], + "OperationStats" : { + "Commit!" : 1, + "DqQuery!" : 1, + "DqStage" : 2 + } + }, + "Basic" : { + "nodes" : [ + { + "id" : 5, + "level" : 1, + "name" : "DqStage #5", + "type" : "op" + }, + { + "id" : 3, + "level" : 2, + "name" : "DqStage #3", + "type" : "op" + }, + { + "id" : 2, + "level" : 3, + "name" : "DqQuery!", + "type" : "op" + }, + { + "id" : 1, + "level" : 4, + "name" : "Commit!", + "type" : "op" + } + ], + "links" : [ + { + "source" : 5, + "target" : 3 + }, + { + "source" : 3, + "target" : 2 + }, + { + "source" : 2, + "target" : 1 + } + ] + } +} \ No newline at end of file diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowNoKey-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowNoKey-default.txt_/ast.txt new file mode 100644 index 000000000000..fd18956704bc --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowNoKey-default.txt_/ast.txt @@ -0,0 +1,51 @@ +( +(let $1 (Configure! world (DataSource '"config") '"DqEngine" '"force")) +(let $2 (DataSource '"dq" '"$all")) +(let $3 (Configure! $1 $2 '"Attr" '"maxtasksperstage" '"2")) +(let $4 (Configure! $3 $2 '"Attr" '"watermarksmode" '"default")) +(let $5 (Configure! $4 $2 '"Attr" '"computeactortype" '"async")) +(let $6 (Configure! $5 (DataSource '"pq" '"$all") '"Attr" '"consumer" '"test_client")) +(let $7 (DataSource '"pq" '"pq")) +(let $8 '('('"PartitionsCount" '"1"))) +(let $9 (DataType 'Uint64)) +(let $10 (OptionalType $9)) +(let $11 '('"t" $10)) +(let $12 (StructType $11 '('"v" $10))) +(let $13 (PqTopic '"pq" '"local" '"test_topic_input" $8 '() $12)) +(let $14 '('"Endpoint" '"")) +(let $15 '('"SharedReading" '"1")) +(let $16 '('"UseSsl" '"1")) +(let $17 '('('"Consumer" '"test_client") $14 $15 '('"ReconnectPeriod" '"") '('"Format" '"json_each_row") '('"ReadGroup" '"fqrun") $16 '('"WatermarksEnable" '"1") '('"WatermarksGranularityUs" '"1000000") '('"WatermarksLateArrivalDelayUs" '"5000000"))) +(let $18 (SecureParam '"cluster:default_pq")) +(let $19 (DqPqTopicSource $6 $13 '('"t" '"v") $17 $18 '"" $12 '"")) +(let $20 (DqStage '((DqSource $7 $19)) (lambda '($24) (block '( + (let $25 '('('"format" '"json_each_row") '('"formatSettings" '('('"data.datetime.formatname" '"POSIX") '('"data.timestamp.formatname" '"POSIX"))) '('"settings" '($15)))) + (let $26 (DqSourceWideWrap $24 $7 $12 $25)) + (return (NarrowMap $26 (lambda '($27 $28) (AsStruct '('"t" $27) '('"v" $28))))) +))) '('('"_logical_id" '0)))) +(let $21 (DataSink '"pq" '"pq")) +(let $22 (PqTopic '"pq" '"local" '"test_topic_output" $8 '() (StructType '('"Data" (DataType 'String))))) +(let $23 (DqPqTopicSink $22 '($14 $16) $18)) +(return (Commit! (DqQuery! $6 '((DqStage '((DqCnUnionAll (TDqOutput $20 '0))) (lambda '($29) (block '( + (let $30 '('"strict")) + (let $31 (lambda '($42) $42)) + (let $32 (MultiHoppingCore (FromFlow $29) (lambda '($33) (Uint32 '0)) (lambda '($34) (FlatMap (Member (SafeCast $34 (StructType $11)) '"t") (lambda '($35) (block '( + (let $36 '($9 '"" '"1")) + (let $37 (CallableType '() '((OptionalType (DataType 'Timestamp))) $36)) + (let $38 (Udf '"DateTime2.FromMilliseconds" (Void) (VoidType) '"" $37 (VoidType) '"" '('('"blocks") $30))) + (return (Apply $38 $35)) + ))))) (Interval '"5000") (Interval '"10000") (Interval '"5000000") 'true (lambda '($39) (AsStruct '('Sum0 (Member $39 '"v")))) (lambda '($40 $41) (AsStruct '('Sum0 (AggrAdd (Member $40 '"v") (Member $41 'Sum0))))) $31 $31 (lambda '($43 $44) (AsStruct '('Sum0 (AggrAdd (Member $43 'Sum0) (Member $44 'Sum0))))) (lambda '($45 $46 $47) (AsStruct '('Sum0 (Member $46 'Sum0)) '('"group0" $47))) '"1")) + (return (FlatMap (ExtractMembers $32 '('Sum0)) (lambda '($48) (block '( + (let $49 (ResourceType '"Yson2.Node")) + (let $50 '($49 '"" '"1")) + (let $51 (CallableType '() '((DataType 'Yson)) $50)) + (let $52 '($30)) + (let $53 (Udf '"Yson2.SerializeText" (Void) (VoidType) '"" $51 (VoidType) '"" $52)) + (let $54 (StructType '('"sum" $10))) + (let $55 (TupleType (TupleType $54) (StructType) (TupleType))) + (let $56 (CallableType '() '($49) '($54))) + (let $57 (Udf '"Yson2.From" (Void) $55 '"" $56 (VoidType) '"" $52)) + (return (Just (AsStruct '('"column0" (Apply $53 (Apply $57 (AsStruct '('"sum" (Member $48 'Sum0))))))))) + ))))) +))) '('('"_logical_id" '0)) '((DqSink '0 $21 $23))))) $21)) +) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowNoKey-default.txt_/plan.json b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowNoKey-default.txt_/plan.json new file mode 100644 index 000000000000..48462f24e7c6 --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowNoKey-default.txt_/plan.json @@ -0,0 +1,106 @@ +{ + "Detailed" : { + "Operations" : [ + { + "Id" : 5, + "Name" : "DqStage", + "Streams" : { + "Program" : [ + { + "Name" : "DqSourceWideWrap" + }, + { + "Name" : "NarrowMap" + } + ] + } + }, + { + "Id" : 3, + "Name" : "DqStage", + "Streams" : { + "Program" : [ + { + "Name" : "FromFlow" + }, + { + "Name" : "MultiHoppingCore" + }, + { + "Name" : "ExtractMembers" + }, + { + "Name" : "FlatMap" + } + ] + }, + "DependsOn" : [ + 5 + ] + }, + { + "Id" : 2, + "Name" : "DqQuery!", + "DependsOn" : [ + 3 + ] + }, + { + "Id" : 1, + "Name" : "Commit!", + "DependsOn" : [ + 2 + ] + } + ], + "OperationRoot" : 1, + "Providers" : [ ], + "OperationStats" : { + "Commit!" : 1, + "DqQuery!" : 1, + "DqStage" : 2 + } + }, + "Basic" : { + "nodes" : [ + { + "id" : 5, + "level" : 1, + "name" : "DqStage #5", + "type" : "op" + }, + { + "id" : 3, + "level" : 2, + "name" : "DqStage #3", + "type" : "op" + }, + { + "id" : 2, + "level" : 3, + "name" : "DqQuery!", + "type" : "op" + }, + { + "id" : 1, + "level" : 4, + "name" : "Commit!", + "type" : "op" + } + ], + "links" : [ + { + "source" : 5, + "target" : 3 + }, + { + "source" : 3, + "target" : 2 + }, + { + "source" : 2, + "target" : 1 + } + ] + } +} \ No newline at end of file diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowPercentile-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowPercentile-default.txt_/ast.txt new file mode 100644 index 000000000000..4d178658ba0f --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowPercentile-default.txt_/ast.txt @@ -0,0 +1,97 @@ +( +(let $1 (Configure! world (DataSource '"config") '"DqEngine" '"force")) +(let $2 (DataSource '"dq" '"$all")) +(let $3 (Configure! $1 $2 '"Attr" '"maxtasksperstage" '"2")) +(let $4 (Configure! $3 $2 '"Attr" '"watermarksmode" '"default")) +(let $5 (Configure! $4 $2 '"Attr" '"computeactortype" '"async")) +(let $6 (Configure! $5 (DataSource '"pq" '"$all") '"Attr" '"consumer" '"test_client")) +(let $7 (DataSource '"pq" '"pq")) +(let $8 '('('"PartitionsCount" '1))) +(let $9 (DataType 'Uint64)) +(let $10 (OptionalType $9)) +(let $11 '('"t" $10)) +(let $12 (StructType $11 '('"v" $10))) +(let $13 (PqTopic '"pq" '"local" '"test_topic_input" $8 '() $12)) +(let $14 '('"Endpoint" '"")) +(let $15 '('"SharedReading" '1)) +(let $16 '('"UseSsl" '1)) +(let $17 '('('"Consumer" '"test_client") $14 $15 '('"ReconnectPeriod" '"") '('"Format" '"json_each_row") '('"ReadGroup" '"fqrun") $16 '('"WatermarksEnable" '1) '('"WatermarksGranularityUs" '"1000000") '('"WatermarksLateArrivalDelayUs" '"5000000"))) +(let $18 (SecureParam '"cluster:default_pq")) +(let $19 (DqPqTopicSource $6 $13 '('"t" '"v") $17 $18 '"" $12 '"")) +(let $20 (DqStage '((DqSource $7 $19)) (lambda '($25) (block '( + (let $26 '('('"format" '"json_each_row") '('"formatSettings" '('('"data.datetime.formatname" '"POSIX") '('"data.timestamp.formatname" '"POSIX"))) '('"settings" '($15)))) + (let $27 (DqSourceWideWrap $25 $7 $12 $26)) + (return (NarrowMap $27 (lambda '($28 $29) (AsStruct '('"t" $28) '('"v" $29))))) +))) '('('"_logical_id" '0)))) +(let $21 (DataType 'String)) +(let $22 (DataSink '"pq" '"pq")) +(let $23 (PqTopic '"pq" '"local" '"test_topic_output" $8 '() (StructType '('"Data" $21)))) +(let $24 (DqPqTopicSink $23 '($14 $16) $18)) +(return (Commit! (DqQuery! $6 '((DqStage '((DqCnUnionAll (TDqOutput $20 '0))) (lambda '($30) (block '( + (let $31 (Uint32 '0)) + (let $32 '('"strict")) + (let $33 '"Percentile0_Percentile1_") + (let $34 '((ResourceType '"Stat.TDigestResource"))) + (let $35 (DataType 'Double)) + (let $36 '($35)) + (let $37 (OptionalType $35)) + (let $38 (DependsOn $31)) + (let $39 (lambda '($50) (block '( + (let $51 '($37)) + (let $52 (CallableType '('"2") $34 $36 $51 $51)) + (let $53 (Udf '"Stat.TDigest_Create" (Void) (VoidType) '"" $52 (VoidType) '"" '())) + (return (Just (NamedApply $53 '((Convert $50 $35)) (AsStruct) $38))) + )))) + (let $40 '($32)) + (let $41 '($21)) + (let $42 (MultiHoppingCore (FromFlow $30) (lambda '($43) $31) (lambda '($44) (FlatMap (Member (SafeCast $44 (StructType $11)) '"t") (lambda '($45) (block '( + (let $46 '($9 '"" '1)) + (let $47 (CallableType '() '((OptionalType (DataType 'Timestamp))) $46)) + (let $48 (Udf '"DateTime2.FromMilliseconds" (Void) (VoidType) '"" $47 (VoidType) '"" '('('"blocks") $32))) + (return (Apply $48 $45)) + ))))) (Interval '"5000") (Interval '"10000") (Interval '"5000000") 'true (lambda '($49) (AsStruct '($33 (FlatMap (Member $49 '"v") $39)))) (lambda '($54 $55) (block '( + (let $56 (Member $54 '"v")) + (let $57 (IfPresent (Member $55 $33) (lambda '($58) (IfPresent $56 (lambda '($59) (block '( + (let $60 (CallableType '() $34 $34 $36)) + (let $61 (Udf '"Stat.TDigest_AddValue" (Void) (VoidType) '"" $60 (VoidType) '"" $40)) + (return (Just (NamedApply $61 '($58 (Convert $59 $35)) (AsStruct) $38))) + ))) (Just $58))) (FlatMap $56 $39))) + (return (AsStruct '($33 $57))) + ))) (lambda '($62) (AsStruct '($33 (FlatMap (Member $62 $33) (lambda '($63) (block '( + (let $64 (CallableType '() $41 $34)) + (let $65 (Udf '"Stat.TDigest_Serialize" (Void) (VoidType) '"" $64 (VoidType) '"" $40)) + (return (Just (Apply $65 $63))) + ))))))) (lambda '($66) (AsStruct '($33 (FlatMap (Member $66 $33) (lambda '($67) (block '( + (let $68 (CallableType '() $34 $41)) + (let $69 (Udf '"Stat.TDigest_Deserialize" (Void) (VoidType) '"" $68 (VoidType) '"" '())) + (return (Just (Apply $69 $67))) + ))))))) (lambda '($70 $71) (block '( + (let $72 (OptionalReduce (Member $70 $33) (Member $71 $33) (lambda '($73 $74) (block '( + (let $75 (CallableType '() $34 $34 $34)) + (let $76 (Udf '"Stat.TDigest_Merge" (Void) (VoidType) '"" $75 (VoidType) '"" $40)) + (return (Apply $76 $73 $74)) + ))))) + (return (AsStruct '($33 $72))) + ))) (lambda '($77 $78 $79) (block '( + (let $80 (FlatMap (Member $78 $33) (lambda '($81) (block '( + (let $82 (CallableType '() $36 $34 $36)) + (let $83 (Udf '"Stat.TDigest_GetPercentile" (Void) (VoidType) '"" $82 (VoidType) '"" $40)) + (let $84 (Apply $83 $81 (Double '"0.75"))) + (let $85 (Apply $83 $81 (Double '"0.9"))) + (return (Just '($84 $85))) + ))))) + (return (AsStruct '('Percentile0 (Nth $80 '0)) '('Percentile1 (Nth $80 '1)) '('"group0" $79))) + ))) '1)) + (return (FlatMap (ExtractMembers $42 '('Percentile0 'Percentile1)) (lambda '($86) (block '( + (let $87 (ResourceType '"Yson2.Node")) + (let $88 '($87 '"" '1)) + (let $89 (CallableType '() '((DataType 'Yson)) $88)) + (let $90 (Udf '"Yson2.SerializeText" (Void) (VoidType) '"" $89 (VoidType) '"" $40)) + (let $91 (StructType '('"p75" $37) '('"p90" $37))) + (let $92 (TupleType (TupleType $91) (StructType) (TupleType))) + (let $93 (CallableType '() '($87) '($91))) + (let $94 (Udf '"Yson2.From" (Void) $92 '"" $93 (VoidType) '"" $40)) + (return (Just (AsStruct '('"column0" (Apply $90 (Apply $94 (AsStruct '('"p75" (Member $86 'Percentile0)) '('"p90" (Member $86 'Percentile1))))))))) + ))))) +))) '('('"_logical_id" '0)) '((DqSink '0 $22 $24))))) $22)) +) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowPercentile-default.txt_/plan.json b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowPercentile-default.txt_/plan.json new file mode 100644 index 000000000000..48462f24e7c6 --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowPercentile-default.txt_/plan.json @@ -0,0 +1,106 @@ +{ + "Detailed" : { + "Operations" : [ + { + "Id" : 5, + "Name" : "DqStage", + "Streams" : { + "Program" : [ + { + "Name" : "DqSourceWideWrap" + }, + { + "Name" : "NarrowMap" + } + ] + } + }, + { + "Id" : 3, + "Name" : "DqStage", + "Streams" : { + "Program" : [ + { + "Name" : "FromFlow" + }, + { + "Name" : "MultiHoppingCore" + }, + { + "Name" : "ExtractMembers" + }, + { + "Name" : "FlatMap" + } + ] + }, + "DependsOn" : [ + 5 + ] + }, + { + "Id" : 2, + "Name" : "DqQuery!", + "DependsOn" : [ + 3 + ] + }, + { + "Id" : 1, + "Name" : "Commit!", + "DependsOn" : [ + 2 + ] + } + ], + "OperationRoot" : 1, + "Providers" : [ ], + "OperationStats" : { + "Commit!" : 1, + "DqQuery!" : 1, + "DqStage" : 2 + } + }, + "Basic" : { + "nodes" : [ + { + "id" : 5, + "level" : 1, + "name" : "DqStage #5", + "type" : "op" + }, + { + "id" : 3, + "level" : 2, + "name" : "DqStage #3", + "type" : "op" + }, + { + "id" : 2, + "level" : 3, + "name" : "DqQuery!", + "type" : "op" + }, + { + "id" : 1, + "level" : 4, + "name" : "Commit!", + "type" : "op" + } + ], + "links" : [ + { + "source" : 5, + "target" : 3 + }, + { + "source" : 3, + "target" : 2 + }, + { + "source" : 2, + "target" : 1 + } + ] + } +} \ No newline at end of file diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowTimeExtractorUnusedColumns-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowTimeExtractorUnusedColumns-default.txt_/ast.txt new file mode 100644 index 000000000000..4d178658ba0f --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowTimeExtractorUnusedColumns-default.txt_/ast.txt @@ -0,0 +1,97 @@ +( +(let $1 (Configure! world (DataSource '"config") '"DqEngine" '"force")) +(let $2 (DataSource '"dq" '"$all")) +(let $3 (Configure! $1 $2 '"Attr" '"maxtasksperstage" '"2")) +(let $4 (Configure! $3 $2 '"Attr" '"watermarksmode" '"default")) +(let $5 (Configure! $4 $2 '"Attr" '"computeactortype" '"async")) +(let $6 (Configure! $5 (DataSource '"pq" '"$all") '"Attr" '"consumer" '"test_client")) +(let $7 (DataSource '"pq" '"pq")) +(let $8 '('('"PartitionsCount" '1))) +(let $9 (DataType 'Uint64)) +(let $10 (OptionalType $9)) +(let $11 '('"t" $10)) +(let $12 (StructType $11 '('"v" $10))) +(let $13 (PqTopic '"pq" '"local" '"test_topic_input" $8 '() $12)) +(let $14 '('"Endpoint" '"")) +(let $15 '('"SharedReading" '1)) +(let $16 '('"UseSsl" '1)) +(let $17 '('('"Consumer" '"test_client") $14 $15 '('"ReconnectPeriod" '"") '('"Format" '"json_each_row") '('"ReadGroup" '"fqrun") $16 '('"WatermarksEnable" '1) '('"WatermarksGranularityUs" '"1000000") '('"WatermarksLateArrivalDelayUs" '"5000000"))) +(let $18 (SecureParam '"cluster:default_pq")) +(let $19 (DqPqTopicSource $6 $13 '('"t" '"v") $17 $18 '"" $12 '"")) +(let $20 (DqStage '((DqSource $7 $19)) (lambda '($25) (block '( + (let $26 '('('"format" '"json_each_row") '('"formatSettings" '('('"data.datetime.formatname" '"POSIX") '('"data.timestamp.formatname" '"POSIX"))) '('"settings" '($15)))) + (let $27 (DqSourceWideWrap $25 $7 $12 $26)) + (return (NarrowMap $27 (lambda '($28 $29) (AsStruct '('"t" $28) '('"v" $29))))) +))) '('('"_logical_id" '0)))) +(let $21 (DataType 'String)) +(let $22 (DataSink '"pq" '"pq")) +(let $23 (PqTopic '"pq" '"local" '"test_topic_output" $8 '() (StructType '('"Data" $21)))) +(let $24 (DqPqTopicSink $23 '($14 $16) $18)) +(return (Commit! (DqQuery! $6 '((DqStage '((DqCnUnionAll (TDqOutput $20 '0))) (lambda '($30) (block '( + (let $31 (Uint32 '0)) + (let $32 '('"strict")) + (let $33 '"Percentile0_Percentile1_") + (let $34 '((ResourceType '"Stat.TDigestResource"))) + (let $35 (DataType 'Double)) + (let $36 '($35)) + (let $37 (OptionalType $35)) + (let $38 (DependsOn $31)) + (let $39 (lambda '($50) (block '( + (let $51 '($37)) + (let $52 (CallableType '('"2") $34 $36 $51 $51)) + (let $53 (Udf '"Stat.TDigest_Create" (Void) (VoidType) '"" $52 (VoidType) '"" '())) + (return (Just (NamedApply $53 '((Convert $50 $35)) (AsStruct) $38))) + )))) + (let $40 '($32)) + (let $41 '($21)) + (let $42 (MultiHoppingCore (FromFlow $30) (lambda '($43) $31) (lambda '($44) (FlatMap (Member (SafeCast $44 (StructType $11)) '"t") (lambda '($45) (block '( + (let $46 '($9 '"" '1)) + (let $47 (CallableType '() '((OptionalType (DataType 'Timestamp))) $46)) + (let $48 (Udf '"DateTime2.FromMilliseconds" (Void) (VoidType) '"" $47 (VoidType) '"" '('('"blocks") $32))) + (return (Apply $48 $45)) + ))))) (Interval '"5000") (Interval '"10000") (Interval '"5000000") 'true (lambda '($49) (AsStruct '($33 (FlatMap (Member $49 '"v") $39)))) (lambda '($54 $55) (block '( + (let $56 (Member $54 '"v")) + (let $57 (IfPresent (Member $55 $33) (lambda '($58) (IfPresent $56 (lambda '($59) (block '( + (let $60 (CallableType '() $34 $34 $36)) + (let $61 (Udf '"Stat.TDigest_AddValue" (Void) (VoidType) '"" $60 (VoidType) '"" $40)) + (return (Just (NamedApply $61 '($58 (Convert $59 $35)) (AsStruct) $38))) + ))) (Just $58))) (FlatMap $56 $39))) + (return (AsStruct '($33 $57))) + ))) (lambda '($62) (AsStruct '($33 (FlatMap (Member $62 $33) (lambda '($63) (block '( + (let $64 (CallableType '() $41 $34)) + (let $65 (Udf '"Stat.TDigest_Serialize" (Void) (VoidType) '"" $64 (VoidType) '"" $40)) + (return (Just (Apply $65 $63))) + ))))))) (lambda '($66) (AsStruct '($33 (FlatMap (Member $66 $33) (lambda '($67) (block '( + (let $68 (CallableType '() $34 $41)) + (let $69 (Udf '"Stat.TDigest_Deserialize" (Void) (VoidType) '"" $68 (VoidType) '"" '())) + (return (Just (Apply $69 $67))) + ))))))) (lambda '($70 $71) (block '( + (let $72 (OptionalReduce (Member $70 $33) (Member $71 $33) (lambda '($73 $74) (block '( + (let $75 (CallableType '() $34 $34 $34)) + (let $76 (Udf '"Stat.TDigest_Merge" (Void) (VoidType) '"" $75 (VoidType) '"" $40)) + (return (Apply $76 $73 $74)) + ))))) + (return (AsStruct '($33 $72))) + ))) (lambda '($77 $78 $79) (block '( + (let $80 (FlatMap (Member $78 $33) (lambda '($81) (block '( + (let $82 (CallableType '() $36 $34 $36)) + (let $83 (Udf '"Stat.TDigest_GetPercentile" (Void) (VoidType) '"" $82 (VoidType) '"" $40)) + (let $84 (Apply $83 $81 (Double '"0.75"))) + (let $85 (Apply $83 $81 (Double '"0.9"))) + (return (Just '($84 $85))) + ))))) + (return (AsStruct '('Percentile0 (Nth $80 '0)) '('Percentile1 (Nth $80 '1)) '('"group0" $79))) + ))) '1)) + (return (FlatMap (ExtractMembers $42 '('Percentile0 'Percentile1)) (lambda '($86) (block '( + (let $87 (ResourceType '"Yson2.Node")) + (let $88 '($87 '"" '1)) + (let $89 (CallableType '() '((DataType 'Yson)) $88)) + (let $90 (Udf '"Yson2.SerializeText" (Void) (VoidType) '"" $89 (VoidType) '"" $40)) + (let $91 (StructType '('"p75" $37) '('"p90" $37))) + (let $92 (TupleType (TupleType $91) (StructType) (TupleType))) + (let $93 (CallableType '() '($87) '($91))) + (let $94 (Udf '"Yson2.From" (Void) $92 '"" $93 (VoidType) '"" $40)) + (return (Just (AsStruct '('"column0" (Apply $90 (Apply $94 (AsStruct '('"p75" (Member $86 'Percentile0)) '('"p90" (Member $86 'Percentile1))))))))) + ))))) +))) '('('"_logical_id" '0)) '((DqSink '0 $22 $24))))) $22)) +) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowTimeExtractorUnusedColumns-default.txt_/plan.json b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowTimeExtractorUnusedColumns-default.txt_/plan.json new file mode 100644 index 000000000000..48462f24e7c6 --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowTimeExtractorUnusedColumns-default.txt_/plan.json @@ -0,0 +1,106 @@ +{ + "Detailed" : { + "Operations" : [ + { + "Id" : 5, + "Name" : "DqStage", + "Streams" : { + "Program" : [ + { + "Name" : "DqSourceWideWrap" + }, + { + "Name" : "NarrowMap" + } + ] + } + }, + { + "Id" : 3, + "Name" : "DqStage", + "Streams" : { + "Program" : [ + { + "Name" : "FromFlow" + }, + { + "Name" : "MultiHoppingCore" + }, + { + "Name" : "ExtractMembers" + }, + { + "Name" : "FlatMap" + } + ] + }, + "DependsOn" : [ + 5 + ] + }, + { + "Id" : 2, + "Name" : "DqQuery!", + "DependsOn" : [ + 3 + ] + }, + { + "Id" : 1, + "Name" : "Commit!", + "DependsOn" : [ + 2 + ] + } + ], + "OperationRoot" : 1, + "Providers" : [ ], + "OperationStats" : { + "Commit!" : 1, + "DqQuery!" : 1, + "DqStage" : 2 + } + }, + "Basic" : { + "nodes" : [ + { + "id" : 5, + "level" : 1, + "name" : "DqStage #5", + "type" : "op" + }, + { + "id" : 3, + "level" : 2, + "name" : "DqStage #3", + "type" : "op" + }, + { + "id" : 2, + "level" : 3, + "name" : "DqQuery!", + "type" : "op" + }, + { + "id" : 1, + "level" : 4, + "name" : "Commit!", + "type" : "op" + } + ], + "links" : [ + { + "source" : 5, + "target" : 3 + }, + { + "source" : 3, + "target" : 2 + }, + { + "source" : 2, + "target" : 1 + } + ] + } +} \ No newline at end of file diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopic-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadTopic-default.txt_/ast.txt similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopic-default.txt_/ast.txt rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadTopic-default.txt_/ast.txt diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopic-default.txt_/plan.json b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadTopic-default.txt_/plan.json similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopic-default.txt_/plan.json rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadTopic-default.txt_/plan.json diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadata-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadTopicWithMetadata-default.txt_/ast.txt similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadata-default.txt_/ast.txt rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadTopicWithMetadata-default.txt_/ast.txt diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadata-default.txt_/plan.json b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadTopicWithMetadata-default.txt_/plan.json similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadata-default.txt_/plan.json rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadTopicWithMetadata-default.txt_/plan.json diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadataInsideFilter-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadTopicWithMetadataInsideFilter-default.txt_/ast.txt similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadataInsideFilter-default.txt_/ast.txt rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadTopicWithMetadataInsideFilter-default.txt_/ast.txt diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadataInsideFilter-default.txt_/plan.json b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadTopicWithMetadataInsideFilter-default.txt_/plan.json similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadataInsideFilter-default.txt_/plan.json rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadTopicWithMetadataInsideFilter-default.txt_/plan.json diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadataNestedDeep-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadTopicWithMetadataNestedDeep-default.txt_/ast.txt similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadataNestedDeep-default.txt_/ast.txt rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadTopicWithMetadataNestedDeep-default.txt_/ast.txt diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadataNestedDeep-default.txt_/plan.json b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadTopicWithMetadataNestedDeep-default.txt_/plan.json similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadataNestedDeep-default.txt_/plan.json rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadTopicWithMetadataNestedDeep-default.txt_/plan.json diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadataWithFilter-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadTopicWithMetadataWithFilter-default.txt_/ast.txt similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadataWithFilter-default.txt_/ast.txt rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadTopicWithMetadataWithFilter-default.txt_/ast.txt diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadataWithFilter-default.txt_/plan.json b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadTopicWithMetadataWithFilter-default.txt_/plan.json similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadataWithFilter-default.txt_/plan.json rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadTopicWithMetadataWithFilter-default.txt_/plan.json diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithSchema-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadTopicWithSchema-default.txt_/ast.txt similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithSchema-default.txt_/ast.txt rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadTopicWithSchema-default.txt_/ast.txt diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithSchema-default.txt_/plan.json b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadTopicWithSchema-default.txt_/plan.json similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithSchema-default.txt_/plan.json rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadTopicWithSchema-default.txt_/plan.json diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTwoTopics-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadTwoTopics-default.txt_/ast.txt similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTwoTopics-default.txt_/ast.txt rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadTwoTopics-default.txt_/ast.txt diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTwoTopics-default.txt_/plan.json b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadTwoTopics-default.txt_/plan.json similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTwoTopics-default.txt_/plan.json rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadTwoTopics-default.txt_/plan.json diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadWriteSameTopic-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadWriteSameTopic-default.txt_/ast.txt similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadWriteSameTopic-default.txt_/ast.txt rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadWriteSameTopic-default.txt_/ast.txt diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadWriteSameTopic-default.txt_/plan.json b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadWriteSameTopic-default.txt_/plan.json similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadWriteSameTopic-default.txt_/plan.json rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadWriteSameTopic-default.txt_/plan.json diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadWriteTopic-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadWriteTopic-default.txt_/ast.txt similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadWriteTopic-default.txt_/ast.txt rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadWriteTopic-default.txt_/ast.txt diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadWriteTopic-default.txt_/plan.json b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadWriteTopic-default.txt_/plan.json similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadWriteTopic-default.txt_/plan.json rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadWriteTopic-default.txt_/plan.json diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadWriteTopicWithSchema-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadWriteTopicWithSchema-default.txt_/ast.txt similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadWriteTopicWithSchema-default.txt_/ast.txt rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadWriteTopicWithSchema-default.txt_/ast.txt diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadWriteTopicWithSchema-default.txt_/plan.json b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadWriteTopicWithSchema-default.txt_/plan.json similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadWriteTopicWithSchema-default.txt_/plan.json rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-ReadWriteTopicWithSchema-default.txt_/plan.json diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-WriteTwoTopics-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-WriteTwoTopics-default.txt_/ast.txt similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-WriteTwoTopics-default.txt_/ast.txt rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-WriteTwoTopics-default.txt_/ast.txt diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-WriteTwoTopics-default.txt_/plan.json b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-WriteTwoTopics-default.txt_/plan.json similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-WriteTwoTopics-default.txt_/plan.json rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_pq-WriteTwoTopics-default.txt_/plan.json diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicGroupWriteToSolomon-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_solomon-ReadTopicGroupWriteToSolomon-default.txt_/ast.txt similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicGroupWriteToSolomon-default.txt_/ast.txt rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_solomon-ReadTopicGroupWriteToSolomon-default.txt_/ast.txt diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicGroupWriteToSolomon-default.txt_/plan.json b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_solomon-ReadTopicGroupWriteToSolomon-default.txt_/plan.json similarity index 100% rename from ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicGroupWriteToSolomon-default.txt_/plan.json rename to ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_solomon-ReadTopicGroupWriteToSolomon-default.txt_/plan.json diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHoppingWithDataWatermarks-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHoppingWithDataWatermarks-default.txt_/ast.txt deleted file mode 100644 index 0f7c993101a4..000000000000 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHoppingWithDataWatermarks-default.txt_/ast.txt +++ /dev/null @@ -1,81 +0,0 @@ -( -(let $1 (Configure! world (DataSource '"config") '"DqEngine" '"force")) -(let $2 (Configure! $1 (DataSource '"dq" '"$all") '"Attr" '"maxtasksperstage" '"2")) -(let $3 (Configure! $2 (DataSource '"pq" '"$all") '"Attr" '"consumer" '"test_client")) -(let $4 (DataSource '"pq" '"pq")) -(let $5 '('('"PartitionsCount" '"1"))) -(let $6 (DataType 'String)) -(let $7 (StructType '('"Data" $6))) -(let $8 (PqTopic '"pq" '"local" '"test_topic_input" $5 '() $7)) -(let $9 '('"Endpoint" '"")) -(let $10 '('"SharedReading" '"1")) -(let $11 '('"UseSsl" '"1")) -(let $12 '('('"Consumer" '"test_client") $9 $10 '('"ReconnectPeriod" '"") '('"Format" '"raw") '('"ReadGroup" '"fqrun") $11)) -(let $13 (SecureParam '"cluster:default_pq")) -(let $14 (DqPqTopicSource $3 $8 '('"Data") $12 $13 '"" $7 '"")) -(let $15 (ResourceType '"Yson2.Node")) -(let $16 '('"strict")) -(let $17 '($16)) -(let $18 (DataType 'Yson)) -(let $19 '($15 '"" '"1")) -(let $20 (CallableType '() '($18) $19)) -(let $21 (OptionalType (DataType 'Int64))) -(let $22 '($15)) -(let $23 (DataType 'Uint64)) -(let $24 (OptionalType $23)) -(let $25 (DqStage '((DqSource $4 $14)) (lambda '($29) (block '( - (let $30 '('('"format" '"raw") '('"formatSettings" '('('"data.datetime.formatname" '"POSIX") '('"data.timestamp.formatname" '"POSIX"))) '('"settings" '($10)))) - (let $31 (DqSourceWideWrap $29 $4 $7 $30)) - (return (FlatMap (NarrowMap $31 (lambda '($32) (AsStruct '('"Data" $32)))) (lambda '($33) (block '( - (let $34 (ResourceType '"Yson2.Options")) - (let $35 (TupleType (TupleType $6 $34) (StructType) (TupleType))) - (let $36 '('"1")) - (let $37 '($6 '"" '"1")) - (let $38 '((OptionalType $34))) - (let $39 (CallableType $36 '((OptionalType $15)) $37 $38)) - (let $40 (Udf '"Yson2.Parse" (Void) $35 '"" $39 (VoidType) '"" '())) - (let $41 (OptionalType (DataType 'Bool))) - (let $42 (CallableType '('"2") '($34) '($41 '"AutoConvert") '($41 '"Strict"))) - (let $43 (Udf '"Yson2.Options" (Void) (VoidType) '"" $42 (VoidType) '"" $17)) - (let $44 (Just (NamedApply $43 '() (AsStruct '('"AutoConvert" (Just (Bool '"false"))) '('"Strict" (Just (Bool 'true))))))) - (let $45 (Apply $40 (Member $33 '"Data") $44)) - (let $46 (FlatMap $45 (lambda '($54) (block '( - (let $55 (Udf '"Yson2.Serialize" (Void) (VoidType) '"" $20 (VoidType) '"" $17)) - (return (Just (Apply $55 $54))) - ))))) - (let $47 '($6)) - (let $48 (CallableType $36 '($21) $19 $47 $38)) - (let $49 (Udf '"Yson2.LookupInt64" (Void) (VoidType) '"" $48 (VoidType) '"" '())) - (let $50 (TupleType (TupleType $18) (StructType) (TupleType))) - (let $51 '($18 '"" '"1")) - (let $52 (CallableType $36 $22 $51 $38)) - (let $53 (Udf '"Yson2.Parse" (Void) $50 '"" $52 (VoidType) '"" $17)) - (return (Just (AsStruct '('"k" (FlatMap $46 (lambda '($56) (Apply $49 (Apply $53 $56) (String '"key") $44)))) '('"t" (FlatMap $46 (lambda '($57) (block '( - (let $58 (CallableType $36 '($24) $19 $47 $38)) - (let $59 (Udf '"Yson2.LookupUint64" (Void) (VoidType) '"" $58 (VoidType) '"" '())) - (return (Apply $59 (Apply $53 $57) (String '"time") $44)) - ))))) '('"v" (FlatMap $46 (lambda '($60) (Apply $49 (Apply $53 $60) (String '"val") $44))))))) - ))))) -))) '('('"_logical_id" '0)))) -(let $26 (DataSink '"pq" '"pq")) -(let $27 (PqTopic '"pq" '"local" '"test_topic_output" $5 '() $7)) -(let $28 (DqPqTopicSink $27 '($9 $11) $13)) -(return (Commit! (DqQuery! $3 '((DqStage '((DqCnHashShuffle (TDqOutput $25 '0) '('"k"))) (lambda '($61) (block '( - (let $62 (Interval '10000)) - (let $63 (lambda '($74) $74)) - (let $64 (MultiHoppingCore (FromFlow $61) (lambda '($65) (Member $65 '"k")) (lambda '($66) (FlatMap (SafeCast (Unwrap (Member (SafeCast $66 (StructType '('"t" $24))) '"t")) (OptionalType (DataType 'Uint32))) (lambda '($67) (block '( - (let $68 '($23 '"" '"1")) - (let $69 (CallableType '() '((OptionalType (DataType 'Timestamp))) $68)) - (let $70 (Udf '"DateTime2.FromMilliseconds" (Void) (VoidType) '"" $69 (VoidType) '"" '('('"blocks") $16))) - (return (Apply $70 (Convert $67 $23))) - ))))) (Interval '5000) $62 $62 'true (lambda '($71) (AsStruct '('Sum0 (Member $71 '"v")))) (lambda '($72 $73) (AsStruct '('Sum0 (AggrAdd (Member $72 '"v") (Member $73 'Sum0))))) $63 $63 (lambda '($75 $76) (AsStruct '('Sum0 (AggrAdd (Member $75 'Sum0) (Member $76 'Sum0))))) (lambda '($77 $78 $79) (AsStruct '('Sum0 (Member $78 'Sum0)) '('"_yql_time" $79) '('"k" $77))) '0)) - (return (FlatMap (ExtractMembers $64 '('Sum0 '"k")) (lambda '($80) (block '( - (let $81 (Udf '"Yson2.SerializeText" (Void) (VoidType) '"" $20 (VoidType) '"" $17)) - (let $82 (StructType '('"k" $21) '('"sum" $21))) - (let $83 (TupleType (TupleType $82) (StructType) (TupleType))) - (let $84 (CallableType '() $22 '($82))) - (let $85 (Udf '"Yson2.From" (Void) $83 '"" $84 (VoidType) '"" $17)) - (return (Just (AsStruct '('"column0" (Apply $81 (Apply $85 (AsStruct '('"k" (Member $80 '"k")) '('"sum" (Member $80 'Sum0))))))))) - ))))) -))) '('('"_logical_id" '0)) '((DqSink '0 $26 $28))))) $26)) -) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_watermarks-watermarks-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_watermarks-watermarks-default.txt_/ast.txt new file mode 100644 index 000000000000..2ea4364f2ed4 --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_watermarks-watermarks-default.txt_/ast.txt @@ -0,0 +1,20 @@ +( +(let $1 (Configure! world (DataSource '"config") '"DqEngine" '"force")) +(let $2 (Configure! $1 (DataSource '"dq" '"$all") '"Attr" '"watermarksmode" '"default")) +(let $3 (Configure! $2 (DataSource '"pq" '"$all") '"Attr" '"consumer" '"test_client")) +(let $4 (DataSink 'result)) +(let $5 (DataSource '"pq" '"pq")) +(let $6 (StructType '('"ts" (DataType 'Timestamp)))) +(let $7 (PqTopic '"pq" '"local" '"test_topic_input" '('('"PartitionsCount" '"1")) '() $6)) +(let $8 '('"SharedReading" '"1")) +(let $9 '('('"Consumer" '"test_client") '('"Endpoint" '"") $8 '('"ReconnectPeriod" '"") '('"Format" '"json_each_row") '('"ReadGroup" '"fqrun") '('"UseSsl" '"1") '('"WatermarksEnable" '"1") '('"WatermarksGranularityUs" '"1000000") '('"WatermarksLateArrivalDelayUs" '"5000000"))) +(let $10 (DqPqTopicSource world $7 '('"ts") $9 (SecureParam '"cluster:default_pq") '"" $6 '"B\x1F\n\x1D\x1A\x1B\b\x03\x12\x04\x12\x02ts\x1A\x11\n\x0F\n\x02\b3\x12\t!@KL\x00\x00\x00\x00\x00")) +(let $11 (DqStage '((DqSource $5 $10)) (lambda '($14) (block '( + (let $15 '('('"format" '"json_each_row") '('"formatSettings" '('('"data.datetime.formatname" '"POSIX") '('"data.timestamp.formatname" '"POSIX"))) '('"settings" '($8)))) + (let $16 (DqSourceWideWrap $14 $5 $6 $15)) + (return (NarrowMap $16 (lambda '($17) (AsStruct '('"ts" $17))))) +))) '('('"_logical_id" '0)))) +(let $12 (DqStage '((DqCnUnionAll (TDqOutput $11 '"0"))) (lambda '($18) $18) '('('"_logical_id" '0)))) +(let $13 (ResPull! $3 $4 (Key) (DqCnResult (TDqOutput $12 '"0") '()) '('('type) '('autoref)) '"dq")) +(return (Commit! (Commit! $13 $4) (DataSink '"pq" '"pq"))) +) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_watermarks-watermarks-default.txt_/plan.json b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_watermarks-watermarks-default.txt_/plan.json new file mode 100644 index 000000000000..a0919980b586 --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_watermarks-watermarks-default.txt_/plan.json @@ -0,0 +1,93 @@ +{ + "Detailed" : { + "Operations" : [ + { + "Id" : 10, + "Name" : "DqStage", + "Streams" : { + "Program" : [ + { + "Name" : "DqSourceWideWrap" + }, + { + "Name" : "NarrowMap" + } + ] + } + }, + { + "Id" : 8, + "Name" : "DqStage", + "Streams" : { + "Program" : [ ] + }, + "DependsOn" : [ + 10 + ] + }, + { + "Id" : 3, + "Name" : "ResPull!", + "DependsOn" : [ + 8 + ] + }, + { + "Id" : 1, + "Name" : "Commit!", + "DependsOn" : [ + 3 + ] + } + ], + "OperationRoot" : 1, + "Providers" : [ ], + "OperationStats" : { + "Commit!" : 1, + "DqStage" : 2, + "ResPull!" : 1 + } + }, + "Basic" : { + "nodes" : [ + { + "id" : 10, + "level" : 1, + "name" : "DqStage #10", + "type" : "op" + }, + { + "id" : 8, + "level" : 2, + "name" : "DqStage #8", + "type" : "op" + }, + { + "id" : 3, + "level" : 3, + "name" : "ResPull!", + "type" : "op" + }, + { + "id" : 1, + "level" : 4, + "name" : "Commit!", + "type" : "op" + } + ], + "links" : [ + { + "source" : 10, + "target" : 8 + }, + { + "source" : 8, + "target" : 3 + }, + { + "source" : 3, + "target" : 1 + } + ] + } +} \ No newline at end of file diff --git a/ydb/tests/fq/streaming_optimize/suites/GroupByHoppingWithDataWatermarks.sql b/ydb/tests/fq/streaming_optimize/suites/GroupByHoppingWithDataWatermarks.sql deleted file mode 100644 index fecbee9e760e..000000000000 --- a/ydb/tests/fq/streaming_optimize/suites/GroupByHoppingWithDataWatermarks.sql +++ /dev/null @@ -1,27 +0,0 @@ -/* syntax version 1 */ -/* dqfile can not */ - -PRAGMA dq.MaxTasksPerStage="2"; -PRAGMA DataWatermarks="enable"; - -PRAGMA pq.Consumer="test_client"; - -INSERT INTO pq.test_topic_output -SELECT STREAM - Yson::SerializeText(Yson::From(TableRow())) -FROM ( - SELECT STREAM - k, - Sum(v) as sum - FROM ( - SELECT STREAM - Yson::LookupUint64(ys, "time") as t, - Yson::LookupInt64(ys, "key") as k, - Yson::LookupInt64(ys, "val") as v - FROM ( - SELECT STREAM - Yson::Parse(Data) AS ys - FROM pq.test_topic_input)) - GROUP BY - k, - HOP(DateTime::FromMilliseconds(CAST(Unwrap(t) as Uint32)), "PT0.005S", "PT0.01S", "PT0.01S")); diff --git a/ydb/tests/fq/streaming_optimize/suites/GroupByHop.sql b/ydb/tests/fq/streaming_optimize/suites/hop/GroupByHop.sql similarity index 100% rename from ydb/tests/fq/streaming_optimize/suites/GroupByHop.sql rename to ydb/tests/fq/streaming_optimize/suites/hop/GroupByHop.sql diff --git a/ydb/tests/fq/streaming_optimize/suites/GroupByHopByStringKey.sql b/ydb/tests/fq/streaming_optimize/suites/hop/GroupByHopByStringKey.sql similarity index 100% rename from ydb/tests/fq/streaming_optimize/suites/GroupByHopByStringKey.sql rename to ydb/tests/fq/streaming_optimize/suites/hop/GroupByHopByStringKey.sql diff --git a/ydb/tests/fq/streaming_optimize/suites/GroupByHopExprKey.sql b/ydb/tests/fq/streaming_optimize/suites/hop/GroupByHopExprKey.sql similarity index 100% rename from ydb/tests/fq/streaming_optimize/suites/GroupByHopExprKey.sql rename to ydb/tests/fq/streaming_optimize/suites/hop/GroupByHopExprKey.sql diff --git a/ydb/tests/fq/streaming_optimize/suites/GroupByHopListKey.sql b/ydb/tests/fq/streaming_optimize/suites/hop/GroupByHopListKey.sql similarity index 100% rename from ydb/tests/fq/streaming_optimize/suites/GroupByHopListKey.sql rename to ydb/tests/fq/streaming_optimize/suites/hop/GroupByHopListKey.sql diff --git a/ydb/tests/fq/streaming_optimize/suites/GroupByHopNoKey.sql b/ydb/tests/fq/streaming_optimize/suites/hop/GroupByHopNoKey.sql similarity index 100% rename from ydb/tests/fq/streaming_optimize/suites/GroupByHopNoKey.sql rename to ydb/tests/fq/streaming_optimize/suites/hop/GroupByHopNoKey.sql diff --git a/ydb/tests/fq/streaming_optimize/suites/GroupByHopPercentile.sql b/ydb/tests/fq/streaming_optimize/suites/hop/GroupByHopPercentile.sql similarity index 100% rename from ydb/tests/fq/streaming_optimize/suites/GroupByHopPercentile.sql rename to ydb/tests/fq/streaming_optimize/suites/hop/GroupByHopPercentile.sql diff --git a/ydb/tests/fq/streaming_optimize/suites/GroupByHopTimeExtractorUnusedColumns.sql b/ydb/tests/fq/streaming_optimize/suites/hop/GroupByHopTimeExtractorUnusedColumns.sql similarity index 100% rename from ydb/tests/fq/streaming_optimize/suites/GroupByHopTimeExtractorUnusedColumns.sql rename to ydb/tests/fq/streaming_optimize/suites/hop/GroupByHopTimeExtractorUnusedColumns.sql diff --git a/ydb/tests/fq/streaming_optimize/suites/GroupByHopWithDataWatermarks.sql b/ydb/tests/fq/streaming_optimize/suites/hop/GroupByHopWithDataWatermarks.sql similarity index 100% rename from ydb/tests/fq/streaming_optimize/suites/GroupByHopWithDataWatermarks.sql rename to ydb/tests/fq/streaming_optimize/suites/hop/GroupByHopWithDataWatermarks.sql diff --git a/ydb/tests/fq/streaming_optimize/suites/hop/default.cfg b/ydb/tests/fq/streaming_optimize/suites/hop/default.cfg new file mode 100644 index 000000000000..4c25fdbd7ece --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/suites/hop/default.cfg @@ -0,0 +1 @@ +providers pq diff --git a/ydb/tests/fq/streaming_optimize/suites/hopping_window/GroupByHoppingWindow.sql b/ydb/tests/fq/streaming_optimize/suites/hopping_window/GroupByHoppingWindow.sql new file mode 100644 index 000000000000..44f9c05fb880 --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/suites/hopping_window/GroupByHoppingWindow.sql @@ -0,0 +1,34 @@ +/* syntax version 1 */ +/* dq can not */ + +PRAGMA dq.MaxTasksPerStage="2"; +PRAGMA dq.WatermarksMode="default"; +PRAGMA dq.ComputeActorType="async"; + +PRAGMA pq.Consumer="test_client"; + +$input = + SELECT + * + FROM pq.test_topic_input + WITH ( + FORMAT=json_each_row, + SCHEMA( + t Uint64, + k String, + v Uint64 + ) + ); + +$output = + SELECT + k, + Sum(v) AS sum + FROM $input + GROUP BY + k, + HoppingWindow(DateTime::FromMilliseconds(t), "PT0.005S", "PT0.01S"); + +INSERT INTO pq.test_topic_output +SELECT Yson::SerializeText(Yson::From(TableRow())) +FROM $output; diff --git a/ydb/tests/fq/streaming_optimize/suites/hopping_window/GroupByHoppingWindowByStringKey.sql b/ydb/tests/fq/streaming_optimize/suites/hopping_window/GroupByHoppingWindowByStringKey.sql new file mode 100644 index 000000000000..21dcd6be80b1 --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/suites/hopping_window/GroupByHoppingWindowByStringKey.sql @@ -0,0 +1,32 @@ +/* syntax version 1 */ +/* dq can not */ + +PRAGMA dq.MaxTasksPerStage="2"; +PRAGMA dq.WatermarksMode="default"; +PRAGMA dq.ComputeActorType="async"; + +PRAGMA pq.Consumer="test_client"; + +$input = + SELECT + * + FROM pq.test_topic_input + WITH ( + FORMAT=json_each_row, + SCHEMA( + t Uint64, + k String, + v Uint64 + ) + ); + +$output = + SELECT + k, + Sum(v) AS sum + FROM $input + GROUP BY k, HoppingWindow(DateTime::FromMilliseconds(t), "PT0.005S", "PT0.01S"); + +INSERT INTO pq.test_topic_output +SELECT Yson::SerializeText(Yson::From(TableRow())) +FROM $output; diff --git a/ydb/tests/fq/streaming_optimize/suites/hopping_window/GroupByHoppingWindowExprKey.sql b/ydb/tests/fq/streaming_optimize/suites/hopping_window/GroupByHoppingWindowExprKey.sql new file mode 100644 index 000000000000..4dbf41309009 --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/suites/hopping_window/GroupByHoppingWindowExprKey.sql @@ -0,0 +1,33 @@ +/* syntax version 1 */ +/* dq can not */ + +PRAGMA dq.MaxTasksPerStage="2"; +PRAGMA dq.WatermarksMode="default"; +PRAGMA dq.ComputeActorType="async"; + +PRAGMA pq.Consumer="test_client"; + +$input = + SELECT + * + FROM pq.test_topic_input + WITH ( + FORMAT=json_each_row, + SCHEMA( + t Uint64, + k Uint64, + v Uint64 + ) + ); + +$output = + SELECT + Sum(v) AS sum + FROM $input + GROUP BY + MIN_OF(k, 2), + HoppingWindow(DateTime::FromMilliseconds(t), "PT0.005S", "PT0.01S"); + +INSERT INTO pq.test_topic_output +SELECT Yson::SerializeText(Yson::From(TableRow())) +FROM $output; diff --git a/ydb/tests/fq/streaming_optimize/suites/hopping_window/GroupByHoppingWindowListKey.sql b/ydb/tests/fq/streaming_optimize/suites/hopping_window/GroupByHoppingWindowListKey.sql new file mode 100644 index 000000000000..f27c6ff36fd1 --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/suites/hopping_window/GroupByHoppingWindowListKey.sql @@ -0,0 +1,34 @@ +/* syntax version 1 */ +/* dq can not */ + +PRAGMA dq.MaxTasksPerStage="2"; +PRAGMA dq.WatermarksMode="default"; +PRAGMA dq.ComputeActorType="async"; + +PRAGMA pq.Consumer="test_client"; + +$input = + SELECT + * + FROM pq.test_topic_input + WITH ( + FORMAT=json_each_row, + SCHEMA( + t Uint64, + k Uint64, + v Uint64 + ) + ); + +$output = + SELECT + Sum(v) AS sum + FROM $input + GROUP BY + k, + AsList(1, k), + HoppingWindow(DateTime::FromMilliseconds(t), "PT0.005S", "PT0.01S"); + +INSERT INTO pq.test_topic_output +SELECT Yson::SerializeText(Yson::From(TableRow())) +FROM $output; diff --git a/ydb/tests/fq/streaming_optimize/suites/hopping_window/GroupByHoppingWindowNoKey.sql b/ydb/tests/fq/streaming_optimize/suites/hopping_window/GroupByHoppingWindowNoKey.sql new file mode 100644 index 000000000000..b630a122aa6c --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/suites/hopping_window/GroupByHoppingWindowNoKey.sql @@ -0,0 +1,32 @@ +/* syntax version 1 */ +/* dq can not */ + +PRAGMA dq.MaxTasksPerStage="2"; +PRAGMA dq.WatermarksMode="default"; +PRAGMA dq.ComputeActorType="async"; + +PRAGMA pq.Consumer="test_client"; + +$input = + SELECT + * + FROM pq.test_topic_input + WITH ( + FORMAT=json_each_row, + SCHEMA( + t Uint64, + k String, + v Uint64 + ) + ); + +$output = + SELECT + Sum(v) AS sum + FROM $input + GROUP BY + HoppingWindow(DateTime::FromMilliseconds(t), "PT0.005S", "PT0.01S"); + +INSERT INTO pq.test_topic_output +SELECT Yson::SerializeText(Yson::From(TableRow())) +FROM $output; diff --git a/ydb/tests/fq/streaming_optimize/suites/hopping_window/GroupByHoppingWindowPercentile.sql b/ydb/tests/fq/streaming_optimize/suites/hopping_window/GroupByHoppingWindowPercentile.sql new file mode 100644 index 000000000000..471b401fa1ce --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/suites/hopping_window/GroupByHoppingWindowPercentile.sql @@ -0,0 +1,33 @@ +/* syntax version 1 */ +/* dq can not */ + +PRAGMA dq.MaxTasksPerStage="2"; +PRAGMA dq.WatermarksMode="default"; +PRAGMA dq.ComputeActorType="async"; + +PRAGMA pq.Consumer="test_client"; + +$input = + SELECT + * + FROM pq.test_topic_input + WITH ( + FORMAT=json_each_row, + SCHEMA( + t Uint64, + k String, + v Uint64 + ) + ); + +$output = + SELECT + percentile(v, 0.75) AS p75, + percentile(v, 0.9) AS p90 + FROM $input + GROUP BY + HoppingWindow(DateTime::FromMilliseconds(t), "PT0.005S", "PT0.01S"); + +INSERT INTO pq.test_topic_output +SELECT Yson::SerializeText(Yson::From(TableRow())) +FROM $output; diff --git a/ydb/tests/fq/streaming_optimize/suites/hopping_window/GroupByHoppingWindowTimeExtractorUnusedColumns.sql b/ydb/tests/fq/streaming_optimize/suites/hopping_window/GroupByHoppingWindowTimeExtractorUnusedColumns.sql new file mode 100644 index 000000000000..471b401fa1ce --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/suites/hopping_window/GroupByHoppingWindowTimeExtractorUnusedColumns.sql @@ -0,0 +1,33 @@ +/* syntax version 1 */ +/* dq can not */ + +PRAGMA dq.MaxTasksPerStage="2"; +PRAGMA dq.WatermarksMode="default"; +PRAGMA dq.ComputeActorType="async"; + +PRAGMA pq.Consumer="test_client"; + +$input = + SELECT + * + FROM pq.test_topic_input + WITH ( + FORMAT=json_each_row, + SCHEMA( + t Uint64, + k String, + v Uint64 + ) + ); + +$output = + SELECT + percentile(v, 0.75) AS p75, + percentile(v, 0.9) AS p90 + FROM $input + GROUP BY + HoppingWindow(DateTime::FromMilliseconds(t), "PT0.005S", "PT0.01S"); + +INSERT INTO pq.test_topic_output +SELECT Yson::SerializeText(Yson::From(TableRow())) +FROM $output; diff --git a/ydb/tests/fq/streaming_optimize/suites/hopping_window/default.cfg b/ydb/tests/fq/streaming_optimize/suites/hopping_window/default.cfg new file mode 100644 index 000000000000..4c25fdbd7ece --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/suites/hopping_window/default.cfg @@ -0,0 +1 @@ +providers pq diff --git a/ydb/tests/fq/streaming_optimize/suites/ReadTopic.sql b/ydb/tests/fq/streaming_optimize/suites/pq/ReadTopic.sql similarity index 100% rename from ydb/tests/fq/streaming_optimize/suites/ReadTopic.sql rename to ydb/tests/fq/streaming_optimize/suites/pq/ReadTopic.sql diff --git a/ydb/tests/fq/streaming_optimize/suites/ReadTopicWithMetadata.sql b/ydb/tests/fq/streaming_optimize/suites/pq/ReadTopicWithMetadata.sql similarity index 100% rename from ydb/tests/fq/streaming_optimize/suites/ReadTopicWithMetadata.sql rename to ydb/tests/fq/streaming_optimize/suites/pq/ReadTopicWithMetadata.sql diff --git a/ydb/tests/fq/streaming_optimize/suites/ReadTopicWithMetadataInsideFilter.sql b/ydb/tests/fq/streaming_optimize/suites/pq/ReadTopicWithMetadataInsideFilter.sql similarity index 100% rename from ydb/tests/fq/streaming_optimize/suites/ReadTopicWithMetadataInsideFilter.sql rename to ydb/tests/fq/streaming_optimize/suites/pq/ReadTopicWithMetadataInsideFilter.sql diff --git a/ydb/tests/fq/streaming_optimize/suites/ReadTopicWithMetadataNestedDeep.sql b/ydb/tests/fq/streaming_optimize/suites/pq/ReadTopicWithMetadataNestedDeep.sql similarity index 100% rename from ydb/tests/fq/streaming_optimize/suites/ReadTopicWithMetadataNestedDeep.sql rename to ydb/tests/fq/streaming_optimize/suites/pq/ReadTopicWithMetadataNestedDeep.sql diff --git a/ydb/tests/fq/streaming_optimize/suites/ReadTopicWithMetadataWithFilter.sql b/ydb/tests/fq/streaming_optimize/suites/pq/ReadTopicWithMetadataWithFilter.sql similarity index 100% rename from ydb/tests/fq/streaming_optimize/suites/ReadTopicWithMetadataWithFilter.sql rename to ydb/tests/fq/streaming_optimize/suites/pq/ReadTopicWithMetadataWithFilter.sql diff --git a/ydb/tests/fq/streaming_optimize/suites/ReadTopicWithSchema.sql b/ydb/tests/fq/streaming_optimize/suites/pq/ReadTopicWithSchema.sql similarity index 100% rename from ydb/tests/fq/streaming_optimize/suites/ReadTopicWithSchema.sql rename to ydb/tests/fq/streaming_optimize/suites/pq/ReadTopicWithSchema.sql diff --git a/ydb/tests/fq/streaming_optimize/suites/ReadTwoTopics.sql b/ydb/tests/fq/streaming_optimize/suites/pq/ReadTwoTopics.sql similarity index 100% rename from ydb/tests/fq/streaming_optimize/suites/ReadTwoTopics.sql rename to ydb/tests/fq/streaming_optimize/suites/pq/ReadTwoTopics.sql diff --git a/ydb/tests/fq/streaming_optimize/suites/ReadWriteSameTopic.sql b/ydb/tests/fq/streaming_optimize/suites/pq/ReadWriteSameTopic.sql similarity index 100% rename from ydb/tests/fq/streaming_optimize/suites/ReadWriteSameTopic.sql rename to ydb/tests/fq/streaming_optimize/suites/pq/ReadWriteSameTopic.sql diff --git a/ydb/tests/fq/streaming_optimize/suites/ReadWriteTopic.sql b/ydb/tests/fq/streaming_optimize/suites/pq/ReadWriteTopic.sql similarity index 100% rename from ydb/tests/fq/streaming_optimize/suites/ReadWriteTopic.sql rename to ydb/tests/fq/streaming_optimize/suites/pq/ReadWriteTopic.sql diff --git a/ydb/tests/fq/streaming_optimize/suites/ReadWriteTopicWithSchema.sql b/ydb/tests/fq/streaming_optimize/suites/pq/ReadWriteTopicWithSchema.sql similarity index 100% rename from ydb/tests/fq/streaming_optimize/suites/ReadWriteTopicWithSchema.sql rename to ydb/tests/fq/streaming_optimize/suites/pq/ReadWriteTopicWithSchema.sql diff --git a/ydb/tests/fq/streaming_optimize/suites/WriteTwoTopics.sql b/ydb/tests/fq/streaming_optimize/suites/pq/WriteTwoTopics.sql similarity index 100% rename from ydb/tests/fq/streaming_optimize/suites/WriteTwoTopics.sql rename to ydb/tests/fq/streaming_optimize/suites/pq/WriteTwoTopics.sql diff --git a/ydb/tests/fq/streaming_optimize/suites/pq/default.cfg b/ydb/tests/fq/streaming_optimize/suites/pq/default.cfg new file mode 100644 index 000000000000..4c25fdbd7ece --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/suites/pq/default.cfg @@ -0,0 +1 @@ +providers pq diff --git a/ydb/tests/fq/streaming_optimize/suites/ReadTopicGroupWriteToSolomon.sql b/ydb/tests/fq/streaming_optimize/suites/solomon/ReadTopicGroupWriteToSolomon.sql similarity index 100% rename from ydb/tests/fq/streaming_optimize/suites/ReadTopicGroupWriteToSolomon.sql rename to ydb/tests/fq/streaming_optimize/suites/solomon/ReadTopicGroupWriteToSolomon.sql diff --git a/ydb/tests/fq/streaming_optimize/suites/default.cfg b/ydb/tests/fq/streaming_optimize/suites/solomon/default.cfg similarity index 100% rename from ydb/tests/fq/streaming_optimize/suites/default.cfg rename to ydb/tests/fq/streaming_optimize/suites/solomon/default.cfg diff --git a/ydb/tests/fq/streaming_optimize/suites/watermarks/bad_column.sqlx b/ydb/tests/fq/streaming_optimize/suites/watermarks/bad_column.sqlx new file mode 100644 index 000000000000..a8e10f91992d --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/suites/watermarks/bad_column.sqlx @@ -0,0 +1,13 @@ +PRAGMA dq.WatermarksMode="default"; +PRAGMA pq.Consumer="test_client"; + +SELECT + * +FROM pq.test_topic_input +WITH( + FORMAT=json_each_row, + SCHEMA( + ts Timestamp NOT NULL + ), + WATERMARK AS (UNWRAP(timestamp - Interval("PT5S"))) +); diff --git a/ydb/tests/fq/streaming_optimize/suites/watermarks/bad_pushdown.sqlx b/ydb/tests/fq/streaming_optimize/suites/watermarks/bad_pushdown.sqlx new file mode 100644 index 000000000000..baa4df32b51b --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/suites/watermarks/bad_pushdown.sqlx @@ -0,0 +1,13 @@ +PRAGMA dq.WatermarksMode="default"; +PRAGMA pq.Consumer="test_client"; + +SELECT + * +FROM pq.test_topic_input +WITH( + FORMAT=json_each_row, + SCHEMA( + ts Timestamp NOT NULL + ), + WATERMARK AS (Unwrap(ts - DateTime::IntervalFromSeconds(5))) +); diff --git a/ydb/tests/fq/streaming_optimize/suites/watermarks/default.cfg b/ydb/tests/fq/streaming_optimize/suites/watermarks/default.cfg new file mode 100644 index 000000000000..4c25fdbd7ece --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/suites/watermarks/default.cfg @@ -0,0 +1 @@ +providers pq diff --git a/ydb/tests/fq/streaming_optimize/suites/watermarks/watermarks.sql b/ydb/tests/fq/streaming_optimize/suites/watermarks/watermarks.sql new file mode 100644 index 000000000000..809c2cd56348 --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/suites/watermarks/watermarks.sql @@ -0,0 +1,13 @@ +PRAGMA dq.WatermarksMode="default"; +PRAGMA pq.Consumer="test_client"; + +SELECT + * +FROM pq.test_topic_input +WITH( + FORMAT=json_each_row, + SCHEMA( + ts Timestamp NOT NULL + ), + WATERMARK AS (UNWRAP(ts - Interval("PT5S"))) +); diff --git a/ydb/tests/fq/streaming_optimize/test_sql_negative.py b/ydb/tests/fq/streaming_optimize/test_sql_negative.py new file mode 100644 index 000000000000..e1f2fddd223e --- /dev/null +++ b/ydb/tests/fq/streaming_optimize/test_sql_negative.py @@ -0,0 +1,60 @@ +import os +import pytest +import re +import yatest.common + +from test_utils import pytest_generate_tests_for_run, get_case_file +from ydb.tests.fq.tools.fqrun import FqRun + +NEGATIVE_TEMPLATE = ".sqlx" +DATA_PATH = yatest.common.source_path("ydb/tests/fq/streaming_optimize/suites") + + +@pytest.fixture +def fq_run(request) -> FqRun: + result = FqRun( + config_file=os.path.join("ydb/tests/fq/streaming_optimize/cfg", "app_config.conf"), + path_prefix=f"{request.function.__name__}_" + ) + result.replace_config(lambda config: config.replace("${SOLOMON_ENDPOINT}", os.getenv("SOLOMON_HTTP_ENDPOINT"))) + result.add_topic("test_topic_input", []) + result.add_topic("test_topic_input2", []) + result.add_topic("test_topic_output", []) + result.add_topic("test_topic_output2", []) + return result + + +def sanitize_issues(s): + # 2022-08-13T16:11:21Z -> ISOTIME + # 2022-08-13T16:11:21.549879Z -> ISOTIME + s = re.sub(r"2\d{3}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(.\d+)?Z", "ISOTIME", s) + # (yexception) ... -> + s = re.sub(r"\(yexception\).*", "", s) + # library/cpp/json/json_reader.cpp:420 -> library/cpp/json/json_reader.cpp:xxx + return re.sub(r"cpp:\d+", "cpp:xxx", s) + + +def pytest_generate_tests(metafunc): + pytest_generate_tests_for_run(metafunc, NEGATIVE_TEMPLATE, data_path=DATA_PATH) + + +def test(suite, case, cfg, tmpdir, fq_run): + program_sql = get_case_file(DATA_PATH, suite, case, exts=NEGATIVE_TEMPLATE) + out_dir = tmpdir.mkdir(suite).mkdir(case).dirname + with open(program_sql, encoding="utf-8") as f: + sql_query = f.read() + + fq_run.add_query(sql_query) + result = fq_run.yql_exec(check_error=False, action="explain") + + assert result.execution_result.exit_code != 0, \ + f"execute finished without error, on file: {program_sql}, query:\n{sql_query}" + sql_stderr = result.std_err.strip() + assert sql_stderr, \ + f"exit code is {result.execution_result.exit_code}, but error is empty, on file: {program_sql}, query:\n{sql_query}" + + err_file_path = os.path.join(out_dir, "err_file.out") + with open(err_file_path, "wb") as err_file: + err_file.write(sanitize_issues(sql_stderr.decode()).encode()) + + return [yatest.common.canonical_file(err_file_path)] diff --git a/ydb/tests/fq/streaming_optimize/test_sql_streaming.py b/ydb/tests/fq/streaming_optimize/test_sql_streaming.py index 5c2a30f60b27..82fd88d7d0db 100644 --- a/ydb/tests/fq/streaming_optimize/test_sql_streaming.py +++ b/ydb/tests/fq/streaming_optimize/test_sql_streaming.py @@ -1,14 +1,13 @@ import os -import codecs -import yatest.common import pytest +import yatest.common -from test_utils import pytest_generate_tests_for_run +from test_utils import pytest_generate_tests_for_run, get_case_file from ydb.tests.fq.tools.fqrun import FqRun from yql_utils import yql_binary_path ASTDIFF_PATH = yql_binary_path('yql/essentials/tools/astdiff/astdiff') -DATA_PATH = yatest.common.source_path('ydb/tests/fq/streaming_optimize') +DATA_PATH = yatest.common.source_path('ydb/tests/fq/streaming_optimize/suites') @pytest.fixture @@ -26,12 +25,12 @@ def fq_run(request) -> FqRun: def pytest_generate_tests(metafunc): - return pytest_generate_tests_for_run(metafunc, suites=["suites"], data_path=DATA_PATH) + return pytest_generate_tests_for_run(metafunc, data_path=DATA_PATH) def test(suite, case, cfg, fq_run): - program_sql = os.path.join(DATA_PATH, suite, f"{case}.sql") - with codecs.open(program_sql, encoding="utf-8") as f: + program_sql = get_case_file(DATA_PATH, suite, case) + with open(program_sql, encoding="utf-8") as f: sql_query = f.read() fq_run.add_query(sql_query) diff --git a/ydb/tests/fq/streaming_optimize/ya.make b/ydb/tests/fq/streaming_optimize/ya.make index d3a60de74ad1..04b92db3c452 100644 --- a/ydb/tests/fq/streaming_optimize/ya.make +++ b/ydb/tests/fq/streaming_optimize/ya.make @@ -6,6 +6,7 @@ FORK_SUBTESTS() SPLIT_FACTOR(4) TEST_SRCS( + test_sql_negative.py test_sql_streaming.py ) From b39e9bbb30a1b50c90d97b18aeb1c0a362b2c215 Mon Sep 17 00:00:00 2001 From: vokayndzop Date: Sat, 1 Nov 2025 12:15:29 +0300 Subject: [PATCH 2/4] Watermarks: YQL: fix constraints commit_hash:f2ada639a636eb38defc0e8ecf61d3e35845538a --- .../core/common_opt/yql_co_simple1.cpp | 7 ++- .../core/expr_nodes/yql_expr_nodes.json | 3 +- .../core/type_ann/type_ann_list.cpp | 18 +++++- .../core/ut/yql_expr_constraint_ut.cpp | 55 ++++++++++++++----- yql/essentials/core/yql_expr_constraint.cpp | 5 ++ .../suites/Stream/MultiHoppingCore.yqls | 12 ++-- 6 files changed, 76 insertions(+), 24 deletions(-) diff --git a/yql/essentials/core/common_opt/yql_co_simple1.cpp b/yql/essentials/core/common_opt/yql_co_simple1.cpp index f109c4fb2895..db6619dc397c 100644 --- a/yql/essentials/core/common_opt/yql_co_simple1.cpp +++ b/yql/essentials/core/common_opt/yql_co_simple1.cpp @@ -3357,7 +3357,8 @@ TExprNode::TPtr RewriteAsHoppingWindowFullOutput(const TCoAggregate& aggregate, .FinishHandler(finishLambda) .SaveHandler(saveLambda) .LoadHandler(loadLambda) - .template WatermarkMode().Build(ToString(false)); + .WatermarkMode().Build(ToString(false)) + .HoppingColumn().Build(hopTraits.Column); return Build(ctx, pos) .Input(aggregate.Input()) @@ -3370,10 +3371,10 @@ TExprNode::TPtr RewriteAsHoppingWindowFullOutput(const TCoAggregate& aggregate, .SortKeySelectorLambda(timeExtractorLambda) .ListHandlerLambda() .Args(streamArg) - .template Body() + .Body() .Stream(Build(ctx, pos) .Input(multiHoppingCoreBuilder - .template Input() + .Input() .List(streamArg) .Build() .Done()) diff --git a/yql/essentials/core/expr_nodes/yql_expr_nodes.json b/yql/essentials/core/expr_nodes/yql_expr_nodes.json index 740fae8474f1..3a38d7de7d6e 100644 --- a/yql/essentials/core/expr_nodes/yql_expr_nodes.json +++ b/yql/essentials/core/expr_nodes/yql_expr_nodes.json @@ -2053,7 +2053,8 @@ {"Index": 10, "Name": "LoadHandler", "Type": "TExprBase"}, {"Index": 11, "Name": "MergeHandler", "Type": "TCoLambda"}, {"Index": 12, "Name": "FinishHandler", "Type": "TCoLambda"}, - {"Index": 13, "Name": "WatermarkMode", "Type": "TExprBase"} + {"Index": 13, "Name": "WatermarkMode", "Type": "TExprBase"}, + {"Index": 14, "Name": "HoppingColumn", "Type": "TCoAtom"} ] }, { diff --git a/yql/essentials/core/type_ann/type_ann_list.cpp b/yql/essentials/core/type_ann/type_ann_list.cpp index 94c0c0557b3e..d53cd4036f1c 100644 --- a/yql/essentials/core/type_ann/type_ann_list.cpp +++ b/yql/essentials/core/type_ann/type_ann_list.cpp @@ -7009,7 +7009,7 @@ namespace { IGraphTransformer::TStatus MultiHoppingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { Y_UNUSED(output); - if (!EnsureArgsCount(*input, 14, ctx.Expr)) { + if (!EnsureArgsCount(*input, 15, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } auto& item = input->ChildRef(0); @@ -7019,6 +7019,7 @@ namespace { auto* hop = input->Child(3); auto* interval = input->Child(4); auto* delay = input->Child(5); + auto dataWatermarks = input->Child(6); auto& lambdaInit = input->ChildRef(7); auto& lambdaUpdate = input->ChildRef(8); @@ -7027,6 +7028,9 @@ namespace { auto& lambdaMerge = input->ChildRef(11); auto& lambdaFinish = input->ChildRef(12); + auto watermarkMode = input->Child(13); + auto hoppingColumn = input->Child(14); + if (!EnsureStreamType(*item, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -7076,6 +7080,10 @@ namespace { return IGraphTransformer::TStatus::Error; } + if (!EnsureAtom(*dataWatermarks, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + if (!UpdateLambdaAllArgumentsTypes(lambdaInit, {itemType}, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -7165,6 +7173,14 @@ namespace { } } + if (!EnsureAtom(*watermarkMode, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureAtom(*hoppingColumn, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + input->SetTypeAnn(ctx.Expr.MakeType(lambdaFinish->GetTypeAnn())); return IGraphTransformer::TStatus::Ok; } diff --git a/yql/essentials/core/ut/yql_expr_constraint_ut.cpp b/yql/essentials/core/ut/yql_expr_constraint_ut.cpp index a4145b9c5a81..f60a1cc2c956 100644 --- a/yql/essentials/core/ut/yql_expr_constraint_ut.cpp +++ b/yql/essentials/core/ut/yql_expr_constraint_ut.cpp @@ -3310,8 +3310,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { CheckConstraint(exprRoot, "LazyList", ""); } - Y_UNIT_TEST(GroupByHop) { - const TStringBuf s = R"(( +Y_UNIT_TEST(HOP) { + const TStringBuf s = R"(( (let list (AsList (AsStruct '('"time" (String '"2024-01-01T00:00:01Z")) '('"user" (Int32 '"1")) '('"data" (Null))) (AsStruct '('"time" (String '"2024-01-01T00:00:02Z")) '('"user" (Int32 '"1")) '('"data" (Null))) @@ -3321,23 +3321,52 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let keySelector (lambda '(row) '((StablePickle (Member row '"data")) (StablePickle (Member row 'group0))))) (let sortKeySelector (lambda '(row) (SafeCast (Member row '"time") (OptionalType (DataType 'Timestamp))))) (let result (PartitionsByKeys input keySelector (Bool 'true) sortKeySelector (lambda '(row) (block '( - (let interval (Interval '1000000)) - (let map (lambda '(item) (AsStruct))) - (let reduce (lambda '(lhs rhs) (AsStruct))) - (let hopping (MultiHoppingCore (Iterator row) keySelector sortKeySelector interval interval interval 'true map reduce map map reduce (lambda '(key state time) (AsStruct '('_yql_time time) '('"data" (Nth key '"0")) '('group0 (Nth key '"1")))) '"0")) - (return (ForwardList (FlatMap hopping (lambda '(row) (Just (AsStruct '('_yql_time (Member row '_yql_time)) '('"data" (Unpickle (NullType) (Member row '"data"))) '('group0 (Unpickle (ListType (DataType 'Int32)) (Member row 'group0))))))))) + (let interval (Interval '1000000)) + (let map (lambda '(item) (AsStruct))) + (let reduce (lambda '(lhs rhs) (AsStruct))) + (let hopping (MultiHoppingCore (Iterator row) keySelector sortKeySelector interval interval interval 'true map reduce map map reduce (lambda '(key state time) (AsStruct '('_yql_time time) '('"data" (Nth key '"0")) '('group0 (Nth key '"1")))) '"0" '"_yql_time")) + (return (ForwardList (FlatMap hopping (lambda '(row) (Just (AsStruct '('_yql_time (Member row '_yql_time)) '('"data" (Unpickle (NullType) (Member row '"data"))) '('group0 (Unpickle (ListType (DataType 'Int32)) (Member row 'group0))))))))) ))))) (let res (DataSink 'result)) (let world (Write! world res (Key) result '())) (return (Commit! world res)) - ))"; + ))"; - TExprContext exprCtx; - const auto exprRoot = ParseAndAnnotate(s, exprCtx); - CheckConstraint(exprRoot, "PartitionsByKeys", "Distinct((data,group0))"); - CheckConstraint(exprRoot, "PartitionsByKeys", "Unique((data,group0))"); - } + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint(exprRoot, "MultiHoppingCore", "Distinct((data,group0))"); + CheckConstraint(exprRoot, "MultiHoppingCore", "Unique((data,group0))"); +} + +Y_UNIT_TEST(HoppingWindow) { + const TStringBuf s = R"(( +(let list (AsList + (AsStruct '('"time" (String '"2024-01-01T00:00:01Z")) '('"user" (Int32 '"1")) '('"data" (Null))) + (AsStruct '('"time" (String '"2024-01-01T00:00:02Z")) '('"user" (Int32 '"1")) '('"data" (Null))) + (AsStruct '('"time" (String '"2024-01-01T00:00:03Z")) '('"user" (Int32 '"1")) '('"data" (Null))) +)) +(let input (FlatMap list (lambda '(row) (Just (AsStruct '('"data" (Member row '"data")) '('group0 (AsList (Member row '"user"))) '('"time" (Member row '"time")) '('"user" (Member row '"user"))))))) +(let keySelector (lambda '(row) '((StablePickle (Member row '"data")) (StablePickle (Member row 'group0))))) +(let sortKeySelector (lambda '(row) (SafeCast (Member row '"time") (OptionalType (DataType 'Timestamp))))) +(let result (PartitionsByKeys input keySelector (Bool 'true) sortKeySelector (lambda '(row) (block '( + (let interval (Interval '1000000)) + (let map (lambda '(item) (AsStruct))) + (let reduce (lambda '(lhs rhs) (AsStruct))) + (let hopping (MultiHoppingCore (Iterator row) keySelector sortKeySelector interval interval interval 'true map reduce map map reduce (lambda '(key state time) (AsStruct '('_yql_time time) '('"data" (Nth key '"0")) '('group0 (Nth key '"1")))) '"0" 'group0)) + (return (ForwardList (FlatMap hopping (lambda '(row) (Just (AsStruct '('_yql_time (Member row '_yql_time)) '('"data" (Unpickle (NullType) (Member row '"data"))) '('group0 (Unpickle (ListType (DataType 'Int32)) (Member row 'group0))))))))) +))))) + +(let res (DataSink 'result)) +(let world (Write! world res (Key) result '())) +(return (Commit! world res)) + ))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint(exprRoot, "MultiHoppingCore", "Distinct((data,group0))"); + CheckConstraint(exprRoot, "MultiHoppingCore", "Unique((data,group0))"); +} Y_UNIT_TEST(StablePickleOfComplexUnique) { const TStringBuf s = R"( diff --git a/yql/essentials/core/yql_expr_constraint.cpp b/yql/essentials/core/yql_expr_constraint.cpp index 7fcf560a2cfa..65c57dcc6773 100644 --- a/yql/essentials/core/yql_expr_constraint.cpp +++ b/yql/essentials/core/yql_expr_constraint.cpp @@ -3053,6 +3053,11 @@ class TCallableConstraintTransformer : public TCallableTransformerBase std::string_view { return path.front(); }); + if (const auto hoppingColumn = input->Child(TCoMultiHoppingCore::idx_HoppingColumn)->Content(); + "_yql_time" != hoppingColumn) { + columns.push_back(hoppingColumn); + } + if (!columns.empty()) { input->AddConstraint(ctx.MakeConstraint(columns)); input->AddConstraint(ctx.MakeConstraint(columns)); diff --git a/yql/essentials/tests/s-expressions/suites/Stream/MultiHoppingCore.yqls b/yql/essentials/tests/s-expressions/suites/Stream/MultiHoppingCore.yqls index f05078fd268e..cd4bef42bde1 100644 --- a/yql/essentials/tests/s-expressions/suites/Stream/MultiHoppingCore.yqls +++ b/yql/essentials/tests/s-expressions/suites/Stream/MultiHoppingCore.yqls @@ -10,12 +10,12 @@ (let interval (Interval '1000000)) (let map (lambda '(item) (AsStruct))) (let reduce (lambda '(lhs rhs) (AsStruct))) -(let hopping (MultiHoppingCore (Iterator input) keySelector sortKeySelector interval interval interval - 'true map reduce map map reduce - (lambda '(key state time) (AsStruct - '('_yql_time time) - '('"data" (Nth key '"0")) - '('group0 (Nth key '"1")))) '0)) +(let hopping (MultiHoppingCore (Iterator input) keySelector sortKeySelector interval interval interval + 'true map reduce map map reduce + (lambda '(key state time) (AsStruct + '('_yql_time time) + '('"data" (Nth key '"0")) + '('group0 (Nth key '"1")))) '0 'group0)) (let data (FormatType (TypeOf hopping))) # no typical comp node (let res (DataSink 'result)) (let world (Write! world res (Key) data '('('type)))) From 1720e87e6317b34ebc7de906800e8146f4a15639 Mon Sep 17 00:00:00 2001 From: Aleksei Pozdniakov Date: Sat, 1 Nov 2025 13:42:35 +0000 Subject: [PATCH 3/4] fix sync --- ydb/library/yql/dq/opt/dq_opt_hopping.cpp | 11 ++++---- .../ast.txt | 2 +- .../ast.txt | 2 +- .../ast.txt | 2 +- .../ast.txt | 2 +- .../ast.txt | 2 +- .../ast.txt | 28 +++++++++++-------- .../ast.txt | 28 +++++++++++-------- .../ast.txt | 2 +- .../ast.txt | 2 +- .../ast.txt | 2 +- .../ast.txt | 2 +- .../ast.txt | 2 +- .../ast.txt | 2 +- .../ast.txt | 2 +- .../ast.txt | 2 +- .../ast.txt | 2 +- 17 files changed, 52 insertions(+), 43 deletions(-) diff --git a/ydb/library/yql/dq/opt/dq_opt_hopping.cpp b/ydb/library/yql/dq/opt/dq_opt_hopping.cpp index ead01ac76a6b..d3d55beab5bc 100644 --- a/ydb/library/yql/dq/opt/dq_opt_hopping.cpp +++ b/ydb/library/yql/dq/opt/dq_opt_hopping.cpp @@ -157,12 +157,13 @@ TMaybeNode RewriteAsHoppingWindowFullOutput( .FinishHandler(finishLambda) .SaveHandler(saveLambda) .LoadHandler(loadLambda) - .template WatermarkMode().Build(ToString(*enableWatermarks)); + .WatermarkMode().Build(ToString(*enableWatermarks)) + .HoppingColumn().Build(hopTraits.Column); if (*enableWatermarks) { const auto hop = hopTraits.Hop; const auto delay = lateArrivalDelay ? (lateArrivalDelay.MicroSeconds() + hop - 1) / hop * hop : hop; - multiHoppingCoreBuilder.template Delay() + multiHoppingCoreBuilder.Delay() .Literal().Build(ToString(delay)) .Build(); } else { @@ -181,9 +182,9 @@ TMaybeNode RewriteAsHoppingWindowFullOutput( .SortKeySelectorLambda(timeExtractorLambda) .ListHandlerLambda() .Args(streamArg) - .template Body() + .Body() .Stream(multiHoppingCoreBuilder - .template Input() + .Input() .List(streamArg) .Build() .Done()) @@ -208,7 +209,7 @@ TMaybeNode RewriteAsHoppingWindowFullOutput( .Args(streamArg) .Body() .Input(multiHoppingCoreBuilder - .template Input() + .Input() .Input(streamArg) .Build() .Done()) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHop-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHop-default.txt_/ast.txt index 0f7c993101a4..64c6937b664e 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHop-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHop-default.txt_/ast.txt @@ -68,7 +68,7 @@ (let $69 (CallableType '() '((OptionalType (DataType 'Timestamp))) $68)) (let $70 (Udf '"DateTime2.FromMilliseconds" (Void) (VoidType) '"" $69 (VoidType) '"" '('('"blocks") $16))) (return (Apply $70 (Convert $67 $23))) - ))))) (Interval '5000) $62 $62 'true (lambda '($71) (AsStruct '('Sum0 (Member $71 '"v")))) (lambda '($72 $73) (AsStruct '('Sum0 (AggrAdd (Member $72 '"v") (Member $73 'Sum0))))) $63 $63 (lambda '($75 $76) (AsStruct '('Sum0 (AggrAdd (Member $75 'Sum0) (Member $76 'Sum0))))) (lambda '($77 $78 $79) (AsStruct '('Sum0 (Member $78 'Sum0)) '('"_yql_time" $79) '('"k" $77))) '0)) + ))))) (Interval '5000) $62 $62 'true (lambda '($71) (AsStruct '('Sum0 (Member $71 '"v")))) (lambda '($72 $73) (AsStruct '('Sum0 (AggrAdd (Member $72 '"v") (Member $73 'Sum0))))) $63 $63 (lambda '($75 $76) (AsStruct '('Sum0 (AggrAdd (Member $75 'Sum0) (Member $76 'Sum0))))) (lambda '($77 $78 $79) (AsStruct '('Sum0 (Member $78 'Sum0)) '('"_yql_time" $79) '('"k" $77))) '0 '"_yql_time")) (return (FlatMap (ExtractMembers $64 '('Sum0 '"k")) (lambda '($80) (block '( (let $81 (Udf '"Yson2.SerializeText" (Void) (VoidType) '"" $20 (VoidType) '"" $17)) (let $82 (StructType '('"k" $21) '('"sum" $21))) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopByStringKey-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopByStringKey-default.txt_/ast.txt index 17bf4fb989be..75cf64cbfe09 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopByStringKey-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopByStringKey-default.txt_/ast.txt @@ -75,7 +75,7 @@ (let $72 (CallableType '() '((OptionalType (DataType 'Timestamp))) $71)) (let $73 (Udf '"DateTime2.FromMilliseconds" (Void) (VoidType) '"" $72 (VoidType) '"" '('('"blocks") $16))) (return (Apply $73 (Convert $70 $23))) - ))))) (Interval '5000) $65 $65 'true (lambda '($74) (AsStruct '('Sum0 (Member $74 '"v")))) (lambda '($75 $76) (AsStruct '('Sum0 (AggrAdd (Member $75 '"v") (Member $76 'Sum0))))) $66 $66 (lambda '($78 $79) (AsStruct '('Sum0 (AggrAdd (Member $78 'Sum0) (Member $79 'Sum0))))) (lambda '($80 $81 $82) (AsStruct '('Sum0 (Member $81 'Sum0)) '('"_yql_time" $82) '('"k" $80))) '0)) + ))))) (Interval '5000) $65 $65 'true (lambda '($74) (AsStruct '('Sum0 (Member $74 '"v")))) (lambda '($75 $76) (AsStruct '('Sum0 (AggrAdd (Member $75 '"v") (Member $76 'Sum0))))) $66 $66 (lambda '($78 $79) (AsStruct '('Sum0 (AggrAdd (Member $78 'Sum0) (Member $79 'Sum0))))) (lambda '($80 $81 $82) (AsStruct '('Sum0 (Member $81 'Sum0)) '('"_yql_time" $82) '('"k" $80))) '0 '"_yql_time")) (return (FlatMap (ExtractMembers $67 '('Sum0 '"k")) (lambda '($83) (block '( (let $84 (Udf '"Yson2.SerializeText" (Void) (VoidType) '"" $20 (VoidType) '"" $17)) (let $85 (StructType '('"k" $21) '('"sum" $25))) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopExprKey-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopExprKey-default.txt_/ast.txt index 0883ec3e9fb9..802fca0bc255 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopExprKey-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopExprKey-default.txt_/ast.txt @@ -68,7 +68,7 @@ (let $69 (CallableType '() '((OptionalType (DataType 'Timestamp))) $68)) (let $70 (Udf '"DateTime2.FromMilliseconds" (Void) (VoidType) '"" $69 (VoidType) '"" '('('"blocks") $16))) (return (Apply $70 (Convert $67 $23))) - ))))) (Interval '5000) $62 $62 'true (lambda '($71) (AsStruct '('Sum0 (Member $71 '"v")))) (lambda '($72 $73) (AsStruct '('Sum0 (AggrAdd (Member $72 '"v") (Member $73 'Sum0))))) $63 $63 (lambda '($75 $76) (AsStruct '('Sum0 (AggrAdd (Member $75 'Sum0) (Member $76 'Sum0))))) (lambda '($77 $78 $79) (AsStruct '('Sum0 (Member $78 'Sum0)) '('"_yql_time" $79) '('group0 $77))) '0)) + ))))) (Interval '5000) $62 $62 'true (lambda '($71) (AsStruct '('Sum0 (Member $71 '"v")))) (lambda '($72 $73) (AsStruct '('Sum0 (AggrAdd (Member $72 '"v") (Member $73 'Sum0))))) $63 $63 (lambda '($75 $76) (AsStruct '('Sum0 (AggrAdd (Member $75 'Sum0) (Member $76 'Sum0))))) (lambda '($77 $78 $79) (AsStruct '('Sum0 (Member $78 'Sum0)) '('"_yql_time" $79) '('group0 $77))) '0 '"_yql_time")) (return (FlatMap (ExtractMembers $64 '('Sum0)) (lambda '($80) (block '( (let $81 (Udf '"Yson2.SerializeText" (Void) (VoidType) '"" $20 (VoidType) '"" $17)) (let $82 (StructType '('"sum" $21))) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopListKey-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopListKey-default.txt_/ast.txt index 5d34171a7405..5458dce6e036 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopListKey-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopListKey-default.txt_/ast.txt @@ -69,7 +69,7 @@ (let $70 (CallableType '() '((OptionalType (DataType 'Timestamp))) $69)) (let $71 (Udf '"DateTime2.FromMilliseconds" (Void) (VoidType) '"" $70 (VoidType) '"" '('('"blocks") $16))) (return (Apply $71 (Convert $68 $23))) - ))))) (Interval '5000) $63 $63 'true (lambda '($72) (AsStruct '('Sum0 (Member $72 '"v")))) (lambda '($73 $74) (AsStruct '('Sum0 (AggrAdd (Member $73 '"v") (Member $74 'Sum0))))) $64 $64 (lambda '($76 $77) (AsStruct '('Sum0 (AggrAdd (Member $76 'Sum0) (Member $77 'Sum0))))) (lambda '($78 $79 $80) (AsStruct '('Sum0 (Member $79 'Sum0)) '('"_yql_time" $80) '('group0 (Nth $78 '0)) '('"k" (Nth $78 '1)))) '0)) + ))))) (Interval '5000) $63 $63 'true (lambda '($72) (AsStruct '('Sum0 (Member $72 '"v")))) (lambda '($73 $74) (AsStruct '('Sum0 (AggrAdd (Member $73 '"v") (Member $74 'Sum0))))) $64 $64 (lambda '($76 $77) (AsStruct '('Sum0 (AggrAdd (Member $76 'Sum0) (Member $77 'Sum0))))) (lambda '($78 $79 $80) (AsStruct '('Sum0 (Member $79 'Sum0)) '('"_yql_time" $80) '('group0 (Nth $78 '0)) '('"k" (Nth $78 '1)))) '0 '"_yql_time")) (return (FlatMap (ExtractMembers $65 '('Sum0)) (lambda '($81) (block '( (let $82 (Udf '"Yson2.SerializeText" (Void) (VoidType) '"" $20 (VoidType) '"" $17)) (let $83 (StructType '('"sum" $21))) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopNoKey-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopNoKey-default.txt_/ast.txt index 159a81f00aef..0d0eef4c4e00 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopNoKey-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopNoKey-default.txt_/ast.txt @@ -70,7 +70,7 @@ (let $68 (CallableType '() '((OptionalType (DataType 'Timestamp))) $67)) (let $69 (Udf '"DateTime2.FromMilliseconds" (Void) (VoidType) '"" $68 (VoidType) '"" '('('"blocks") $16))) (return (Apply $69 (Convert $66 $21))) - ))))) (Interval '5000) $61 $61 'true (lambda '($70) (AsStruct '('Sum0 (Member $70 '"v")))) (lambda '($71 $72) (AsStruct '('Sum0 (AggrAdd (Member $71 '"v") (Member $72 'Sum0))))) $62 $62 (lambda '($74 $75) (AsStruct '('Sum0 (AggrAdd (Member $74 'Sum0) (Member $75 'Sum0))))) (lambda '($76 $77 $78) (AsStruct '('Sum0 (Member $77 'Sum0)) '('"_yql_time" $78))) '0)) + ))))) (Interval '5000) $61 $61 'true (lambda '($70) (AsStruct '('Sum0 (Member $70 '"v")))) (lambda '($71 $72) (AsStruct '('Sum0 (AggrAdd (Member $71 '"v") (Member $72 'Sum0))))) $62 $62 (lambda '($74 $75) (AsStruct '('Sum0 (AggrAdd (Member $74 'Sum0) (Member $75 'Sum0))))) (lambda '($76 $77 $78) (AsStruct '('Sum0 (Member $77 'Sum0)) '('"_yql_time" $78))) '0 '"_yql_time")) (return (FlatMap (ExtractMembers $63 '('Sum0)) (lambda '($79) (block '( (let $80 (Udf '"Yson2.SerializeText" (Void) (VoidType) '"" $20 (VoidType) '"" $17)) (let $81 (StructType '('"sum" $24))) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopPercentile-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopPercentile-default.txt_/ast.txt index dbd0dd95df0c..464b776739ae 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopPercentile-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopPercentile-default.txt_/ast.txt @@ -106,18 +106,22 @@ ))))) (return (AsStruct '($63 $100))) ))) (lambda '($105 $106 $107) (block '( - (let $108 (Member $106 $63)) - (let $109 (CallableType '() $66 $64 $66)) - (let $110 (Udf '"Stat.TDigest_GetPercentile" (Void) (VoidType) '"" $109 (VoidType) '"" $18)) - (return (AsStruct '('Percentile0 (FlatMap $108 (lambda '($111) (Just (Apply $110 $111 (Double '"0.75")))))) '('Percentile1 (FlatMap $108 (lambda '($112) (Just (Apply $110 $112 (Double '"0.9")))))) '('"_yql_time" $107))) - ))) '0)) - (return (FlatMap (ExtractMembers $70 '('Percentile0 'Percentile1)) (lambda '($113) (block '( - (let $114 (Udf '"Yson2.SerializeText" (Void) (VoidType) '"" $21 (VoidType) '"" $18)) - (let $115 (StructType '('"p75" $67) '('"p90" $67))) - (let $116 (TupleType (TupleType $115) (StructType) (TupleType))) - (let $117 (CallableType '() $25 '($115))) - (let $118 (Udf '"Yson2.From" (Void) $116 '"" $117 (VoidType) '"" $18)) - (return (Just (AsStruct '('"column0" (Apply $114 (Apply $118 (AsStruct '('"p75" (Member $113 'Percentile0)) '('"p90" (Member $113 'Percentile1))))))))) + (let $108 (FlatMap (Member $106 $63) (lambda '($109) (block '( + (let $110 (CallableType '() $66 $64 $66)) + (let $111 (Udf '"Stat.TDigest_GetPercentile" (Void) (VoidType) '"" $110 (VoidType) '"" $18)) + (let $112 (Apply $111 $109 (Double '"0.75"))) + (let $113 (Apply $111 $109 (Double '"0.9"))) + (return (Just '($112 $113))) + ))))) + (return (AsStruct '('Percentile0 (Nth $108 '0)) '('Percentile1 (Nth $108 '1)) '('"_yql_time" $107))) + ))) '0 '"_yql_time")) + (return (FlatMap (ExtractMembers $70 '('Percentile0 'Percentile1)) (lambda '($114) (block '( + (let $115 (Udf '"Yson2.SerializeText" (Void) (VoidType) '"" $21 (VoidType) '"" $18)) + (let $116 (StructType '('"p75" $67) '('"p90" $67))) + (let $117 (TupleType (TupleType $116) (StructType) (TupleType))) + (let $118 (CallableType '() $25 '($116))) + (let $119 (Udf '"Yson2.From" (Void) $117 '"" $118 (VoidType) '"" $18)) + (return (Just (AsStruct '('"column0" (Apply $115 (Apply $119 (AsStruct '('"p75" (Member $114 'Percentile0)) '('"p90" (Member $114 'Percentile1))))))))) ))))) ))) '('('"_logical_id" '0)) '((DqSink '0 $27 $29))))) $27)) ) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopTimeExtractorUnusedColumns-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopTimeExtractorUnusedColumns-default.txt_/ast.txt index dbd0dd95df0c..464b776739ae 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopTimeExtractorUnusedColumns-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopTimeExtractorUnusedColumns-default.txt_/ast.txt @@ -106,18 +106,22 @@ ))))) (return (AsStruct '($63 $100))) ))) (lambda '($105 $106 $107) (block '( - (let $108 (Member $106 $63)) - (let $109 (CallableType '() $66 $64 $66)) - (let $110 (Udf '"Stat.TDigest_GetPercentile" (Void) (VoidType) '"" $109 (VoidType) '"" $18)) - (return (AsStruct '('Percentile0 (FlatMap $108 (lambda '($111) (Just (Apply $110 $111 (Double '"0.75")))))) '('Percentile1 (FlatMap $108 (lambda '($112) (Just (Apply $110 $112 (Double '"0.9")))))) '('"_yql_time" $107))) - ))) '0)) - (return (FlatMap (ExtractMembers $70 '('Percentile0 'Percentile1)) (lambda '($113) (block '( - (let $114 (Udf '"Yson2.SerializeText" (Void) (VoidType) '"" $21 (VoidType) '"" $18)) - (let $115 (StructType '('"p75" $67) '('"p90" $67))) - (let $116 (TupleType (TupleType $115) (StructType) (TupleType))) - (let $117 (CallableType '() $25 '($115))) - (let $118 (Udf '"Yson2.From" (Void) $116 '"" $117 (VoidType) '"" $18)) - (return (Just (AsStruct '('"column0" (Apply $114 (Apply $118 (AsStruct '('"p75" (Member $113 'Percentile0)) '('"p90" (Member $113 'Percentile1))))))))) + (let $108 (FlatMap (Member $106 $63) (lambda '($109) (block '( + (let $110 (CallableType '() $66 $64 $66)) + (let $111 (Udf '"Stat.TDigest_GetPercentile" (Void) (VoidType) '"" $110 (VoidType) '"" $18)) + (let $112 (Apply $111 $109 (Double '"0.75"))) + (let $113 (Apply $111 $109 (Double '"0.9"))) + (return (Just '($112 $113))) + ))))) + (return (AsStruct '('Percentile0 (Nth $108 '0)) '('Percentile1 (Nth $108 '1)) '('"_yql_time" $107))) + ))) '0 '"_yql_time")) + (return (FlatMap (ExtractMembers $70 '('Percentile0 'Percentile1)) (lambda '($114) (block '( + (let $115 (Udf '"Yson2.SerializeText" (Void) (VoidType) '"" $21 (VoidType) '"" $18)) + (let $116 (StructType '('"p75" $67) '('"p90" $67))) + (let $117 (TupleType (TupleType $116) (StructType) (TupleType))) + (let $118 (CallableType '() $25 '($116))) + (let $119 (Udf '"Yson2.From" (Void) $117 '"" $118 (VoidType) '"" $18)) + (return (Just (AsStruct '('"column0" (Apply $115 (Apply $119 (AsStruct '('"p75" (Member $114 'Percentile0)) '('"p90" (Member $114 'Percentile1))))))))) ))))) ))) '('('"_logical_id" '0)) '((DqSink '0 $27 $29))))) $27)) ) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopWithDataWatermarks-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopWithDataWatermarks-default.txt_/ast.txt index 0f7c993101a4..64c6937b664e 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopWithDataWatermarks-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopWithDataWatermarks-default.txt_/ast.txt @@ -68,7 +68,7 @@ (let $69 (CallableType '() '((OptionalType (DataType 'Timestamp))) $68)) (let $70 (Udf '"DateTime2.FromMilliseconds" (Void) (VoidType) '"" $69 (VoidType) '"" '('('"blocks") $16))) (return (Apply $70 (Convert $67 $23))) - ))))) (Interval '5000) $62 $62 'true (lambda '($71) (AsStruct '('Sum0 (Member $71 '"v")))) (lambda '($72 $73) (AsStruct '('Sum0 (AggrAdd (Member $72 '"v") (Member $73 'Sum0))))) $63 $63 (lambda '($75 $76) (AsStruct '('Sum0 (AggrAdd (Member $75 'Sum0) (Member $76 'Sum0))))) (lambda '($77 $78 $79) (AsStruct '('Sum0 (Member $78 'Sum0)) '('"_yql_time" $79) '('"k" $77))) '0)) + ))))) (Interval '5000) $62 $62 'true (lambda '($71) (AsStruct '('Sum0 (Member $71 '"v")))) (lambda '($72 $73) (AsStruct '('Sum0 (AggrAdd (Member $72 '"v") (Member $73 'Sum0))))) $63 $63 (lambda '($75 $76) (AsStruct '('Sum0 (AggrAdd (Member $75 'Sum0) (Member $76 'Sum0))))) (lambda '($77 $78 $79) (AsStruct '('Sum0 (Member $78 'Sum0)) '('"_yql_time" $79) '('"k" $77))) '0 '"_yql_time")) (return (FlatMap (ExtractMembers $64 '('Sum0 '"k")) (lambda '($80) (block '( (let $81 (Udf '"Yson2.SerializeText" (Void) (VoidType) '"" $20 (VoidType) '"" $17)) (let $82 (StructType '('"k" $21) '('"sum" $21))) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindow-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindow-default.txt_/ast.txt index 2f8a78368260..df020508465a 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindow-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindow-default.txt_/ast.txt @@ -37,7 +37,7 @@ (let $41 (CallableType '() '((OptionalType (DataType 'Timestamp))) $40)) (let $42 (Udf '"DateTime2.FromMilliseconds" (Void) (VoidType) '"" $41 (VoidType) '"" '('('"blocks") $34))) (return (Apply $42 $39)) - ))))) (Interval '"5000") (Interval '"10000") (Interval '"5000000") 'true (lambda '($43) (AsStruct '('Sum0 (Member $43 '"v")))) (lambda '($44 $45) (AsStruct '('Sum0 (AggrAdd (Member $44 '"v") (Member $45 'Sum0))))) $35 $35 (lambda '($47 $48) (AsStruct '('Sum0 (AggrAdd (Member $47 'Sum0) (Member $48 'Sum0))))) (lambda '($49 $50 $51) (AsStruct '('Sum0 (Member $50 'Sum0)) '('"group0" $51) '('"k" $49))) '"1")) + ))))) (Interval '"5000") (Interval '"10000") (Interval '"5000000") 'true (lambda '($43) (AsStruct '('Sum0 (Member $43 '"v")))) (lambda '($44 $45) (AsStruct '('Sum0 (AggrAdd (Member $44 '"v") (Member $45 'Sum0))))) $35 $35 (lambda '($47 $48) (AsStruct '('Sum0 (AggrAdd (Member $47 'Sum0) (Member $48 'Sum0))))) (lambda '($49 $50 $51) (AsStruct '('Sum0 (Member $50 'Sum0)) '('"group0" $51) '('"k" $49))) '"1" '"group0")) (return (FlatMap (ExtractMembers $36 '('Sum0 '"k")) (lambda '($52) (block '( (let $53 (ResourceType '"Yson2.Node")) (let $54 '($53 '"" '"1")) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowByStringKey-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowByStringKey-default.txt_/ast.txt index 2f8a78368260..df020508465a 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowByStringKey-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowByStringKey-default.txt_/ast.txt @@ -37,7 +37,7 @@ (let $41 (CallableType '() '((OptionalType (DataType 'Timestamp))) $40)) (let $42 (Udf '"DateTime2.FromMilliseconds" (Void) (VoidType) '"" $41 (VoidType) '"" '('('"blocks") $34))) (return (Apply $42 $39)) - ))))) (Interval '"5000") (Interval '"10000") (Interval '"5000000") 'true (lambda '($43) (AsStruct '('Sum0 (Member $43 '"v")))) (lambda '($44 $45) (AsStruct '('Sum0 (AggrAdd (Member $44 '"v") (Member $45 'Sum0))))) $35 $35 (lambda '($47 $48) (AsStruct '('Sum0 (AggrAdd (Member $47 'Sum0) (Member $48 'Sum0))))) (lambda '($49 $50 $51) (AsStruct '('Sum0 (Member $50 'Sum0)) '('"group0" $51) '('"k" $49))) '"1")) + ))))) (Interval '"5000") (Interval '"10000") (Interval '"5000000") 'true (lambda '($43) (AsStruct '('Sum0 (Member $43 '"v")))) (lambda '($44 $45) (AsStruct '('Sum0 (AggrAdd (Member $44 '"v") (Member $45 'Sum0))))) $35 $35 (lambda '($47 $48) (AsStruct '('Sum0 (AggrAdd (Member $47 'Sum0) (Member $48 'Sum0))))) (lambda '($49 $50 $51) (AsStruct '('Sum0 (Member $50 'Sum0)) '('"group0" $51) '('"k" $49))) '"1" '"group0")) (return (FlatMap (ExtractMembers $36 '('Sum0 '"k")) (lambda '($52) (block '( (let $53 (ResourceType '"Yson2.Node")) (let $54 '($53 '"" '"1")) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowExprKey-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowExprKey-default.txt_/ast.txt index 73c0a61b6b0b..d96fdc59a893 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowExprKey-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowExprKey-default.txt_/ast.txt @@ -35,7 +35,7 @@ (let $40 (CallableType '() '((OptionalType (DataType 'Timestamp))) $39)) (let $41 (Udf '"DateTime2.FromMilliseconds" (Void) (VoidType) '"" $40 (VoidType) '"" '('('"blocks") $33))) (return (Apply $41 $38)) - ))))) (Interval '"5000") (Interval '"10000") (Interval '"5000000") 'true (lambda '($42) (AsStruct '('Sum0 (Member $42 '"v")))) (lambda '($43 $44) (AsStruct '('Sum0 (AggrAdd (Member $43 '"v") (Member $44 'Sum0))))) $34 $34 (lambda '($46 $47) (AsStruct '('Sum0 (AggrAdd (Member $46 'Sum0) (Member $47 'Sum0))))) (lambda '($48 $49 $50) (AsStruct '('Sum0 (Member $49 'Sum0)) '('group0 $48) '('"group1" $50))) '"1")) + ))))) (Interval '"5000") (Interval '"10000") (Interval '"5000000") 'true (lambda '($42) (AsStruct '('Sum0 (Member $42 '"v")))) (lambda '($43 $44) (AsStruct '('Sum0 (AggrAdd (Member $43 '"v") (Member $44 'Sum0))))) $34 $34 (lambda '($46 $47) (AsStruct '('Sum0 (AggrAdd (Member $46 'Sum0) (Member $47 'Sum0))))) (lambda '($48 $49 $50) (AsStruct '('Sum0 (Member $49 'Sum0)) '('group0 $48) '('"group1" $50))) '"1" '"group1")) (return (FlatMap (ExtractMembers $35 '('Sum0)) (lambda '($51) (block '( (let $52 (ResourceType '"Yson2.Node")) (let $53 '($52 '"" '"1")) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowListKey-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowListKey-default.txt_/ast.txt index 48f9cd9fa2c1..3d10d7f5e2b1 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowListKey-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowListKey-default.txt_/ast.txt @@ -38,7 +38,7 @@ (let $41 (CallableType '() '((OptionalType (DataType 'Timestamp))) $40)) (let $42 (Udf '"DateTime2.FromMilliseconds" (Void) (VoidType) '"" $41 (VoidType) '"" '('('"blocks") $34))) (return (Apply $42 $39)) - ))))) (Interval '"5000") (Interval '"10000") (Interval '"5000000") 'true (lambda '($43) (AsStruct '('Sum0 (Member $43 '"v")))) (lambda '($44 $45) (AsStruct '('Sum0 (AggrAdd (Member $44 '"v") (Member $45 'Sum0))))) $35 $35 (lambda '($47 $48) (AsStruct '('Sum0 (AggrAdd (Member $47 'Sum0) (Member $48 'Sum0))))) (lambda '($49 $50 $51) (AsStruct '('Sum0 (Member $50 'Sum0)) '('group0 (Nth $49 '0)) '('"group1" $51) '('"k" (Nth $49 '1)))) '1)) + ))))) (Interval '"5000") (Interval '"10000") (Interval '"5000000") 'true (lambda '($43) (AsStruct '('Sum0 (Member $43 '"v")))) (lambda '($44 $45) (AsStruct '('Sum0 (AggrAdd (Member $44 '"v") (Member $45 'Sum0))))) $35 $35 (lambda '($47 $48) (AsStruct '('Sum0 (AggrAdd (Member $47 'Sum0) (Member $48 'Sum0))))) (lambda '($49 $50 $51) (AsStruct '('Sum0 (Member $50 'Sum0)) '('group0 (Nth $49 '0)) '('"group1" $51) '('"k" (Nth $49 '1)))) '1 '"group1")) (return (FlatMap (ExtractMembers $36 '('Sum0)) (lambda '($52) (block '( (let $53 (ResourceType '"Yson2.Node")) (let $54 '($53 '"" '1)) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowNoKey-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowNoKey-default.txt_/ast.txt index fd18956704bc..16447d13ceb5 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowNoKey-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowNoKey-default.txt_/ast.txt @@ -34,7 +34,7 @@ (let $37 (CallableType '() '((OptionalType (DataType 'Timestamp))) $36)) (let $38 (Udf '"DateTime2.FromMilliseconds" (Void) (VoidType) '"" $37 (VoidType) '"" '('('"blocks") $30))) (return (Apply $38 $35)) - ))))) (Interval '"5000") (Interval '"10000") (Interval '"5000000") 'true (lambda '($39) (AsStruct '('Sum0 (Member $39 '"v")))) (lambda '($40 $41) (AsStruct '('Sum0 (AggrAdd (Member $40 '"v") (Member $41 'Sum0))))) $31 $31 (lambda '($43 $44) (AsStruct '('Sum0 (AggrAdd (Member $43 'Sum0) (Member $44 'Sum0))))) (lambda '($45 $46 $47) (AsStruct '('Sum0 (Member $46 'Sum0)) '('"group0" $47))) '"1")) + ))))) (Interval '"5000") (Interval '"10000") (Interval '"5000000") 'true (lambda '($39) (AsStruct '('Sum0 (Member $39 '"v")))) (lambda '($40 $41) (AsStruct '('Sum0 (AggrAdd (Member $40 '"v") (Member $41 'Sum0))))) $31 $31 (lambda '($43 $44) (AsStruct '('Sum0 (AggrAdd (Member $43 'Sum0) (Member $44 'Sum0))))) (lambda '($45 $46 $47) (AsStruct '('Sum0 (Member $46 'Sum0)) '('"group0" $47))) '"1" '"group0")) (return (FlatMap (ExtractMembers $32 '('Sum0)) (lambda '($48) (block '( (let $49 (ResourceType '"Yson2.Node")) (let $50 '($49 '"" '"1")) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowPercentile-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowPercentile-default.txt_/ast.txt index 4d178658ba0f..7ed2e1c77831 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowPercentile-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowPercentile-default.txt_/ast.txt @@ -81,7 +81,7 @@ (return (Just '($84 $85))) ))))) (return (AsStruct '('Percentile0 (Nth $80 '0)) '('Percentile1 (Nth $80 '1)) '('"group0" $79))) - ))) '1)) + ))) '1 '"group0")) (return (FlatMap (ExtractMembers $42 '('Percentile0 'Percentile1)) (lambda '($86) (block '( (let $87 (ResourceType '"Yson2.Node")) (let $88 '($87 '"" '1)) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowTimeExtractorUnusedColumns-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowTimeExtractorUnusedColumns-default.txt_/ast.txt index 4d178658ba0f..7ed2e1c77831 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowTimeExtractorUnusedColumns-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowTimeExtractorUnusedColumns-default.txt_/ast.txt @@ -81,7 +81,7 @@ (return (Just '($84 $85))) ))))) (return (AsStruct '('Percentile0 (Nth $80 '0)) '('Percentile1 (Nth $80 '1)) '('"group0" $79))) - ))) '1)) + ))) '1 '"group0")) (return (FlatMap (ExtractMembers $42 '('Percentile0 'Percentile1)) (lambda '($86) (block '( (let $87 (ResourceType '"Yson2.Node")) (let $88 '($87 '"" '1)) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_solomon-ReadTopicGroupWriteToSolomon-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_solomon-ReadTopicGroupWriteToSolomon-default.txt_/ast.txt index adacffa1b577..567bd51ae9df 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_solomon-ReadTopicGroupWriteToSolomon-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_solomon-ReadTopicGroupWriteToSolomon-default.txt_/ast.txt @@ -31,7 +31,7 @@ ))) (lambda '($31 $32) (block '( (let $33 (Size (Member $31 '"Data"))) (return (AsStruct '('Count0 (Inc (Member $32 'Count0))) '('Max0 (AggrMax $33 (Member $32 'Max0))) '('Min0 (AggrMin $33 (Member $32 'Min0))) '('Sum0 (AggrAdd (SafeCast $33 $12) (Member $32 'Sum0))))) - ))) $25 $25 (lambda '($35 $36) (AsStruct '('Count0 (AggrAdd (Member $35 'Count0) (Member $36 'Count0))) '('Max0 (AggrMax (Member $35 'Max0) (Member $36 'Max0))) '('Min0 (AggrMin (Member $35 'Min0) (Member $36 'Min0))) '('Sum0 (AggrAdd (Member $35 'Sum0) (Member $36 'Sum0))))) (lambda '($37 $38 $39) (AsStruct '('Count0 (Member $38 'Count0)) '('Max0 (Member $38 'Max0)) '('Min0 (Member $38 'Min0)) '('Sum0 (Member $38 'Sum0)) '('_yql_time $39) '('"crc" $37))) '0)) + ))) $25 $25 (lambda '($35 $36) (AsStruct '('Count0 (AggrAdd (Member $35 'Count0) (Member $36 'Count0))) '('Max0 (AggrMax (Member $35 'Max0) (Member $36 'Max0))) '('Min0 (AggrMin (Member $35 'Min0) (Member $36 'Min0))) '('Sum0 (AggrAdd (Member $35 'Sum0) (Member $36 'Sum0))))) (lambda '($37 $38 $39) (AsStruct '('Count0 (Member $38 'Count0)) '('Max0 (Member $38 'Max0)) '('Min0 (Member $38 'Min0)) '('Sum0 (Member $38 'Sum0)) '('_yql_time $39) '('"crc" $37))) '0 '_yql_time)) (return (FlatMap $26 (lambda '($40) (Just (AsStruct '('"count" (Member $40 'Count0)) '('"crc8" (SafeCast (Member $40 '"crc") $5)) '('"max_length" (Member $40 'Max0)) '('"min_length" (Member $40 'Min0)) '('"sum" (Member $40 'Sum0)) '('"ts" (Unwrap (Member $40 '_yql_time)))))))) ))) '('('"_logical_id" '0)) '((DqSink '0 $13 $16))))) (DataSink '"pq" '"pq")) $13)) ) From 657727fda98cf2c8da5a05c56a2af8ef417f5abd Mon Sep 17 00:00:00 2001 From: Aleksei Pozdniakov Date: Wed, 5 Nov 2025 13:46:44 +0000 Subject: [PATCH 4/4] fix streaming tests --- .../streaming_optimize/canondata/result.json | 10 ---- .../err_file.out | 10 ---- .../err_file.out | 8 --- .../ast.txt | 26 ++++---- .../ast.txt | 26 ++++---- .../ast.txt | 32 +++++----- .../ast.txt | 32 +++++----- .../suites/watermarks/bad_column.sqlx | 13 ---- .../suites/watermarks/bad_pushdown.sqlx | 13 ---- .../streaming_optimize/test_sql_negative.py | 60 ------------------- .../streaming_optimize/test_sql_streaming.py | 4 +- ydb/tests/fq/streaming_optimize/ya.make | 1 - 12 files changed, 52 insertions(+), 183 deletions(-) delete mode 100644 ydb/tests/fq/streaming_optimize/canondata/test_sql_negative.test_watermarks-bad_column-default.txt_/err_file.out delete mode 100644 ydb/tests/fq/streaming_optimize/canondata/test_sql_negative.test_watermarks-bad_pushdown-default.txt_/err_file.out delete mode 100644 ydb/tests/fq/streaming_optimize/suites/watermarks/bad_column.sqlx delete mode 100644 ydb/tests/fq/streaming_optimize/suites/watermarks/bad_pushdown.sqlx delete mode 100644 ydb/tests/fq/streaming_optimize/test_sql_negative.py diff --git a/ydb/tests/fq/streaming_optimize/canondata/result.json b/ydb/tests/fq/streaming_optimize/canondata/result.json index a6e9dbfe9651..365a1b967571 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/result.json +++ b/ydb/tests/fq/streaming_optimize/canondata/result.json @@ -1,14 +1,4 @@ { - "test_sql_negative.test[watermarks-bad_column-default.txt]": [ - { - "uri": "file://test_sql_negative.test_watermarks-bad_column-default.txt_/err_file.out" - } - ], - "test_sql_negative.test[watermarks-bad_pushdown-default.txt]": [ - { - "uri": "file://test_sql_negative.test_watermarks-bad_pushdown-default.txt_/err_file.out" - } - ], "test_sql_streaming.test[hop-GroupByHop-default.txt]": { "Ast": { "uri": "file://test_sql_streaming.test_hop-GroupByHop-default.txt_/ast.txt" diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_negative.test_watermarks-bad_column-default.txt_/err_file.out b/ydb/tests/fq/streaming_optimize/canondata/test_sql_negative.test_watermarks-bad_column-default.txt_/err_file.out deleted file mode 100644 index 981d1c4232a9..000000000000 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_negative.test_watermarks-bad_column-default.txt_/err_file.out +++ /dev/null @@ -1,10 +0,0 @@ -Failed to execute query, invalid final status FAILED, issues: -
: Error: Query failed with code ABORTED at ISOTIME -
: Error: Run query failed: Error -
: Error: Type annotation, code: 1030 -
:4:1: Error: At function: RemovePrefixMembers, At function: RemoveSystemMembers, At function: PersistableRepr, At function: SqlProject -
:12:5: Error: At lambda -
:12:19: Error: At function: Unwrap -
:12:36: Error: At function: - -
:12:26: Error: Member not found: timestamp - diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_negative.test_watermarks-bad_pushdown-default.txt_/err_file.out b/ydb/tests/fq/streaming_optimize/canondata/test_sql_negative.test_watermarks-bad_pushdown-default.txt_/err_file.out deleted file mode 100644 index 84ee45fb6765..000000000000 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_negative.test_watermarks-bad_pushdown-default.txt_/err_file.out +++ /dev/null @@ -1,8 +0,0 @@ -Failed to execute query, invalid final status FAILED, issues: -
: Error: Query failed with code ABORTED at ISOTIME -
: Error: Run query failed: Error -
: Error: Type annotation, code: 1030 -
:4:1: Error: At function: RemovePrefixMembers, At function: RemoveSystemMembers, At function: PersistableRepr, At function: SqlProject -
:6:6: Error: At function: PqReadTopic! -
:12:5: Error: Bad watermark expression - diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopPercentile-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopPercentile-default.txt_/ast.txt index 464b776739ae..0e054b7c82bc 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopPercentile-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopPercentile-default.txt_/ast.txt @@ -106,22 +106,18 @@ ))))) (return (AsStruct '($63 $100))) ))) (lambda '($105 $106 $107) (block '( - (let $108 (FlatMap (Member $106 $63) (lambda '($109) (block '( - (let $110 (CallableType '() $66 $64 $66)) - (let $111 (Udf '"Stat.TDigest_GetPercentile" (Void) (VoidType) '"" $110 (VoidType) '"" $18)) - (let $112 (Apply $111 $109 (Double '"0.75"))) - (let $113 (Apply $111 $109 (Double '"0.9"))) - (return (Just '($112 $113))) - ))))) - (return (AsStruct '('Percentile0 (Nth $108 '0)) '('Percentile1 (Nth $108 '1)) '('"_yql_time" $107))) + (let $108 (Member $106 $63)) + (let $109 (CallableType '() $66 $64 $66)) + (let $110 (Udf '"Stat.TDigest_GetPercentile" (Void) (VoidType) '"" $109 (VoidType) '"" $18)) + (return (AsStruct '('Percentile0 (FlatMap $108 (lambda '($111) (Just (Apply $110 $111 (Double '"0.75")))))) '('Percentile1 (FlatMap $108 (lambda '($112) (Just (Apply $110 $112 (Double '"0.9")))))) '('"_yql_time" $107))) ))) '0 '"_yql_time")) - (return (FlatMap (ExtractMembers $70 '('Percentile0 'Percentile1)) (lambda '($114) (block '( - (let $115 (Udf '"Yson2.SerializeText" (Void) (VoidType) '"" $21 (VoidType) '"" $18)) - (let $116 (StructType '('"p75" $67) '('"p90" $67))) - (let $117 (TupleType (TupleType $116) (StructType) (TupleType))) - (let $118 (CallableType '() $25 '($116))) - (let $119 (Udf '"Yson2.From" (Void) $117 '"" $118 (VoidType) '"" $18)) - (return (Just (AsStruct '('"column0" (Apply $115 (Apply $119 (AsStruct '('"p75" (Member $114 'Percentile0)) '('"p90" (Member $114 'Percentile1))))))))) + (return (FlatMap (ExtractMembers $70 '('Percentile0 'Percentile1)) (lambda '($113) (block '( + (let $114 (Udf '"Yson2.SerializeText" (Void) (VoidType) '"" $21 (VoidType) '"" $18)) + (let $115 (StructType '('"p75" $67) '('"p90" $67))) + (let $116 (TupleType (TupleType $115) (StructType) (TupleType))) + (let $117 (CallableType '() $25 '($115))) + (let $118 (Udf '"Yson2.From" (Void) $116 '"" $117 (VoidType) '"" $18)) + (return (Just (AsStruct '('"column0" (Apply $114 (Apply $118 (AsStruct '('"p75" (Member $113 'Percentile0)) '('"p90" (Member $113 'Percentile1))))))))) ))))) ))) '('('"_logical_id" '0)) '((DqSink '0 $27 $29))))) $27)) ) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopTimeExtractorUnusedColumns-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopTimeExtractorUnusedColumns-default.txt_/ast.txt index 464b776739ae..0e054b7c82bc 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopTimeExtractorUnusedColumns-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hop-GroupByHopTimeExtractorUnusedColumns-default.txt_/ast.txt @@ -106,22 +106,18 @@ ))))) (return (AsStruct '($63 $100))) ))) (lambda '($105 $106 $107) (block '( - (let $108 (FlatMap (Member $106 $63) (lambda '($109) (block '( - (let $110 (CallableType '() $66 $64 $66)) - (let $111 (Udf '"Stat.TDigest_GetPercentile" (Void) (VoidType) '"" $110 (VoidType) '"" $18)) - (let $112 (Apply $111 $109 (Double '"0.75"))) - (let $113 (Apply $111 $109 (Double '"0.9"))) - (return (Just '($112 $113))) - ))))) - (return (AsStruct '('Percentile0 (Nth $108 '0)) '('Percentile1 (Nth $108 '1)) '('"_yql_time" $107))) + (let $108 (Member $106 $63)) + (let $109 (CallableType '() $66 $64 $66)) + (let $110 (Udf '"Stat.TDigest_GetPercentile" (Void) (VoidType) '"" $109 (VoidType) '"" $18)) + (return (AsStruct '('Percentile0 (FlatMap $108 (lambda '($111) (Just (Apply $110 $111 (Double '"0.75")))))) '('Percentile1 (FlatMap $108 (lambda '($112) (Just (Apply $110 $112 (Double '"0.9")))))) '('"_yql_time" $107))) ))) '0 '"_yql_time")) - (return (FlatMap (ExtractMembers $70 '('Percentile0 'Percentile1)) (lambda '($114) (block '( - (let $115 (Udf '"Yson2.SerializeText" (Void) (VoidType) '"" $21 (VoidType) '"" $18)) - (let $116 (StructType '('"p75" $67) '('"p90" $67))) - (let $117 (TupleType (TupleType $116) (StructType) (TupleType))) - (let $118 (CallableType '() $25 '($116))) - (let $119 (Udf '"Yson2.From" (Void) $117 '"" $118 (VoidType) '"" $18)) - (return (Just (AsStruct '('"column0" (Apply $115 (Apply $119 (AsStruct '('"p75" (Member $114 'Percentile0)) '('"p90" (Member $114 'Percentile1))))))))) + (return (FlatMap (ExtractMembers $70 '('Percentile0 'Percentile1)) (lambda '($113) (block '( + (let $114 (Udf '"Yson2.SerializeText" (Void) (VoidType) '"" $21 (VoidType) '"" $18)) + (let $115 (StructType '('"p75" $67) '('"p90" $67))) + (let $116 (TupleType (TupleType $115) (StructType) (TupleType))) + (let $117 (CallableType '() $25 '($115))) + (let $118 (Udf '"Yson2.From" (Void) $116 '"" $117 (VoidType) '"" $18)) + (return (Just (AsStruct '('"column0" (Apply $114 (Apply $118 (AsStruct '('"p75" (Member $113 'Percentile0)) '('"p90" (Member $113 'Percentile1))))))))) ))))) ))) '('('"_logical_id" '0)) '((DqSink '0 $27 $29))))) $27)) ) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowPercentile-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowPercentile-default.txt_/ast.txt index 7ed2e1c77831..fb7bf454dbc0 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowPercentile-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowPercentile-default.txt_/ast.txt @@ -73,25 +73,21 @@ ))))) (return (AsStruct '($33 $72))) ))) (lambda '($77 $78 $79) (block '( - (let $80 (FlatMap (Member $78 $33) (lambda '($81) (block '( - (let $82 (CallableType '() $36 $34 $36)) - (let $83 (Udf '"Stat.TDigest_GetPercentile" (Void) (VoidType) '"" $82 (VoidType) '"" $40)) - (let $84 (Apply $83 $81 (Double '"0.75"))) - (let $85 (Apply $83 $81 (Double '"0.9"))) - (return (Just '($84 $85))) - ))))) - (return (AsStruct '('Percentile0 (Nth $80 '0)) '('Percentile1 (Nth $80 '1)) '('"group0" $79))) + (let $80 (Member $78 $33)) + (let $81 (CallableType '() $36 $34 $36)) + (let $82 (Udf '"Stat.TDigest_GetPercentile" (Void) (VoidType) '"" $81 (VoidType) '"" $40)) + (return (AsStruct '('Percentile0 (FlatMap $80 (lambda '($83) (Just (Apply $82 $83 (Double '"0.75")))))) '('Percentile1 (FlatMap $80 (lambda '($84) (Just (Apply $82 $84 (Double '"0.9")))))) '('"group0" $79))) ))) '1 '"group0")) - (return (FlatMap (ExtractMembers $42 '('Percentile0 'Percentile1)) (lambda '($86) (block '( - (let $87 (ResourceType '"Yson2.Node")) - (let $88 '($87 '"" '1)) - (let $89 (CallableType '() '((DataType 'Yson)) $88)) - (let $90 (Udf '"Yson2.SerializeText" (Void) (VoidType) '"" $89 (VoidType) '"" $40)) - (let $91 (StructType '('"p75" $37) '('"p90" $37))) - (let $92 (TupleType (TupleType $91) (StructType) (TupleType))) - (let $93 (CallableType '() '($87) '($91))) - (let $94 (Udf '"Yson2.From" (Void) $92 '"" $93 (VoidType) '"" $40)) - (return (Just (AsStruct '('"column0" (Apply $90 (Apply $94 (AsStruct '('"p75" (Member $86 'Percentile0)) '('"p90" (Member $86 'Percentile1))))))))) + (return (FlatMap (ExtractMembers $42 '('Percentile0 'Percentile1)) (lambda '($85) (block '( + (let $86 (ResourceType '"Yson2.Node")) + (let $87 '($86 '"" '1)) + (let $88 (CallableType '() '((DataType 'Yson)) $87)) + (let $89 (Udf '"Yson2.SerializeText" (Void) (VoidType) '"" $88 (VoidType) '"" $40)) + (let $90 (StructType '('"p75" $37) '('"p90" $37))) + (let $91 (TupleType (TupleType $90) (StructType) (TupleType))) + (let $92 (CallableType '() '($86) '($90))) + (let $93 (Udf '"Yson2.From" (Void) $91 '"" $92 (VoidType) '"" $40)) + (return (Just (AsStruct '('"column0" (Apply $89 (Apply $93 (AsStruct '('"p75" (Member $85 'Percentile0)) '('"p90" (Member $85 'Percentile1))))))))) ))))) ))) '('('"_logical_id" '0)) '((DqSink '0 $22 $24))))) $22)) ) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowTimeExtractorUnusedColumns-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowTimeExtractorUnusedColumns-default.txt_/ast.txt index 7ed2e1c77831..fb7bf454dbc0 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowTimeExtractorUnusedColumns-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_hopping_window-GroupByHoppingWindowTimeExtractorUnusedColumns-default.txt_/ast.txt @@ -73,25 +73,21 @@ ))))) (return (AsStruct '($33 $72))) ))) (lambda '($77 $78 $79) (block '( - (let $80 (FlatMap (Member $78 $33) (lambda '($81) (block '( - (let $82 (CallableType '() $36 $34 $36)) - (let $83 (Udf '"Stat.TDigest_GetPercentile" (Void) (VoidType) '"" $82 (VoidType) '"" $40)) - (let $84 (Apply $83 $81 (Double '"0.75"))) - (let $85 (Apply $83 $81 (Double '"0.9"))) - (return (Just '($84 $85))) - ))))) - (return (AsStruct '('Percentile0 (Nth $80 '0)) '('Percentile1 (Nth $80 '1)) '('"group0" $79))) + (let $80 (Member $78 $33)) + (let $81 (CallableType '() $36 $34 $36)) + (let $82 (Udf '"Stat.TDigest_GetPercentile" (Void) (VoidType) '"" $81 (VoidType) '"" $40)) + (return (AsStruct '('Percentile0 (FlatMap $80 (lambda '($83) (Just (Apply $82 $83 (Double '"0.75")))))) '('Percentile1 (FlatMap $80 (lambda '($84) (Just (Apply $82 $84 (Double '"0.9")))))) '('"group0" $79))) ))) '1 '"group0")) - (return (FlatMap (ExtractMembers $42 '('Percentile0 'Percentile1)) (lambda '($86) (block '( - (let $87 (ResourceType '"Yson2.Node")) - (let $88 '($87 '"" '1)) - (let $89 (CallableType '() '((DataType 'Yson)) $88)) - (let $90 (Udf '"Yson2.SerializeText" (Void) (VoidType) '"" $89 (VoidType) '"" $40)) - (let $91 (StructType '('"p75" $37) '('"p90" $37))) - (let $92 (TupleType (TupleType $91) (StructType) (TupleType))) - (let $93 (CallableType '() '($87) '($91))) - (let $94 (Udf '"Yson2.From" (Void) $92 '"" $93 (VoidType) '"" $40)) - (return (Just (AsStruct '('"column0" (Apply $90 (Apply $94 (AsStruct '('"p75" (Member $86 'Percentile0)) '('"p90" (Member $86 'Percentile1))))))))) + (return (FlatMap (ExtractMembers $42 '('Percentile0 'Percentile1)) (lambda '($85) (block '( + (let $86 (ResourceType '"Yson2.Node")) + (let $87 '($86 '"" '1)) + (let $88 (CallableType '() '((DataType 'Yson)) $87)) + (let $89 (Udf '"Yson2.SerializeText" (Void) (VoidType) '"" $88 (VoidType) '"" $40)) + (let $90 (StructType '('"p75" $37) '('"p90" $37))) + (let $91 (TupleType (TupleType $90) (StructType) (TupleType))) + (let $92 (CallableType '() '($86) '($90))) + (let $93 (Udf '"Yson2.From" (Void) $91 '"" $92 (VoidType) '"" $40)) + (return (Just (AsStruct '('"column0" (Apply $89 (Apply $93 (AsStruct '('"p75" (Member $85 'Percentile0)) '('"p90" (Member $85 'Percentile1))))))))) ))))) ))) '('('"_logical_id" '0)) '((DqSink '0 $22 $24))))) $22)) ) diff --git a/ydb/tests/fq/streaming_optimize/suites/watermarks/bad_column.sqlx b/ydb/tests/fq/streaming_optimize/suites/watermarks/bad_column.sqlx deleted file mode 100644 index a8e10f91992d..000000000000 --- a/ydb/tests/fq/streaming_optimize/suites/watermarks/bad_column.sqlx +++ /dev/null @@ -1,13 +0,0 @@ -PRAGMA dq.WatermarksMode="default"; -PRAGMA pq.Consumer="test_client"; - -SELECT - * -FROM pq.test_topic_input -WITH( - FORMAT=json_each_row, - SCHEMA( - ts Timestamp NOT NULL - ), - WATERMARK AS (UNWRAP(timestamp - Interval("PT5S"))) -); diff --git a/ydb/tests/fq/streaming_optimize/suites/watermarks/bad_pushdown.sqlx b/ydb/tests/fq/streaming_optimize/suites/watermarks/bad_pushdown.sqlx deleted file mode 100644 index baa4df32b51b..000000000000 --- a/ydb/tests/fq/streaming_optimize/suites/watermarks/bad_pushdown.sqlx +++ /dev/null @@ -1,13 +0,0 @@ -PRAGMA dq.WatermarksMode="default"; -PRAGMA pq.Consumer="test_client"; - -SELECT - * -FROM pq.test_topic_input -WITH( - FORMAT=json_each_row, - SCHEMA( - ts Timestamp NOT NULL - ), - WATERMARK AS (Unwrap(ts - DateTime::IntervalFromSeconds(5))) -); diff --git a/ydb/tests/fq/streaming_optimize/test_sql_negative.py b/ydb/tests/fq/streaming_optimize/test_sql_negative.py deleted file mode 100644 index e1f2fddd223e..000000000000 --- a/ydb/tests/fq/streaming_optimize/test_sql_negative.py +++ /dev/null @@ -1,60 +0,0 @@ -import os -import pytest -import re -import yatest.common - -from test_utils import pytest_generate_tests_for_run, get_case_file -from ydb.tests.fq.tools.fqrun import FqRun - -NEGATIVE_TEMPLATE = ".sqlx" -DATA_PATH = yatest.common.source_path("ydb/tests/fq/streaming_optimize/suites") - - -@pytest.fixture -def fq_run(request) -> FqRun: - result = FqRun( - config_file=os.path.join("ydb/tests/fq/streaming_optimize/cfg", "app_config.conf"), - path_prefix=f"{request.function.__name__}_" - ) - result.replace_config(lambda config: config.replace("${SOLOMON_ENDPOINT}", os.getenv("SOLOMON_HTTP_ENDPOINT"))) - result.add_topic("test_topic_input", []) - result.add_topic("test_topic_input2", []) - result.add_topic("test_topic_output", []) - result.add_topic("test_topic_output2", []) - return result - - -def sanitize_issues(s): - # 2022-08-13T16:11:21Z -> ISOTIME - # 2022-08-13T16:11:21.549879Z -> ISOTIME - s = re.sub(r"2\d{3}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(.\d+)?Z", "ISOTIME", s) - # (yexception) ... -> - s = re.sub(r"\(yexception\).*", "", s) - # library/cpp/json/json_reader.cpp:420 -> library/cpp/json/json_reader.cpp:xxx - return re.sub(r"cpp:\d+", "cpp:xxx", s) - - -def pytest_generate_tests(metafunc): - pytest_generate_tests_for_run(metafunc, NEGATIVE_TEMPLATE, data_path=DATA_PATH) - - -def test(suite, case, cfg, tmpdir, fq_run): - program_sql = get_case_file(DATA_PATH, suite, case, exts=NEGATIVE_TEMPLATE) - out_dir = tmpdir.mkdir(suite).mkdir(case).dirname - with open(program_sql, encoding="utf-8") as f: - sql_query = f.read() - - fq_run.add_query(sql_query) - result = fq_run.yql_exec(check_error=False, action="explain") - - assert result.execution_result.exit_code != 0, \ - f"execute finished without error, on file: {program_sql}, query:\n{sql_query}" - sql_stderr = result.std_err.strip() - assert sql_stderr, \ - f"exit code is {result.execution_result.exit_code}, but error is empty, on file: {program_sql}, query:\n{sql_query}" - - err_file_path = os.path.join(out_dir, "err_file.out") - with open(err_file_path, "wb") as err_file: - err_file.write(sanitize_issues(sql_stderr.decode()).encode()) - - return [yatest.common.canonical_file(err_file_path)] diff --git a/ydb/tests/fq/streaming_optimize/test_sql_streaming.py b/ydb/tests/fq/streaming_optimize/test_sql_streaming.py index 82fd88d7d0db..b2555eb61c6f 100644 --- a/ydb/tests/fq/streaming_optimize/test_sql_streaming.py +++ b/ydb/tests/fq/streaming_optimize/test_sql_streaming.py @@ -2,7 +2,7 @@ import pytest import yatest.common -from test_utils import pytest_generate_tests_for_run, get_case_file +from test_utils import pytest_generate_tests_for_run from ydb.tests.fq.tools.fqrun import FqRun from yql_utils import yql_binary_path @@ -29,7 +29,7 @@ def pytest_generate_tests(metafunc): def test(suite, case, cfg, fq_run): - program_sql = get_case_file(DATA_PATH, suite, case) + program_sql = os.path.join(DATA_PATH, suite, f"{case}.sql") with open(program_sql, encoding="utf-8") as f: sql_query = f.read() diff --git a/ydb/tests/fq/streaming_optimize/ya.make b/ydb/tests/fq/streaming_optimize/ya.make index 04b92db3c452..d3a60de74ad1 100644 --- a/ydb/tests/fq/streaming_optimize/ya.make +++ b/ydb/tests/fq/streaming_optimize/ya.make @@ -6,7 +6,6 @@ FORK_SUBTESTS() SPLIT_FACTOR(4) TEST_SRCS( - test_sql_negative.py test_sql_streaming.py )