implement preserve order for pagination

src/share/parameter/ob_parameter_seed.ipp

@@ -1680,3 +1680,7 @@ DEF_STR_WITH_CHECKER(sql_protocol_min_tls_version, OB_CLUSTER_PARAMETER, "none",
 DEF_MODE_WITH_PARSER(_obkv_feature_mode, OB_CLUSTER_PARAMETER, "", common::ObKvFeatureModeParser,
     "_obkv_feature_mode is a option list to control specified OBKV features on/off.",
     ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
+
+DEF_BOOL(_preserve_order_for_pagination, OB_TENANT_PARAMETER, "False",
+        "enable preserver order for limit",
+        ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));

src/sql/resolver/dml/ob_hint.cpp

@@ -737,6 +737,11 @@ bool ObOptParamHint::is_param_val_valid(const OptParamType param_type, const ObO
                                       || 0 == val.get_varchar().case_compare("false"));
       break;
     }
+    case PRESERVE_ORDER_FOR_PAGINATION: {
+      is_valid = val.is_varchar() && (0 == val.get_varchar().case_compare("true")
+                                      || 0 == val.get_varchar().case_compare("false"));
+      break;
+    }
     default:
       LOG_TRACE("invalid opt param val", K(param_type), K(val));
       break;

src/sql/resolver/dml/ob_hint.h

@@ -92,6 +92,7 @@ struct ObOptParamHint
     DEF(USE_DEFAULT_OPT_STAT,)            \
     DEF(ENABLE_IN_RANGE_OPTIMIZATION,)    \
     DEF(XSOLAPI_GENERATE_WITH_CLAUSE,)   \
+    DEF(PRESERVE_ORDER_FOR_PAGINATION,)   \
 
   DECLARE_ENUM(OptParamType, opt_param, OPT_PARAM_TYPE_DEF, static);
 

src/sql/rewrite/ob_transform_pre_process.cpp

@@ -60,6 +60,7 @@ int ObTransformPreProcess::transform_one_stmt(common::ObIArray<ObParentDMLStmt>
   int ret = OB_SUCCESS;
   trans_happened = false;
   bool is_happened = false;
+  ObDMLStmt *limit_stmt = NULL;
   if (OB_ISNULL(stmt)) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("stmt is NULL", K(ret));
@@ -237,14 +238,25 @@ int ObTransformPreProcess::transform_one_stmt(common::ObIArray<ObParentDMLStmt>
     }
 
     if (OB_SUCC(ret)) {
-      if (OB_FAIL(transform_rownum_as_limit_offset(parent_stmts, stmt, is_happened))) {
+      if (OB_FAIL(transform_rownum_as_limit_offset(parent_stmts, stmt, limit_stmt, is_happened))) {
         LOG_WARN("failed to transform rownum as limit", K(ret));
       } else {
         trans_happened |= is_happened;
         OPT_TRACE("transform rownum as limit:", is_happened);
         LOG_TRACE("succeed to transform rownum as limit", K(is_happened));
       }
     }
+
+    if (OB_SUCC(ret)) {
+      if (OB_FAIL(preserve_order_for_pagination(NULL == limit_stmt ? stmt : limit_stmt, is_happened))) {
+        LOG_WARN("failed to preserve order for pagination", K(ret));
+      } else {
+        trans_happened |= is_happened;
+        OPT_TRACE("preserve order for pagination:", is_happened);
+        LOG_TRACE("succeed to preserve order for pagination", K(is_happened));
+      }
+    }
+
     if (OB_SUCC(ret)) {
       if (OB_FAIL(transform_groupingsets_rollup_cube(stmt, is_happened))) {
         LOG_WARN("failed to transform for transform for grouping sets, rollup and cube.", K(ret));
@@ -6033,14 +6045,15 @@ int ObTransformPreProcess::transformer_aggr_expr(ObDMLStmt *stmt,
 int ObTransformPreProcess::transform_rownum_as_limit_offset(
                                             const ObIArray<ObParentDMLStmt> &parent_stmts,
                                             ObDMLStmt *&stmt,
+                                            ObDMLStmt *&limit_stmt,
                                             bool &trans_happened)
 {
   int ret = OB_SUCCESS;
   //bool is_rownum_gen_col_happened = false;
   bool is_rownum_happened = false;
   bool is_generated_rownum_happened = false;
   trans_happened = false;
-  if (OB_FAIL(transform_common_rownum_as_limit(stmt, is_rownum_happened))) {
+  if (OB_FAIL(transform_common_rownum_as_limit(stmt, limit_stmt, is_rownum_happened))) {
     LOG_WARN("failed to transform common rownum as limit", K(ret));
   } else if (OB_FAIL(transform_generated_rownum_as_limit(parent_stmts, stmt,
                                                          is_generated_rownum_happened))) {
@@ -6058,7 +6071,9 @@ int ObTransformPreProcess::transform_rownum_as_limit_offset(
  * => select * from (select * from t where ... limit ?) order by c1;
  *
 **/
-int ObTransformPreProcess::transform_common_rownum_as_limit(ObDMLStmt *&stmt, bool &trans_happened)
+int ObTransformPreProcess::transform_common_rownum_as_limit(ObDMLStmt *&stmt,
+                                                            ObDMLStmt *&limit_stmt,
+                                                            bool &trans_happened)
 {
   int ret = OB_SUCCESS;
   trans_happened = false;
@@ -6091,6 +6106,7 @@ int ObTransformPreProcess::transform_common_rownum_as_limit(ObDMLStmt *&stmt, bo
     LOG_WARN("get unexpected null", K(ret), K(child_stmt));
   } else {
     child_stmt->set_limit_offset(limit_expr, NULL);
+    limit_stmt = child_stmt;
   }
   return ret;
 }
@@ -10032,5 +10048,66 @@ int ObTransformPreProcess::check_is_correlated_cte(ObSelectStmt *stmt, ObIArray<
   return ret;
 }
 
+int ObTransformPreProcess::preserve_order_for_pagination(ObDMLStmt *stmt,
+                                                    bool &trans_happened)
+{
+  int ret = OB_SUCCESS;
+  trans_happened = false;
+  bool is_valid = false;
+  bool is_hint_enabled = false;
+  bool has_hint = false;
+  ObSelectStmt *sel_stmt = NULL;
+  ObSEArray<ObRawExpr*, 2> select_exprs;
+  ObSEArray<ObRawExpr*, 2> order_by_exprs;
+  ObSEArray<ObRawExpr*, 2> new_order_by_exprs;
+  if (OB_ISNULL(stmt) || OB_ISNULL(ctx_) ||
+      OB_ISNULL(ctx_->session_info_)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("unexpect null", K(ret), K(stmt), K(ctx_));
+  } else if (OB_FAIL(stmt->get_query_ctx()->get_global_hint().opt_params_.get_bool_opt_param(
+              ObOptParamHint::PRESERVE_ORDER_FOR_PAGINATION, is_hint_enabled, has_hint))) {
+    LOG_WARN("failed to check has opt param", K(ret));
+  } else if (has_hint && !is_hint_enabled) {
+    OPT_TRACE("query hint disable preserve order for pagination");
+  } else if (OB_FAIL(ctx_->session_info_->is_preserve_order_for_pagination_enabled(is_valid))) {
+    LOG_WARN("failed to check preserve order for pagination enabled", K(ret));
+  } else if (!is_valid && !has_hint) {
+    OPT_TRACE("system config disable preserve order for pagination");
+  } else if (!stmt->is_select_stmt()) {
+    OPT_TRACE("dml query can not preserve order for pagination");
+  } else if (OB_FALSE_IT(sel_stmt=static_cast<ObSelectStmt*>(stmt))) {
+  } else if (!sel_stmt->has_limit() || NULL != sel_stmt->get_limit_percent_expr()) {
+    OPT_TRACE("query do not have normal limit offset");
+  } else if (OB_FAIL(sel_stmt->get_order_exprs(order_by_exprs))) {
+    LOG_WARN("failed to get order exprs", K(ret));
+  } else if (OB_FAIL(ObTransformUtils::check_stmt_unique(sel_stmt,
+                                                         ctx_->session_info_,
+                                                         ctx_->schema_checker_,
+                                                         order_by_exprs,
+                                                         true,
+                                                         is_valid))) {
+    LOG_WARN("failed to check stmt unique on exprs", K(ret));
+  } else if (is_valid) {
+    OPT_TRACE("current order by exprs is unique, do not need add extra order by exprs");
+  } else if OB_FAIL(sel_stmt->get_select_exprs_without_lob(select_exprs)) {
+    LOG_WARN("failed to get select exprs", K(ret));
+  } else if (OB_FAIL(ObOptimizerUtil::except_exprs(select_exprs,
+                                                   order_by_exprs,
+                                                   new_order_by_exprs))) {
+    LOG_WARN("failed to except exprs", K(ret));
+  } else {
+    for (int64_t i = 0; OB_SUCC(ret) && i < new_order_by_exprs.count(); ++i) {
+      OrderItem item(new_order_by_exprs.at(i));
+      if (OB_FAIL(sel_stmt->add_order_item(item))) {
+        LOG_WARN("failed to add order item", K(ret));
+      }
+    }
+    if (OB_SUCC(ret) && !new_order_by_exprs.empty()) {
+      trans_happened = true;
+    }
+  }
+  return ret;
+}
+
 } // end namespace sql
 } // end namespace oceanbase

src/sql/rewrite/ob_transform_pre_process.h

@@ -440,8 +440,9 @@ struct DistinctObjMeta
   int transformer_aggr_expr(ObDMLStmt *stmt, bool &trans_happened);
   int transform_rownum_as_limit_offset(const ObIArray<ObParentDMLStmt> &parent_stmts,
                                        ObDMLStmt *&stmt,
+                                       ObDMLStmt *&limit_stmt,
                                        bool &trans_happened);
-  int transform_common_rownum_as_limit(ObDMLStmt *&stmt, bool &trans_happened);
+  int transform_common_rownum_as_limit(ObDMLStmt *&stmt, ObDMLStmt *&limit_stmt, bool &trans_happened);
   int try_transform_common_rownum_as_limit_or_false(ObDMLStmt *stmt, ObRawExpr *&limit_expr, bool& is_valid);
   int transform_generated_rownum_as_limit(const ObIArray<ObParentDMLStmt> &parent_stmts,
                                           ObDMLStmt *stmt,
@@ -627,6 +628,8 @@ struct DistinctObjMeta
   int check_exec_param_correlated(const ObRawExpr *expr, bool &is_correlated);
   int check_is_correlated_cte(ObSelectStmt *stmt, ObIArray<ObSelectStmt *> &visited_cte, bool &is_correlated);
   int convert_join_preds_vector_to_scalar(JoinedTable &joined_table, bool &trans_happened);
+  int preserve_order_for_pagination(ObDMLStmt *stmt,
+                               bool &trans_happened);
 private:
   DISALLOW_COPY_AND_ASSIGN(ObTransformPreProcess);
 };

src/sql/session/ob_sql_session_info.cpp

@@ -501,6 +501,18 @@ int ObSQLSessionInfo::is_better_inlist_enabled(bool &enabled) const
   return ret;
 }
 
+int ObSQLSessionInfo::is_preserve_order_for_pagination_enabled(bool &enabled) const
+{
+  int ret = OB_SUCCESS;
+  enabled = false;
+  int64_t tenant_id = get_effective_tenant_id();
+  omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id));
+  if (tenant_config.is_valid()) {
+    enabled = tenant_config->_preserve_order_for_pagination;
+  }
+  return ret;
+}
+
 bool ObSQLSessionInfo::is_index_skip_scan_enabled() const
 {
   bool bret = false;

src/sql/session/ob_sql_session_info.h

@@ -1151,6 +1151,7 @@ class ObSQLSessionInfo: public common::ObVersionProvider, public ObBasicSessionI
   int is_better_inlist_enabled(bool &enabled) const;
   bool is_index_skip_scan_enabled() const;
   bool is_var_assign_use_das_enabled() const;
+  int is_preserve_order_for_pagination_enabled(bool &enabled) const;
 
   ObSessionDDLInfo &get_ddl_info() { return ddl_info_; }
   void set_ddl_info(const ObSessionDDLInfo &ddl_info) { ddl_info_ = ddl_info; }

tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result

@@ -349,6 +349,7 @@ _parallel_max_active_sessions
 _parallel_min_message_pool
 _parallel_server_sleep_time
 _pipelined_table_function_memory_limit
+_preserve_order_for_pagination
 _print_sample_ppm
 _private_buffer_size
 _publish_schema_mode