MDEV-35072: Assertion with optimizer_join_limit_pref_ratio and 1-table select

Pre-11.0 variant:

1. In recompute_join_cost_with_limit(), add an assertion that that
partial_join_cost >= 0.0.

2. best_extension_by_limited_search() subtracts COST_EPS from
   join->best_read. But it is not subtracted from
   join->positions[0].read_time, add it back.

2. We could get very small negative partial_join_cost due to rounding
  errors. For fraction=1.0, we were computing essentially this (denote
  as EXPR-1):

  $row_read_cost + $where_cost - ($row_read_cost + $where_cost)

which should compute to 0.
But the computation was done in the following order (left-to-right):
EXPR-2:

  ($row_read_cost + $where_cost) - $row_read_cost - $where_cost

this produced a value of -1.1102230246251565e-16 due to a rounding
error. Change the computation use EXPR-1 instead of EXPR-2.

Author: spetrunia

mysql-test/main/order_by_limit_join.result

@@ -111,7 +111,7 @@ JS
         "full_join_cost": "REPLACED",
         "risk_ratio": 10,
         "shortcut_join_cost": "REPLACED",
-        "shortcut_cost_with_risk": 972.8224614,
+        "shortcut_cost_with_risk": 972.8224714,
         "use_shortcut_cost": true
     }
 ]
@@ -163,7 +163,7 @@ JS
         "full_join_cost": "REPLACED",
         "risk_ratio": 10,
         "shortcut_join_cost": "REPLACED",
-        "shortcut_cost_with_risk": 20972.81246,
+        "shortcut_cost_with_risk": 20972.81247,
         "use_shortcut_cost": true
     }
 ]
@@ -247,7 +247,7 @@ JS
         "full_join_cost": "REPLACED",
         "risk_ratio": 10,
         "shortcut_join_cost": "REPLACED",
-        "shortcut_cost_with_risk": 240591.2698,
+        "shortcut_cost_with_risk": 240591.2748,
         "use_shortcut_cost": false
     }
 ]
@@ -366,7 +366,7 @@ JS
         "full_join_cost": "REPLACED",
         "risk_ratio": 10,
         "shortcut_join_cost": "REPLACED",
-        "shortcut_cost_with_risk": 982.9697856,
+        "shortcut_cost_with_risk": 982.9697956,
         "use_shortcut_cost": true
     }
 ]
@@ -451,11 +451,21 @@ JS
         "full_join_cost": "REPLACED",
         "risk_ratio": 10,
         "shortcut_join_cost": "REPLACED",
-        "shortcut_cost_with_risk": 972.8224614,
+        "shortcut_cost_with_risk": 972.8224714,
         "use_shortcut_cost": true
     }
 ]
 set optimizer_search_depth=@tmp_osd;
 set optimizer_trace=@tmp_os;
-set optimizer_join_limit_pref_ratio=default;
 drop table t1, t10, t11;
+#
+# MDEV-35072: Assertion failure with optimizer_join_limit_pref_ratio and 1-table select
+#
+SET optimizer_join_limit_pref_ratio=1;
+CREATE TABLE t1 (c1 INT, INDEX(c1));
+INSERT INTO t1  VALUES (1),(2);
+SELECT * FROM t1  ORDER BY c1 LIMIT 1;
+c1
+1
+DROP TABLE t1;
+set optimizer_join_limit_pref_ratio=default;

mysql-test/main/order_by_limit_join.test

@@ -208,6 +208,15 @@ select json_detailed(json_extract(@trace, '$**.join_limit_shortcut_choice')) as
 
 set optimizer_search_depth=@tmp_osd;
 set optimizer_trace=@tmp_os;
-set optimizer_join_limit_pref_ratio=default;
 drop table t1, t10, t11;
 
+--echo #
+--echo # MDEV-35072: Assertion failure with optimizer_join_limit_pref_ratio and 1-table select
+--echo #
+SET optimizer_join_limit_pref_ratio=1;
+CREATE TABLE t1 (c1 INT, INDEX(c1));
+INSERT INTO t1  VALUES (1),(2);
+SELECT * FROM t1  ORDER BY c1 LIMIT 1;
+DROP TABLE t1;
+
+set optimizer_join_limit_pref_ratio=default;

sql/sql_select.cc

@@ -10616,8 +10616,11 @@ double recompute_join_cost_with_limit(const JOIN *join, bool skip_sorting,
   POSITION *pos= join->best_positions + join->const_tables;
   /*
     Generally, we assume that producing X% of output takes X% of the cost.
+
+    best_extension_by_limited_search() subtracts COST_EPS from
+    join->best_read, add it back.
   */
-  double partial_join_cost= join->best_read * fraction;
+  double partial_join_cost= (join->best_read + COST_EPS) * fraction;
 
   if (skip_sorting)
   {
@@ -10634,12 +10637,20 @@ double recompute_join_cost_with_limit(const JOIN *join, bool skip_sorting,
     */
     if (first_table_cost)
     {
+      /*
+        Compute the cost of accessing the first table in the same way as
+        it was done in greedy_search():
+      */
+      double prev_first_table_cost;
+      prev_first_table_cost= pos->read_time +
+                             (double)pos->records_read / TIME_FOR_COMPARE;
+      prev_first_table_cost *= fraction;
       /*
         Subtract the remainder of the first table's cost we had in
         join->best_read:
       */
-      partial_join_cost -= pos->read_time*fraction;
-      partial_join_cost -= pos->records_read*fraction / TIME_FOR_COMPARE;
+      partial_join_cost -= prev_first_table_cost;
+      DBUG_ASSERT(partial_join_cost >= 0.0);
 
       /* Add the cost of the new access method we've got: */
       partial_join_cost= COST_ADD(partial_join_cost, *first_table_cost);
@@ -11093,6 +11104,10 @@ best_extension_by_limited_search(JOIN      *join,
           memcpy((uchar*) join->best_positions, (uchar*) join->positions,
                  sizeof(POSITION) * (idx + 1));
           join->join_record_count= partial_join_cardinality;
+          /*
+            note: recompute_join_cost_with_limit() relies on this COST_EPS
+            subtraction:
+          */
           join->best_read= current_read_time - COST_EPS;
         }
         DBUG_EXECUTE("opt", print_plan(join, idx+1,