MDEV-37404 InnoDB: Failing assertion: node->pcur->rel_pos == BTR_PCUR_ON

Caused by optimization done in 2e2b2a0.

Cannot use lookup_handler in default branch of locate_dup_record() as
InnoDB update depends on positioned record and update is done in table
main handler.

The patch reverts some non-pure changes done by 2e2b2a0 to
original logic from 72429ca. There was no long_unique_table condition
to init search on table->file, so we get into default branch with long
unique and table->file search uninitialized.

ha_rnd_init_with_error() on demand for HA_DUPLICATE_POS branch was
original logic as well.

More info: 2e2b2a0 reverts 5e34528, but it seems to be OK as
MDEV-3888 test case passes. mysql-5.6.13 has the original code with
HA_WHOLE_KEY as well.

Author: midenok

mysql-test/main/long_unique_bugs.result

@@ -843,3 +843,11 @@ insert into t (a) values (1);
 update t set a=2;
 drop table t;
 # End of 10.6 tests
+#
+# MDEV-37404 InnoDB: Failing assertion: node->pcur->rel_pos == BTR_PCUR_ON
+#
+create table t(c int primary key, c2 blob unique) engine=innodb;
+insert into t values (0, '');
+replace into t values (0, '123');
+drop table t;
+# End of 10.11 tests

mysql-test/main/long_unique_bugs.test

@@ -767,3 +767,13 @@ update t set a=2;
 drop table t;
 
 --echo # End of 10.6 tests
+
+--echo #
+--echo # MDEV-37404 InnoDB: Failing assertion: node->pcur->rel_pos == BTR_PCUR_ON
+--echo #
+create table t(c int primary key, c2 blob unique) engine=innodb;
+insert into t values (0, '');
+replace into t values (0, '123');
+drop table t;
+
+--echo # End of 10.11 tests

sql/sql_insert.cc

@@ -702,8 +702,7 @@ int prepare_for_replace(TABLE *table, enum_duplicates handle_duplicates,
   {
     create_lookup_handler= true;
     table->file->extra(HA_EXTRA_IGNORE_DUP_KEY);
-    if (table->file->ha_table_flags() & HA_DUPLICATE_POS ||
-        table->s->long_unique_table)
+    if (table->file->ha_table_flags() & HA_DUPLICATE_POS)
     {
       if (table->file->ha_rnd_init_with_error(false))
         return 1;
@@ -2018,28 +2017,34 @@ int vers_insert_history_row(TABLE *table)
  */
 int Write_record::locate_dup_record()
 {
-  handler *h= table->file;
   int error= 0;
-  if (h->ha_table_flags() & HA_DUPLICATE_POS || h->lookup_errkey != (uint)-1)
+  if (table->file->ha_table_flags() & HA_DUPLICATE_POS ||
+      table->file->lookup_errkey != (uint)-1)
   {
     DBUG_PRINT("info", ("Locating offending record using rnd_pos()"));
 
-    error= h->ha_rnd_pos(table->record[1], h->dup_ref);
+    const bool init_lookup_handler= (table->file->inited == handler::NONE);
+    if (init_lookup_handler)
+    {
+      error= table->file->ha_rnd_init_with_error(false);
+      if (error)
+        return error;
+    }
+    error= table->file->ha_rnd_pos(table->record[1], table->file->dup_ref);
+    if (init_lookup_handler)
+      table->file->ha_rnd_end();
     if (unlikely(error))
     {
       DBUG_PRINT("info", ("rnd_pos() returns error %d",error));
-      h->print_error(error, MYF(0));
+      table->file->print_error(error, MYF(0));
     }
   }
   else
   {
     DBUG_PRINT("info",
                ("Locating offending record using ha_index_read_idx_map"));
 
-    if (h->lookup_handler)
-      h= h->lookup_handler;
-
-    error= h->extra(HA_EXTRA_FLUSH_CACHE);
+    error= table->file->extra(HA_EXTRA_FLUSH_CACHE);
     if (unlikely(error))
     {
       DBUG_PRINT("info",("Error when setting HA_EXTRA_FLUSH_CACHE"));
@@ -2058,12 +2063,12 @@ int Write_record::locate_dup_record()
 
     key_copy(key, table->record[0], table->key_info + key_nr, 0);
 
-    error= h->ha_index_read_idx_map(table->record[1], key_nr, key,
+    error= table->file->ha_index_read_idx_map(table->record[1], key_nr, key,
                                     HA_WHOLE_KEY, HA_READ_KEY_EXACT);
     if (unlikely(error))
     {
       DBUG_PRINT("info", ("index_read_idx() returns %d", error));
-      h->print_error(error, MYF(0));
+      table->file->print_error(error, MYF(0));
     }
   }