MDEV-34580: Assertion `(key_part->key_part_flag & 4) == 0' failed key_hashnr

Remove an assert added by fix for MDEV-34417. BNL-H join can be used with
prefix keys. This happens when there are real prefix indexes on the
equi-join columns (although it probably doesn't make a lot of sense).

Anyway, remove the assert. The code receives properly truncated key values
for hashing/comparison so it can handle them just fine.

Author: spetrunia

mysql-test/main/join_cache.result

@@ -6420,3 +6420,23 @@ DROP TABLE t1,t2,t3;
 #
 # End of 10.4 tests
 #
+#
+# MDEV-34580: Assertion `(key_part->key_part_flag & 4) == 0' failed key_hashnr
+#
+SET join_cache_level=3;
+CREATE TABLE t1 ( a TIMESTAMP , b varchar(100), c varchar(10) ) ;
+INSERT INTO t1 (b,c) VALUES ('GHOBS','EMLCG'),('t','p');
+CREATE TABLE t2 (a varchar(100), b varchar(100), c varchar(10) , KEY b (b(66))) ;
+insert into t2 select seq, seq, seq from seq_1_to_20;
+explain
+SELECT t1.a FROM t1  JOIN t2 ON t1.b = t2.b ;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	2	Using where
+1	SIMPLE	t2	hash_ALL	b	#hash#b	69	test.t1.b	20	Using where; Using join buffer (flat, BNLH join)
+SELECT t1.a FROM t1  JOIN t2 ON t1.b = t2.b ;
+a
+set join_cache_level=default;
+DROP TABLE t1, t2;
+#
+# End of 10.5 tests
+#

mysql-test/main/join_cache.test

@@ -4300,3 +4300,25 @@ DROP TABLE t1,t2,t3;
 --echo #
 --echo # End of 10.4 tests
 --echo #
+
+--echo #
+--echo # MDEV-34580: Assertion `(key_part->key_part_flag & 4) == 0' failed key_hashnr
+--echo #
+--source include/have_sequence.inc
+SET join_cache_level=3;
+
+CREATE TABLE t1 ( a TIMESTAMP , b varchar(100), c varchar(10) ) ;
+INSERT INTO t1 (b,c) VALUES ('GHOBS','EMLCG'),('t','p');
+
+CREATE TABLE t2 (a varchar(100), b varchar(100), c varchar(10) , KEY b (b(66))) ;
+insert into t2 select seq, seq, seq from seq_1_to_20;
+
+explain
+SELECT t1.a FROM t1  JOIN t2 ON t1.b = t2.b ;
+SELECT t1.a FROM t1  JOIN t2 ON t1.b = t2.b ;
+
+set join_cache_level=default;
+DROP TABLE t1, t2;
+--echo #
+--echo # End of 10.5 tests
+--echo #

sql/key.cc

@@ -755,10 +755,12 @@ ulong key_hashnr(KEY *key_info, uint used_key_parts, const uchar *key)
     if (is_string)
     {
       /*
-        Prefix keys are not possible in BNLH joins.
-        Use the whole string to calculate the hash.
+        Surprisingly, BNL-H joins may use prefix keys. This may happen
+        when there is a real index on the column used in equi-join.
+
+        In this case, the passed key tuple is already a prefix, no
+        special handling is required.
       */
-      DBUG_ASSERT((key_part->key_part_flag & HA_PART_KEY_SEG) == 0);
       cs->hash_sort(pos+pack_length, length, &nr, &nr2);
       key+= pack_length;
     }
@@ -862,10 +864,10 @@ bool key_buf_cmp(KEY *key_info, uint used_key_parts,
     if (is_string)
     {
       /*
-        Prefix keys are not possible in BNLH joins.
-        Compare whole strings.
+        Surprisingly, BNL-H joins may use prefix keys. This may happen
+        when there is a real index on the column used in equi-join.
+        In this case, we get properly truncated prefixes here.
       */
-      DBUG_ASSERT((key_part->key_part_flag & HA_PART_KEY_SEG) == 0);
       if (cs->strnncollsp(pos1 + pack_length, length1,
                           pos2 + pack_length, length2))
         return true;