MDEV-21856 - xid_t::formatID has to be constrained to 4 byte size

Engine (InnoDB) and XA replication MDEV-742 requires the XID member
be of a constant minimum across supported platform ulong size which
is 4 bytes.

Author: svoj

mysql-test/main/xa.result

@@ -369,3 +369,8 @@ XA PREPARE 'Я_упaлa_c_сеновала_тормозила_головой';
 XA ROLLBACK 'Я_упaлa_c_сеновала_тормозила_головой';
 SET NAMES default;
 DROP TABLE t1;
+#
+# MDEV-21856 - xid_t::formatID has to be constrained to 4 byte size
+#
+XA START 'gtrid', 'bqual', 0x80000000;
+ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '0x80000000' at line 1

mysql-test/main/xa.test

@@ -506,4 +506,11 @@ SET NAMES default;
 
 DROP TABLE t1;
 
+
+--echo #
+--echo # MDEV-21856 - xid_t::formatID has to be constrained to 4 byte size
+--echo #
+--error ER_PARSE_ERROR
+XA START 'gtrid', 'bqual', 0x80000000;
+
 --source include/wait_until_count_sessions.inc

sql/sql_yacc.yy

@@ -17553,7 +17553,9 @@ xid:
           }
           | text_string ',' text_string ',' ulong_num
           {
-            MYSQL_YYABORT_UNLESS($1->length() <= MAXGTRIDSIZE && $3->length() <= MAXBQUALSIZE);
+            MYSQL_YYABORT_UNLESS($1->length() <= MAXGTRIDSIZE &&
+                                 $3->length() <= MAXBQUALSIZE &&
+                                 $5 <= INT32_MAX);
             if (unlikely(!(Lex->xid=(XID *)thd->alloc(sizeof(XID)))))
               MYSQL_YYABORT;
             Lex->xid->set($5, $1->ptr(), $1->length(), $3->ptr(), $3->length());