Reducing memory when using information schema

The background is that one user had a lot of views and using some complex
queries on information schema temporary memory of more than 2G was used.

- Added new element 'total_alloc' to MEM_ROOT for easier debugging.
- Added MAX_MEMORY_USED to information_schema.processlist.
- Added new status variable "Memory_used_initial" that shows how much MariaDB
  uses at startup. This gives the base value for "Memory_used".
- Reuse memory continuously for information schema queries instead of
  only freeing memory at query end.

Other things
- Removed some not needed set_notnull() calls for not null columns.

include/my_alloc.h

@@ -43,6 +43,7 @@ typedef struct st_mem_root
   /* if block have less memory it will be put in 'used' list */
   size_t min_malloc;
   size_t block_size;               /* initial block size */
+  size_t total_alloc;
   unsigned int block_num;          /* allocated blocks counter */
   /* 
      first free block in queue test counter (if it exceed 

include/mysql.h.pp

@@ -236,6 +236,7 @@
   USED_MEM *pre_alloc;
   size_t min_malloc;
   size_t block_size;
+  size_t total_alloc;
   unsigned int block_num;
   unsigned int first_block_usage;
   void (*error_handler)(void);

mysql-test/r/create.result

@@ -1173,6 +1173,7 @@ t1	CREATE TABLE `t1` (
   `MAX_STAGE` tinyint(2) NOT NULL DEFAULT 0,
   `PROGRESS` decimal(7,3) NOT NULL DEFAULT 0.000,
   `MEMORY_USED` bigint(7) NOT NULL DEFAULT 0,
+  `MAX_MEMORY_USED` bigint(7) NOT NULL DEFAULT 0,
   `EXAMINED_ROWS` int(7) NOT NULL DEFAULT 0,
   `QUERY_ID` bigint(4) NOT NULL DEFAULT 0,
   `INFO_BINARY` blob DEFAULT NULL,
@@ -1196,6 +1197,7 @@ t1	CREATE TEMPORARY TABLE `t1` (
   `MAX_STAGE` tinyint(2) NOT NULL DEFAULT 0,
   `PROGRESS` decimal(7,3) NOT NULL DEFAULT 0.000,
   `MEMORY_USED` bigint(7) NOT NULL DEFAULT 0,
+  `MAX_MEMORY_USED` bigint(7) NOT NULL DEFAULT 0,
   `EXAMINED_ROWS` int(7) NOT NULL DEFAULT 0,
   `QUERY_ID` bigint(4) NOT NULL DEFAULT 0,
   `INFO_BINARY` blob DEFAULT NULL,

mysql-test/suite/funcs_1/datadict/processlist_priv.inc

@@ -158,7 +158,7 @@ WHERE DB = 'information_schema' AND COMMAND = 'Sleep' AND USER = 'ddicttestuser1
 eval SHOW CREATE TABLE $table;
 --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS
 eval SHOW $table;
---replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID
+--replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 MAX_MEMORY 15 ROWS 16 QUERY_ID 18 TID
 eval SELECT * FROM $table $select_where ORDER BY id;
 --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID
 eval SELECT $columns FROM $table $select_where ORDER BY id;
@@ -178,7 +178,7 @@ connection con100;
 eval SHOW CREATE TABLE $table;
 --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS
 eval SHOW $table;
---replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID
+--replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 MAX_MEMORY 15 ROWS 16 QUERY_ID 18 TID
 eval SELECT * FROM $table $select_where ORDER BY id;
 --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID
 eval SELECT $columns FROM $table $select_where ORDER BY id;
@@ -204,7 +204,7 @@ connection con100;
 SHOW GRANTS;
 --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS
 SHOW processlist;
---replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID
+--replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 MAX_MEMORY 15 ROWS 16 QUERY_ID 18 TID
 SELECT * FROM information_schema.processlist;
 --real_sleep 0.3
 
@@ -216,7 +216,7 @@ connect (con101,localhost,ddicttestuser1,ddictpass,information_schema);
 SHOW GRANTS;
 --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS
 SHOW processlist;
---replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID
+--replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 MAX_MEMORY 15 ROWS 16 QUERY_ID 18 TID
 SELECT * FROM information_schema.processlist;
 --real_sleep 0.3
 
@@ -239,7 +239,7 @@ connect (anonymous1,localhost,"''",,information_schema);
 SHOW GRANTS;
 --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS
 SHOW processlist;
---replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID
+--replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 MAX_MEMORY 15 ROWS 16 QUERY_ID 18 TID
 SELECT * FROM information_schema.processlist;
 --real_sleep 0.3
 
@@ -261,7 +261,7 @@ connect (con102,localhost,ddicttestuser1,ddictpass,information_schema);
 SHOW GRANTS;
 --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS
 SHOW processlist;
---replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID
+--replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 MAX_MEMORY 15 ROWS 16 QUERY_ID 18 TID
 SELECT * FROM information_schema.processlist;
 --real_sleep 0.3
 
@@ -287,7 +287,7 @@ if ($fixed_bug_30395)
 --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS
 SHOW processlist;
 }
---replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID
+--replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 MAX_MEMORY 15 ROWS 16 QUERY_ID 18 TID
 SELECT * FROM information_schema.processlist;
 --real_sleep 0.3
 
@@ -308,7 +308,7 @@ connect (con103,localhost,ddicttestuser1,ddictpass,information_schema);
 SHOW GRANTS FOR 'ddicttestuser1'@'localhost';
 --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS
 SHOW processlist;
---replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID
+--replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 MAX_MEMORY 15 ROWS 16 QUERY_ID 18 TID
 SELECT * FROM information_schema.processlist;
 --real_sleep 0.3
 
@@ -330,7 +330,7 @@ connect (con104,localhost,ddicttestuser1,ddictpass,information_schema);
 SHOW GRANTS FOR 'ddicttestuser1'@'localhost';
 --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS
 SHOW processlist;
---replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID
+--replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 MAX_MEMORY 15 ROWS 16 QUERY_ID 18 TID
 SELECT * FROM information_schema.processlist;
 --real_sleep 0.3
 
@@ -377,7 +377,7 @@ connect (con200,localhost,ddicttestuser2,ddictpass,information_schema);
 SHOW GRANTS FOR 'ddicttestuser2'@'localhost';
 --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS
 SHOW processlist;
---replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID
+--replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 MAX_MEMORY 15 ROWS 16 QUERY_ID 18 TID
 SELECT * FROM information_schema.processlist;
 --real_sleep 0.3
 
@@ -398,7 +398,7 @@ connect (con201,localhost,ddicttestuser2,ddictpass,information_schema);
 SHOW GRANTS;
 --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS
 SHOW processlist;
---replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID
+--replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 MAX_MEMORY 15 ROWS 16 QUERY_ID 18 TID
 SELECT * FROM information_schema.processlist;
 --real_sleep 0.3
 
@@ -421,7 +421,7 @@ SHOW GRANTS FOR 'ddicttestuser1'@'localhost';
 GRANT PROCESS ON *.* TO 'ddicttestuser2'@'localhost';
 --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS
 SHOW processlist;
---replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID
+--replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 MAX_MEMORY 15 ROWS 16 QUERY_ID 18 TID
 SELECT * FROM information_schema.processlist;
 --real_sleep 0.3
 
@@ -445,7 +445,7 @@ connect (con108,localhost,ddicttestuser1,ddictpass,information_schema);
 SHOW GRANTS FOR 'ddicttestuser1'@'localhost';
 --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS
 SHOW processlist;
---replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID
+--replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 MAX_MEMORY 15 ROWS 16 QUERY_ID 18 TID
 SELECT * FROM information_schema.processlist;
 --real_sleep 0.3
 

mysql-test/suite/funcs_1/datadict/processlist_val.inc

@@ -92,7 +92,7 @@ echo
 #   - INFO must contain the corresponding SHOW/SELECT PROCESSLIST
 #
 # 1. Just dump what we get
---replace_column 1 <ID> 3 <HOST_NAME> 6 <TIME> 9 <TIME_MS> 13 <MEMORY> 14 <ROWS> 15 <QUERY_ID> 17 <TID>
+--replace_column 1 <ID> 3 <HOST_NAME> 6 <TIME> 9 <TIME_MS> 13 <MEMORY> 14 <MAX_MEMORY> 15 <ROWS> 16 <QUERY_ID> 18 <TID>
 SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST;
 --replace_column 1 <ID> 3 <HOST_NAME> 6 <TIME> 9 <TIME_MS> 13 <MEMORY> 14 <ROWS>
 SHOW FULL PROCESSLIST;
@@ -159,7 +159,7 @@ let $wait_condition= SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST
                      WHERE COMMAND = 'Sleep' AND USER = 'test_user';
 --source include/wait_condition.inc
 # 1. Just dump what we get
---replace_column 1 <ID> 3 <HOST_NAME> 6 <TIME> 7 <STATE> 9 <TIME_MS> 13 <MEMORY> 14 <ROWS> 15 <QUERY_ID> 17 <TID>
+--replace_column 1 <ID> 3 <HOST_NAME> 6 <TIME> 7 <STATE> 9 <TIME_MS> 13 <MEMORY> 14 <MAX_MEMORY> 15 <ROWS> 16 <QUERY_ID> 18 <TID>
 SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST;
 --replace_column 1 <ID> 3 <HOST_NAME> 6 <TIME>
 SHOW FULL PROCESSLIST;
@@ -201,7 +201,7 @@ echo
 #----------------------------------------------------------------------------
 ;
 connection con1;
---replace_column 1 <ID> 3 <HOST_NAME> 6 <TIME> 9 <TIME_MS> 13 <MEMORY> 14 <ROWS> 15 <QUERY_ID> 17 <TID>
+--replace_column 1 <ID> 3 <HOST_NAME> 6 <TIME> 9 <TIME_MS> 13 <MEMORY> 14 <MAX_MEMORY> 15 <ROWS> 16 <QUERY_ID> 18 <TID>
 SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST;
 --replace_column 1 <ID> 3 <HOST_NAME> 6 <TIME>
 SHOW FULL PROCESSLIST;
@@ -226,7 +226,7 @@ let $wait_condition= SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.PROCESSLIST
 --source include/wait_condition.inc
 connection con2;
 # Just dump what we get
---replace_column 1 <ID> 3 <HOST_NAME> 6 <TIME> 9 <TIME_MS> 13 <MEMORY> 14 <ROWS> 15 <QUERY_ID> 17 <TID>
+--replace_column 1 <ID> 3 <HOST_NAME> 6 <TIME> 9 <TIME_MS> 13 <MEMORY> 14 <MAX_MEMORY> 15 <ROWS> 16 <QUERY_ID> 18 <TID>
 SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST;
 --replace_column 1 <ID> 3 <HOST_NAME> 6 <TIME>
 SHOW FULL PROCESSLIST;
@@ -277,7 +277,7 @@ WHERE ID = @test_user_con2_id AND Command IN('Query','Execute')
   AND State = 'User sleep' AND INFO IS NOT NULL ;
 --source include/wait_condition.inc
 # 1. Just dump what we get
---replace_column 1 <ID> 3 <HOST_NAME> 6 <TIME> 9 <TIME_MS> 13 <MEMORY> 14 <ROWS> 15 <QUERY_ID> 17 <TID>
+--replace_column 1 <ID> 3 <HOST_NAME> 6 <TIME> 9 <TIME_MS> 13 <MEMORY> 14 <MAX_MEMORY> 15 <ROWS> 16 <QUERY_ID> 18 <TID>
 SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST;
 --replace_column 1 <ID> 3 <HOST_NAME> 6 <TIME>
 SHOW FULL PROCESSLIST;
@@ -336,7 +336,7 @@ let $wait_condition= SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST
 #
 # Expect to see the state 'Waiting for table metadata lock' for the third
 # connection because the SELECT collides with the WRITE TABLE LOCK.
---replace_column 1 <ID> 3 <HOST_NAME> 6 <TIME> 9 <TIME_MS> 13 <MEMORY> 14 <ROWS> 15 <QUERY_ID> 17 <TID>
+--replace_column 1 <ID> 3 <HOST_NAME> 6 <TIME> 9 <TIME_MS> 13 <MEMORY> 14 <MAX_MEMORY> 15 <ROWS> 16 <QUERY_ID> 18 <TID>
 SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST;
 UNLOCK TABLES;
 #
@@ -383,7 +383,7 @@ echo
 # SHOW FULL PROCESSLIST                          Complete statement
 # SHOW PROCESSLIST                               statement truncated after 100 char
 ;
---replace_column 1 <ID> 3 <HOST_NAME> 5 <COMMAND> 6 <TIME> 7 <STATE> 9 <TIME_MS> 13 <MEMORY> 14 <ROWS> 15 <QUERY_ID> 17 <TID>
+--replace_column 1 <ID> 3 <HOST_NAME> 5 <COMMAND> 6 <TIME> 7 <STATE> 9 <TIME_MS> 13 <MEMORY> 14 <MAX_MEMORY> 15 <ROWS> 16 <QUERY_ID> 18 <TID>
 SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST;
 --replace_column 1 <ID> 3 <HOST_NAME> 5 <COMMAND> 6 <TIME> 7 <STATE>
 SHOW FULL PROCESSLIST;

mysql-test/suite/funcs_1/r/is_columns_is.result

@@ -256,18 +256,19 @@ def	information_schema	PLUGINS	PLUGIN_TYPE_VERSION	5	''	NO	varchar	20	60	NULL	NU
 def	information_schema	PLUGINS	PLUGIN_VERSION	2	''	NO	varchar	20	60	NULL	NULL	NULL	utf8	utf8_general_ci	varchar(20)			select		NEVER	NULL
 def	information_schema	PROCESSLIST	COMMAND	5	''	NO	varchar	16	48	NULL	NULL	NULL	utf8	utf8_general_ci	varchar(16)			select		NEVER	NULL
 def	information_schema	PROCESSLIST	DB	4	NULL	YES	varchar	64	192	NULL	NULL	NULL	utf8	utf8_general_ci	varchar(64)			select		NEVER	NULL
-def	information_schema	PROCESSLIST	EXAMINED_ROWS	14	0	NO	int	NULL	NULL	10	0	NULL	NULL	NULL	int(7)			select		NEVER	NULL
+def	information_schema	PROCESSLIST	EXAMINED_ROWS	15	0	NO	int	NULL	NULL	10	0	NULL	NULL	NULL	int(7)			select		NEVER	NULL
 def	information_schema	PROCESSLIST	HOST	3	''	NO	varchar	64	192	NULL	NULL	NULL	utf8	utf8_general_ci	varchar(64)			select		NEVER	NULL
 def	information_schema	PROCESSLIST	ID	1	0	NO	bigint	NULL	NULL	19	0	NULL	NULL	NULL	bigint(4)			select		NEVER	NULL
 def	information_schema	PROCESSLIST	INFO	8	NULL	YES	longtext	4294967295	4294967295	NULL	NULL	NULL	utf8	utf8_general_ci	longtext			select		NEVER	NULL
-def	information_schema	PROCESSLIST	INFO_BINARY	16	NULL	YES	blob	65535	65535	NULL	NULL	NULL	NULL	NULL	blob			select		NEVER	NULL
+def	information_schema	PROCESSLIST	INFO_BINARY	17	NULL	YES	blob	65535	65535	NULL	NULL	NULL	NULL	NULL	blob			select		NEVER	NULL
+def	information_schema	PROCESSLIST	MAX_MEMORY_USED	14	0	NO	bigint	NULL	NULL	19	0	NULL	NULL	NULL	bigint(7)			select		NEVER	NULL
 def	information_schema	PROCESSLIST	MAX_STAGE	11	0	NO	tinyint	NULL	NULL	3	0	NULL	NULL	NULL	tinyint(2)			select		NEVER	NULL
 def	information_schema	PROCESSLIST	MEMORY_USED	13	0	NO	bigint	NULL	NULL	19	0	NULL	NULL	NULL	bigint(7)			select		NEVER	NULL
 def	information_schema	PROCESSLIST	PROGRESS	12	0.000	NO	decimal	NULL	NULL	7	3	NULL	NULL	NULL	decimal(7,3)			select		NEVER	NULL
-def	information_schema	PROCESSLIST	QUERY_ID	15	0	NO	bigint	NULL	NULL	19	0	NULL	NULL	NULL	bigint(4)			select		NEVER	NULL
+def	information_schema	PROCESSLIST	QUERY_ID	16	0	NO	bigint	NULL	NULL	19	0	NULL	NULL	NULL	bigint(4)			select		NEVER	NULL
 def	information_schema	PROCESSLIST	STAGE	10	0	NO	tinyint	NULL	NULL	3	0	NULL	NULL	NULL	tinyint(2)			select		NEVER	NULL
 def	information_schema	PROCESSLIST	STATE	7	NULL	YES	varchar	64	192	NULL	NULL	NULL	utf8	utf8_general_ci	varchar(64)			select		NEVER	NULL
-def	information_schema	PROCESSLIST	TID	17	0	NO	bigint	NULL	NULL	19	0	NULL	NULL	NULL	bigint(4)			select		NEVER	NULL
+def	information_schema	PROCESSLIST	TID	18	0	NO	bigint	NULL	NULL	19	0	NULL	NULL	NULL	bigint(4)			select		NEVER	NULL
 def	information_schema	PROCESSLIST	TIME	6	0	NO	int	NULL	NULL	10	0	NULL	NULL	NULL	int(7)			select		NEVER	NULL
 def	information_schema	PROCESSLIST	TIME_MS	9	0.000	NO	decimal	NULL	NULL	22	3	NULL	NULL	NULL	decimal(22,3)			select		NEVER	NULL
 def	information_schema	PROCESSLIST	USER	2	''	NO	varchar	128	384	NULL	NULL	NULL	utf8	utf8_general_ci	varchar(128)			select		NEVER	NULL
@@ -797,6 +798,7 @@ NULL	information_schema	PROCESSLIST	STAGE	tinyint	NULL	NULL	NULL	NULL	tinyint(2)
 NULL	information_schema	PROCESSLIST	MAX_STAGE	tinyint	NULL	NULL	NULL	NULL	tinyint(2)
 NULL	information_schema	PROCESSLIST	PROGRESS	decimal	NULL	NULL	NULL	NULL	decimal(7,3)
 NULL	information_schema	PROCESSLIST	MEMORY_USED	bigint	NULL	NULL	NULL	NULL	bigint(7)
+NULL	information_schema	PROCESSLIST	MAX_MEMORY_USED	bigint	NULL	NULL	NULL	NULL	bigint(7)
 NULL	information_schema	PROCESSLIST	EXAMINED_ROWS	int	NULL	NULL	NULL	NULL	int(7)
 NULL	information_schema	PROCESSLIST	QUERY_ID	bigint	NULL	NULL	NULL	NULL	bigint(4)
 1.0000	information_schema	PROCESSLIST	INFO_BINARY	blob	65535	65535	NULL	NULL	blob