MDEV-18115 Remove dummy tablespace for the redo log

Redo log subsystem was decoupled from tablespace subsystem. It now manages file
descriptors for redo log files by itself.

FIL_TYPE_LOG: removed, code in various places was simplified

SRV_LOG_SPACE_FIRST_ID: renamed to SRV_SPACE_ID_UPPER_BOUND
  to better match its purpose. Code in various places was simplified

fil_n_log_flushes: replaced with log_sys::flushes
fil_n_pending_log_flushes: replaced with log_sys::pending_flushes

log_t::files::files: redo log file descriptors
log_t::files::file_names: redo log file names

log_t::files::set_file_names(): set file names without opening them
log_t::files::open_files(): opens redo log files
log_t::files::read(): treats several files as one big
log_t::files::write(): treats several files as one big
log_t::files::fsync(): flushes page cache to disk
log_t::files::close_files(): closes redo log files

fil_open_log_and_system_tablespace_files(): renamed to
  fil_open_system_tablespace_files()
  and obviously it now doesn't open redo log files

global files[1000]: removed. Why it was needed at all?

extra/mariabackup/xtrabackup.cc

@@ -3791,14 +3791,14 @@ xb_filters_free()
 	}
 }
 
-/*********************************************************************//**
-Create log file metadata. */
+/**Create log file metadata.
+@param[in]	i		log file number in group
+@param[in,out]	file_names	redo log file names */
 static
 void
 open_or_create_log_file(
-/*====================*/
-	fil_space_t* space,
-	ulint	i)			/*!< in: log file number in group */
+	ulint	i,
+	std::vector<std::string> &file_names)
 {
 	char	name[FN_REFLEN];
 	ulint	dirnamelen;
@@ -3818,9 +3818,7 @@ open_or_create_log_file(
 
 	ut_a(fil_validate());
 
-	space->add(name, OS_FILE_CLOSED,
-		   ulint(srv_log_file_size >> srv_page_size_shift),
-		   false, false);
+	file_names.emplace_back(name);
 }
 
 /***********************************************************************
@@ -4080,14 +4078,16 @@ static bool xtrabackup_backup_func()
 
 	log_sys.create();
 	log_sys.log.create(srv_n_log_files);
-	fil_space_t*	space = fil_space_create(
-		"innodb_redo_log", SRV_LOG_SPACE_FIRST_ID, 0,
-		FIL_TYPE_LOG, NULL);
+
+	std::vector<std::string> file_names;
 
 	for (ulint i = 0; i < srv_n_log_files; i++) {
-		open_or_create_log_file(space, i);
+		open_or_create_log_file(i, file_names);
 	}
 
+	log_sys.log.set_file_names(std::move(file_names));
+	log_sys.log.open_files();
+
 	/* create extra LSN dir if it does not exist. */
 	if (xtrabackup_extra_lsndir
 		&&!my_stat(xtrabackup_extra_lsndir,&stat_info,MYF(0))
@@ -5537,8 +5537,7 @@ static bool xtrabackup_prepare_func(char** argv)
 	}
 
 	/* Check whether the log is applied enough or not. */
-	if ((srv_start_lsn || fil_space_get(SRV_LOG_SPACE_FIRST_ID))
-	    && srv_start_lsn < target_lsn) {
+	if (srv_start_lsn && srv_start_lsn < target_lsn) {
 		msg("mariabackup: error: "
 		    "The log was only applied up to LSN " LSN_PF
 		    ", instead of " LSN_PF,

storage/innobase/buf/buf0buddy.cc

@@ -39,7 +39,7 @@ safe to look at BUF_BUDDY_STAMP_OFFSET.
 The answer lies in following invariants:
 * All blocks allocated by buddy allocator are used for compressed
 page frame.
-* A compressed table always have space_id < SRV_LOG_SPACE_FIRST_ID
+* A compressed table always have space_id < SRV_SPACE_ID_UPPER_BOUND
 * BUF_BUDDY_STAMP_OFFSET always points to the space_id field in
 a frame.
   -- The above is true because we look at these fields when the
@@ -67,7 +67,7 @@ are written.*/
 
 /** Value that we stamp on all buffers that are currently on the zip_free
 list. This value is stamped at BUF_BUDDY_STAMP_OFFSET offset */
-#define BUF_BUDDY_STAMP_FREE	 SRV_LOG_SPACE_FIRST_ID
+#define BUF_BUDDY_STAMP_FREE	 SRV_SPACE_ID_UPPER_BOUND
 
 /** Stamp value for non-free buffers. Will be overwritten by a non-zero
 value by the consumer of the block */

storage/innobase/buf/buf0dump.cc

@@ -352,7 +352,7 @@ buf_dump(
 		     bpage = UT_LIST_GET_NEXT(LRU, bpage)) {
 
 			ut_a(buf_page_in_file(bpage));
-			if (bpage->id.space() >= SRV_LOG_SPACE_FIRST_ID) {
+			if (bpage->id.space() == SRV_TMP_SPACE_ID) {
 				/* Ignore the innodb_temporary tablespace. */
 				continue;
 			}
@@ -679,7 +679,7 @@ buf_load()
 		/* space_id for this iteration of the loop */
 		const ulint	this_space_id = BUF_DUMP_SPACE(dump[i]);
 
-		if (this_space_id >= SRV_LOG_SPACE_FIRST_ID) {
+		if (this_space_id == SRV_TMP_SPACE_ID) {
 			/* Ignore the innodb_temporary tablespace. */
 			continue;
 		}

storage/innobase/buf/buf0lru.cc

@@ -1152,7 +1152,7 @@ buf_LRU_get_free_block(
 			" flush a page!"
 			" Consider increasing innodb_buffer_pool_size."
 			" Pending flushes (fsync) log: "
-			<< fil_n_pending_log_flushes
+			<< log_sys.get_pending_flushes()
 			<< "; buffer pool: "
 			<< fil_n_pending_tablespace_flushes
 			<< ". " << os_n_file_reads << " OS file reads, "

storage/innobase/dict/dict0load.cc

@@ -2764,7 +2764,7 @@ dict_load_tablespace(
 {
 	ut_ad(!table->is_temporary());
 	ut_ad(!table->space);
-	ut_ad(table->space_id < SRV_LOG_SPACE_FIRST_ID);
+	ut_ad(table->space_id < SRV_SPACE_ID_UPPER_BOUND);
 	ut_ad(fil_system.sys_space);
 
 	if (table->space_id == TRX_SYS_SPACE) {

storage/innobase/fil/fil0fil.cc

@@ -162,11 +162,6 @@ const char*	fil_path_to_mysql_datadir;
 /** Common InnoDB file extensions */
 const char* dot_ext[] = { "", ".ibd", ".isl", ".cfg" };
 
-/** The number of fsyncs done to the log */
-ulint	fil_n_log_flushes			= 0;
-
-/** Number of pending redo log flushes */
-ulint	fil_n_pending_log_flushes		= 0;
 /** Number of pending tablespace flushes */
 ulint	fil_n_pending_tablespace_flushes	= 0;
 
@@ -228,7 +223,6 @@ fil_space_belongs_in_lru(
 {
 	switch (space->purpose) {
 	case FIL_TYPE_TEMPORARY:
-	case FIL_TYPE_LOG:
 		return(false);
 	case FIL_TYPE_TABLESPACE:
 		return(fil_is_user_tablespace_id(space->id));
@@ -354,7 +348,6 @@ fil_space_get(
 	mutex_enter(&fil_system.mutex);
 	fil_space_t*	space = fil_space_get_by_id(id);
 	mutex_exit(&fil_system.mutex);
-	ut_ad(space == NULL || space->purpose != FIL_TYPE_LOG);
 	return(space);
 }
 
@@ -560,10 +553,6 @@ static bool fil_node_open_file(fil_node_t* node)
 			goto fail;
 		}
 
-	} else if (space->purpose == FIL_TYPE_LOG) {
-		node->handle = os_file_create(
-			innodb_log_file_key, node->name, OS_FILE_OPEN,
-			OS_FILE_AIO, OS_LOG_FILE, read_only_mode, &success);
 	} else {
 		node->handle = os_file_create(
 			innodb_data_file_key, node->name,
@@ -730,10 +719,6 @@ static void fil_flush_low(fil_space_t* space, bool metadata = false)
 		case FIL_TYPE_IMPORT:
 			fil_n_pending_tablespace_flushes++;
 			break;
-		case FIL_TYPE_LOG:
-			fil_n_pending_log_flushes++;
-			fil_n_log_flushes++;
-			break;
 		}
 #ifdef _WIN32
 		if (node->is_raw_disk) {
@@ -773,9 +758,6 @@ static void fil_flush_low(fil_space_t* space, bool metadata = false)
 		case FIL_TYPE_IMPORT:
 			fil_n_pending_tablespace_flushes--;
 			continue;
-		case FIL_TYPE_LOG:
-			fil_n_pending_log_flushes--;
-			continue;
 		}
 
 		ut_ad(0);
@@ -912,11 +894,6 @@ fil_mutex_enter_and_prepare_for_io(
 	for (ulint count = 0;;) {
 		mutex_enter(&fil_system.mutex);
 
-		if (space_id >= SRV_LOG_SPACE_FIRST_ID) {
-			/* We keep log files always open. */
-			break;
-		}
-
 		fil_space_t*	space = fil_space_get_by_id(space_id);
 
 		if (space == NULL) {
@@ -1214,8 +1191,7 @@ fil_space_create(
 
 	ut_ad(fil_system.is_initialised());
 	ut_ad(fil_space_t::is_valid_flags(flags & ~FSP_FLAGS_MEM_MASK, id));
-	ut_ad(purpose == FIL_TYPE_LOG
-	      || srv_page_size == UNIV_PAGE_SIZE_ORIG || flags != 0);
+	ut_ad(srv_page_size == UNIV_PAGE_SIZE_ORIG || flags != 0);
 
 	DBUG_EXECUTE_IF("fil_space_create_failure", return(NULL););
 
@@ -1286,7 +1262,7 @@ fil_space_create(
 
 	UT_LIST_ADD_LAST(fil_system.space_list, space);
 
-	if (id < SRV_LOG_SPACE_FIRST_ID && id > fil_system.max_assigned_id) {
+	if (id < SRV_SPACE_ID_UPPER_BOUND && id > fil_system.max_assigned_id) {
 
 		fil_system.max_assigned_id = id;
 	}
@@ -1332,16 +1308,16 @@ fil_assign_new_space_id(
 
 	id++;
 
-	if (id > (SRV_LOG_SPACE_FIRST_ID / 2) && (id % 1000000UL == 0)) {
+	if (id > (SRV_SPACE_ID_UPPER_BOUND / 2) && (id % 1000000UL == 0)) {
 		ib::warn() << "You are running out of new single-table"
 			" tablespace id's. Current counter is " << id
-			<< " and it must not exceed" << SRV_LOG_SPACE_FIRST_ID
+			<< " and it must not exceed" <<SRV_SPACE_ID_UPPER_BOUND
 			<< "! To reset the counter to zero you have to dump"
 			" all your tables and recreate the whole InnoDB"
 			" installation.";
 	}
 
-	success = (id < SRV_LOG_SPACE_FIRST_ID);
+	success = (id < SRV_SPACE_ID_UPPER_BOUND);
 
 	if (success) {
 		*space_id = fil_system.max_assigned_id = id;
@@ -1416,8 +1392,6 @@ fil_space_get_space(
 	}
 
 	switch (space->purpose) {
-	case FIL_TYPE_LOG:
-		break;
 	case FIL_TYPE_TEMPORARY:
 	case FIL_TYPE_TABLESPACE:
 	case FIL_TYPE_IMPORT:
@@ -1436,7 +1410,7 @@ fil_space_set_recv_size(ulint id, ulint size)
 {
 	mutex_enter(&fil_system.mutex);
 	ut_ad(size);
-	ut_ad(id < SRV_LOG_SPACE_FIRST_ID);
+	ut_ad(id < SRV_SPACE_ID_UPPER_BOUND);
 
 	if (fil_space_t* space = fil_space_get_space(id)) {
 		space->recv_size = size;
@@ -1620,15 +1594,13 @@ void fil_system_t::close()
 	ut_ad(!spaces);
 }
 
-/*******************************************************************//**
-Opens all log files and system tablespace data files. They stay open until the
+/** Opens all system tablespace data files. They stay open until the
 database server shutdown. This should be called at a server startup after the
-space objects for the log and the system tablespace have been created. The
+space objects for the system tablespace have been created. The
 purpose of this operation is to make sure we never run out of file descriptors
-if we need to read from the insert buffer or to write to the log. */
+if we need to read from the insert buffer. */
 void
-fil_open_log_and_system_tablespace_files(void)
-/*==========================================*/
+fil_open_system_tablespace_files()
 {
 	fil_space_t*	space;
 
@@ -1726,56 +1698,6 @@ fil_close_all_files(void)
 	      || UT_LIST_GET_LEN(fil_system.named_spaces) == 0);
 }
 
-/*******************************************************************//**
-Closes the redo log files. There must not be any pending i/o's or not
-flushed modifications in the files. */
-void
-fil_close_log_files(
-/*================*/
-	bool	free)	/*!< in: whether to free the memory object */
-{
-	fil_space_t*	space;
-
-	mutex_enter(&fil_system.mutex);
-
-	space = UT_LIST_GET_FIRST(fil_system.space_list);
-
-	while (space != NULL) {
-		fil_node_t*	node;
-		fil_space_t*	prev_space = space;
-
-		if (space->purpose != FIL_TYPE_LOG) {
-			space = UT_LIST_GET_NEXT(space_list, space);
-			continue;
-		}
-
-		/* Log files are not in the fil_system.named_spaces list. */
-		ut_ad(space->max_lsn == 0);
-
-		for (node = UT_LIST_GET_FIRST(space->chain);
-		     node != NULL;
-		     node = UT_LIST_GET_NEXT(chain, node)) {
-
-			if (node->is_open()) {
-				node->close();
-			}
-		}
-
-		space = UT_LIST_GET_NEXT(space_list, space);
-
-		if (free) {
-			fil_space_detach(prev_space);
-			fil_space_free_low(prev_space);
-		}
-	}
-
-	mutex_exit(&fil_system.mutex);
-
-	if (free) {
-		log_sys.log.close();
-	}
-}
-
 /*******************************************************************//**
 Sets the max tablespace id counter if the given number is bigger than the
 previous value. */
@@ -1784,7 +1706,7 @@ fil_set_max_space_id_if_bigger(
 /*===========================*/
 	ulint	max_id)	/*!< in: maximum known id */
 {
-	if (max_id >= SRV_LOG_SPACE_FIRST_ID) {
+	if (max_id >= SRV_SPACE_ID_UPPER_BOUND) {
 		ib::fatal() << "Max tablespace id is too high, " << max_id;
 	}
 
@@ -2717,7 +2639,7 @@ fil_rename_tablespace_check(
 	for (fil_space_t* space = UT_LIST_GET_FIRST(fil_system.space_list);
 	     space; space = UT_LIST_GET_NEXT(space_list, space)) {
 		ulint id = space->id;
-		if (id && id < SRV_LOG_SPACE_FIRST_ID
+		if (id
 		    && space->purpose == FIL_TYPE_TABLESPACE
 		    && !strcmp(new_path,
 			       UT_LIST_GET_FIRST(space->chain)->name)) {
@@ -2892,7 +2814,7 @@ fil_ibd_create(
 
 	ut_ad(!is_system_tablespace(space_id));
 	ut_ad(!srv_read_only_mode);
-	ut_a(space_id < SRV_LOG_SPACE_FIRST_ID);
+	ut_a(space_id < SRV_SPACE_ID_UPPER_BOUND);
 	ut_a(size >= FIL_IBD_FILE_INITIAL_SIZE);
 	ut_a(fil_space_t::is_valid_flags(flags & ~FSP_FLAGS_MEM_MASK, space_id));
 
@@ -4259,7 +4181,6 @@ fil_io(
 	req_type.set_fil_node(node);
 
 	ut_ad(!req_type.is_write()
-	      || page_id.space() == SRV_LOG_SPACE_FIRST_ID
 	      || !fil_is_user_tablespace_id(page_id.space())
 	      || offset == page_id.page_no() * zip_size);
 
@@ -4314,7 +4235,6 @@ void fil_aio_callback(const tpool::aiocb *cb)
 	mutex_enter(&fil_system.mutex);
 
 	fil_node_complete_io(node, data->type);
-	ut_ad(node->space->purpose != FIL_TYPE_LOG);
 	const ulint		space_id= node->space->id;
 	bool			dblwr	= node->space->use_doublewrite();
 
@@ -4403,7 +4323,7 @@ fil_flush(fil_space_t* space)
 
 /** Flush to disk the writes in file spaces of the given type
 possibly cached by the OS.
-@param[in]	purpose	FIL_TYPE_TABLESPACE or FIL_TYPE_LOG */
+@param[in]	purpose	FIL_TYPE_TABLESPACE */
 void
 fil_flush_file_spaces(
 	fil_type_t	purpose)
@@ -4412,7 +4332,7 @@ fil_flush_file_spaces(
 	ulint*		space_ids;
 	ulint		n_space_ids;
 
-	ut_ad(purpose == FIL_TYPE_TABLESPACE || purpose == FIL_TYPE_LOG);
+	ut_ad(purpose == FIL_TYPE_TABLESPACE);
 
 	mutex_enter(&fil_system.mutex);
 
@@ -4755,7 +4675,7 @@ fil_names_dirty_and_write(
 				char bogus_name[] = "./test/bogus file.ibd";
 				os_normalize_path(bogus_name);
 				fil_name_write(
-					SRV_LOG_SPACE_FIRST_ID, 0,
+					SRV_SPACE_ID_UPPER_BOUND, 0,
 					bogus_name, mtr);
 			});
 }