implement extendable PPL log block lines, TT lines

Author: trdtnguyen

build_mysql.sh

@@ -2,8 +2,8 @@
 MYSQL_HOME=/home/vldb/mysql-plnvm
 
 #debug mode
-IS_DEBUG=0
-#IS_DEBUG=1
+#IS_DEBUG=0
+IS_DEBUG=1
 
 #BUILD_NAME="-DUNIV_TRACE_FLUSH_TIME"
 #BUILD_NAME="-DUNIV_TRACE_FLUSH_TIME -DUNIV_SKIPLOG"

storage/innobase/btr/btr0btr.cc

@@ -617,9 +617,6 @@ btr_page_free_low(
 	}
 #endif
 
-#if defined (UNIV_PMEMOBJ_PART_PL)
-	printf("====> PMEM_DEBUG: btr_page_free_low() free page(%zu, %zu)\n", block->page.id.space(), block->page.id.page_no());
-#endif
 	fseg_free_page(seg_header,
 		       block->page.id.space(),
 		       block->page.id.page_no(),

storage/innobase/buf/buf0lru.cc

@@ -2142,10 +2142,10 @@ buf_LRU_block_free_non_file_page(
 	/* Wipe page_no and space_id */
 	memset(block->frame + FIL_PAGE_OFFSET, 0xfe, 4);
 	memset(block->frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, 0xfe, 4);
-#if defined (UNIV_PMEMOBJ_PART_PL)
-		printf("PMEM_DEBUG: buf_LRU_block_free_non_file_page() page (%zu, %zu)\n",
-				bpage->id.space(), bpage->id.page_no());
-#endif
+//#if defined (UNIV_PMEMOBJ_PART_PL)
+//		printf("PMEM_DEBUG: buf_LRU_block_free_non_file_page() page (%zu, %zu)\n",
+//				bpage->id.space(), bpage->id.page_no());
+//#endif
 #endif /* UNIV_DEBUG */
 	data = block->page.zip.data;
 

storage/innobase/fil/fil0fil.cc

@@ -5948,8 +5948,8 @@ pm_ppl_test_write_rseg(
 	//get the pointer to the UNDO seg header page
 	buf_pool_t* buf_pool = buf_pool_get(page_id);
 	block = (buf_block_t*) buf_page_hash_get_low(buf_pool, page_id_t(page_id.space(), p1));
-	printf("///////////////////////\n");
-	printf("===>> IS_WRITE %zu FIL_IO: is in BP check RSEG page %zu header UNDO page (%zu, %zu) \n", req_type.is_write(), page_id.page_no(), p1, (block!=NULL));
+	//printf("///////////////////////\n");
+	//printf("===>> IS_WRITE %zu FIL_IO: is in BP check RSEG page %zu header UNDO page (%zu, %zu) \n", req_type.is_write(), page_id.page_no(), p1, (block!=NULL));
 
 	if (block == NULL) {
 		printf("///////////////////////\n");
@@ -8760,8 +8760,6 @@ pm_log_fil_io(
 
 	ulint		start_offset;
 	ulint		end_offset;
-	ulint		area_start;
-	ulint		area_end;
 
 	ulint		next_offset;
 	ulint		write_offset;
@@ -8784,14 +8782,10 @@ pm_log_fil_io(
 	log_src = pdata + plogbuf->pmemaddr;
 
 	start_offset = 0;
-	//end_offset = plogbuf->cur_off;
 	end_offset = plogbuf->size;
 
 	//(1) Align the offset with the page size
-	//area_start = ut_calc_align_down(start_offset, OS_FILE_LOG_BLOCK_SIZE);
-	//area_end = ut_calc_align_down(end_offset, OS_FILE_LOG_BLOCK_SIZE);
 
-	//len = area_end - area_start;
 	len = plogbuf->size;
 	ut_ad(len > 0);
 

storage/innobase/include/mtr0log.h

@@ -87,15 +87,6 @@ mlog_write_initial_log_record(
 
 /********************************************************//**
 Catenates 1 - 4 bytes to the mtr log. The value is not compressed. */
-#if defined (UNIV_PMEMOBJ_PART_PL)
-UNIV_INLINE
-void
-mlog_catenate_ulint(
-/*================*/
-	mtr_buf_t*	dyn_buf,	/*!< in/out: buffer to write */
-	ulint		val,		/*!< in: value to write */
-	mlog_id_t	type);		/*!< in: type of value to write */
-#else
 UNIV_INLINE
 void
 mlog_catenate_ulint(
@@ -112,7 +103,6 @@ mlog_catenate_ulint(
 	mtr_t*		mtr,	/*!< in: mtr */
 	ulint		val,	/*!< in: value to write */
 	mlog_id_t	type);	/*!< in: MLOG_1BYTE, MLOG_2BYTES, MLOG_4BYTES */
-#endif
 /********************************************************//**
 Catenates n bytes to the mtr log. */
 void

storage/innobase/include/mtr0log.ic

@@ -78,6 +78,19 @@ mlog_close(
 	mtr->set_cur_off(new_off);
 }
 
+/*Catenates 1 - 4 bytes to the mtr log. The value is not compressed. */
+UNIV_INLINE
+void
+mlog_catenate_ulint(
+/*================*/
+	mtr_buf_t*	mtr_buf,	/*!< in/out: buffer to write */
+	ulint		val,		/*!< in: value to write */
+	mlog_id_t	type)		/*!< in: type of value to write */
+{
+	//we don't use this function
+	assert(0);
+}
+
 /* PPL FUNCTION
 Catenates 1 - 4 bytes to the mtr log. The value is not compressed.
 This function use push instead of open_log and close_log 

storage/innobase/include/my_pmem_common.h

@@ -70,6 +70,13 @@ enum PMEM_OBJ_TYPES {
 	BUF_TYPE,
 	META_DATA_TYPE
 };
+
+//use in pm_ppl_write()
+#define PMEM_EID_NEW 0
+#define PMEM_EID_REVISIT 1
+#define PMEM_EID_UNDEFINED 2
+
+
 enum PMEM_BLOCK_STATE {
     PMEM_FREE_BLOCK = 1,
     PMEM_IN_USED_BLOCK = 2,

storage/innobase/include/my_pmemobj.h

@@ -545,8 +545,9 @@ struct __pmem_page_log_hashed_line {
 	uint64_t		recv_lsn; //min lsn of block's beginLSN in this line
 	/*end test */
 
-	uint64_t n_blocks; //the total log block in bucket
 	TOID_ARRAY(TOID(PMEM_PAGE_LOG_BLOCK)) arr;
+	uint64_t n_blocks; //the current non-free blocks
+	uint64_t max_blocks; //the total log block in bucket
 
 	//Alternative to recv_sys_t in InnoDB, allocate in DRAM when recovery
 	PMEM_RECV_LINE* recv_line;
@@ -598,7 +599,7 @@ struct __pmem_page_log_buf {
 struct __pmem_page_log_block {
 	PMEMrwlock		lock;
 	bool			is_free; //flag
-	uint32_t		bid; //block id
+	uint64_t		bid; //block id
 	uint64_t		key; //fold id
 
 	//uint64_t		cur_size; //the current offset (0 - log block size) 
@@ -688,7 +689,8 @@ pm_log_redoer_close(PMEM_LOG_REDOER*	redoer);
 // Dirty Page Entry Ref in the log block
 struct __pmem_page_ref {
 	uint64_t	key; //fold of page_id_t.fold() in InnoDB
-	int64_t		idx; // index of the entry in DPT
+	//int64_t		idx; // index of the entry in DPT
+	uint64_t		idx; // index of the entry in DPT
 	uint64_t	pageLSN; //latest LSN on this page caused by the host transction, this value used in reclaiming logs
 };
 
@@ -708,8 +710,10 @@ struct __pmem_tt_hashed_line {
 	PMEMrwlock		lock;
 
 	uint64_t		hashed_id;
-	uint64_t		n_entries; //the total log block in bucket
+
 	TOID_ARRAY(TOID(PMEM_TT_ENTRY)) arr;
+	uint64_t		n_entries; //current number of entries
+	uint64_t		max_entries; //capacity (extendable)
 
 };
 
@@ -956,9 +960,20 @@ void __init_tt_entry(
 		uint64_t k,
 		uint64_t pages_per_tx);
 
+uint64_t
+pm_ppl_create_entry_id(
+	   	uint16_t type,
+	   	uint32_t row,
+	   	uint32_t col);
+void
+pm_ppl_parse_entry_id(
+		uint64_t val,
+	   	uint16_t* type,
+	   	uint32_t* row,
+	   	uint32_t* col);
 
 /*Called when a mini-transaction write log record to log buffer in the traditional InnoDB*/
-int64_t
+uint64_t
 pm_ppl_write(
 			PMEMobjpool*		pop,
 			PMEM_PAGE_PART_LOG*	ppl,
@@ -968,10 +983,9 @@ pm_ppl_write(
 			uint64_t			n_recs,
 			uint64_t*			key_arr,
 			uint64_t*			size_arr,
-			//uint64_t*			LSN_arr,
 			uint64_t*			ret_start_lsn,
 			uint64_t*			ret_end_lsn,
-			int64_t				block_id);
+			uint64_t			block_id);
 
 void __handle_pm_ppl_write_by_entry(
 			PMEMobjpool*		pop,
@@ -988,7 +1002,7 @@ void __handle_pm_ppl_write_by_entry(
 			PMEM_TT_ENTRY*		pe,
 			bool				is_new);
 
-int64_t
+uint64_t
 __update_page_log_block_on_write(
 			PMEMobjpool*		pop,
 			PMEM_PAGE_PART_LOG*	ppl,
@@ -998,7 +1012,7 @@ __update_page_log_block_on_write(
 			uint64_t			key,
 			uint64_t*			LSN,
 			int64_t				eid,
-			int64_t				bid);
+			uint64_t			bid);
 
 PMEM_PAGE_LOG_BLOCK*
 __get_log_block_by_id(
@@ -1094,7 +1108,7 @@ pm_ppl_commit(
 		PMEMobjpool*			pop,
 		PMEM_PAGE_PART_LOG*		ppl,
 		uint64_t				tid,
-		int64_t					eid);
+		uint64_t					eid);
 void 
 pm_ppl_flush_page(
 		PMEMobjpool*		pop,
@@ -1293,12 +1307,24 @@ __update_page_log_block_on_commit(
 		PMEMobjpool*				pop,
 		PMEM_PAGE_PART_LOG*			ppl,
 		PMEM_PAGE_REF*				pref,
-		int64_t						eid);
+		uint64_t					eid);
 void 
 __reset_TT_entry(
 		PMEMobjpool*				pop,
 		PMEM_PAGE_PART_LOG*			ppl,
 		PMEM_TT_ENTRY*				pe);
+
+void __realloc_page_log_block_line(
+		PMEMobjpool*			pop,
+		PMEM_PAGE_PART_LOG*		ppl,
+		PMEM_PAGE_LOG_HASHED_LINE*	pline,
+		uint64_t				new_size);
+
+void __realloc_TT_line(
+		PMEMobjpool*			pop,
+		PMEM_PAGE_PART_LOG*		ppl,
+		PMEM_TT_HASHED_LINE*	pline,
+		uint64_t				new_size);
 void 
 __realloc_TT_entry(
 		PMEMobjpool*				pop,

storage/innobase/include/trx0trx.h

@@ -912,7 +912,9 @@ struct trx_t {
 					this transaction must abort when
 					it can */
 #if defined(UNIV_PMEMOBJ_PART_PL)
-	int64_t		pm_log_block_id;
+	uint64_t		pm_log_block_id; /*1byte type (1: NEW, 2: REVISIT, 3: UNDEFINED
+									   3-byte line_id,
+									   4-byte offset*/
 #endif
 	trx_id_t	id;		/*!< transaction id */
 

storage/innobase/mtr/mtr0mtr.cc

@@ -1351,6 +1351,9 @@ mtr_t::Command::execute()
 		else{
 			//assert(type > 0 && type <= 8);
 			//all type <= 8 is treat as trx_id 0
+			uint64_t dummy_eid;
+			dummy_eid = pm_ppl_create_entry_id(PMEM_EID_UNDEFINED, 0, 0);
+
 			pm_ppl_write(
 					gb_pmw->pop,
 					gb_pmw->ppl,
@@ -1362,7 +1365,7 @@ mtr_t::Command::execute()
 					m_impl->size_arr,
 					&ret_start_lsn,
 					&ret_end_lsn,
-					-2);
+					dummy_eid);
 		}
 
 		//Update m_start_lsn and m_end_lsn. They are required for update pageLSN in release_block()