mysql: allocate memory for each string and blob dynamically depending on its value length (#18214)

Author: Mark aka walkingdevel

vlib/db/mysql/orm.v

@@ -3,10 +3,6 @@ module mysql
 import orm
 import time
 
-type Prims = f32 | f64 | i16 | i64 | i8 | int | string | u16 | u32 | u64 | u8
-
-// sql expr
-
 // @select is used internally by V's ORM for processing `SELECT ` queries
 pub fn (db Connection) @select(config orm.SelectConfig, data orm.QueryData, where orm.QueryData) ![][]orm.Primitive {
 	query := orm.orm_select_gen(config, '`', false, '?', 0, where)
@@ -28,8 +24,8 @@ pub fn (db Connection) @select(config orm.SelectConfig, data orm.QueryData, wher
 	mut dataptr := []&u8{}
 
 	for i in 0 .. num_fields {
-		f := unsafe { fields[i] }
-		match unsafe { FieldType(f.@type) } {
+		field := unsafe { fields[i] }
+		match unsafe { FieldType(field.@type) } {
 			.type_tiny {
 				dataptr << unsafe { malloc(1) }
 			}
@@ -51,38 +47,40 @@ pub fn (db Connection) @select(config orm.SelectConfig, data orm.QueryData, wher
 			.type_time, .type_date, .type_datetime, .type_time2, .type_datetime2 {
 				dataptr << unsafe { malloc(sizeof(C.MYSQL_TIME)) }
 			}
-			.type_string, .type_blob {
-				dataptr << unsafe { malloc(512) }
-			}
-			.type_var_string {
-				dataptr << unsafe { malloc(2) }
+			.type_string, .type_var_string, .type_blob, .type_tiny_blob, .type_medium_blob,
+			.type_long_blob {
+				// Memory will be allocated later dynamically depending on the length of the value.
+				dataptr << &u8(0)
 			}
 			else {
-				return error('\'${unsafe { FieldType(f.@type) }}\' is not yet implemented. Please create a new issue at https://github.com/vlang/v/issues/new')
+				return error('\'${unsafe { FieldType(field.@type) }}\' is not yet implemented. Please create a new issue at https://github.com/vlang/v/issues/new')
 			}
 		}
 	}
 
-	lens := []u32{len: int(num_fields), init: 0}
-	stmt.bind_res(fields, dataptr, lens, num_fields)
+	lengths := []u32{len: int(num_fields), init: 0}
+	stmt.bind_res(fields, dataptr, lengths, num_fields)
 
-	mut row := 0
 	mut types := config.types.clone()
 	mut field_types := []FieldType{}
 	if config.is_count {
 		types = [orm.type_idx['u64']]
 	}
+	// Map stores column indexes and their binds in order to extract values
+	// for these columns separately, with individual memory allocation for each value.
+	mut string_binds_map := map[int]C.MYSQL_BIND{}
 
 	for i, mut mysql_bind in stmt.res {
-		f := unsafe { fields[i] }
-		field_types << unsafe { FieldType(f.@type) }
+		field := unsafe { fields[i] }
+		field_type := unsafe { FieldType(field.@type) }
+		field_types << field_type
+
 		match types[i] {
 			orm.type_string {
-				mysql_bind.buffer_type = C.MYSQL_TYPE_BLOB
-				mysql_bind.buffer_length = FieldType.type_blob.get_len()
+				string_binds_map[i] = mysql_bind
 			}
 			orm.time {
-				match unsafe { FieldType(f.@type) } {
+				match field_type {
 					.type_long {
 						mysql_bind.buffer_type = C.MYSQL_TYPE_LONG
 					}
@@ -92,7 +90,7 @@ pub fn (db Connection) @select(config orm.SelectConfig, data orm.QueryData, wher
 					}
 					.type_string, .type_blob {}
 					else {
-						return error('Unknown type ${f.@type}')
+						return error('Unknown type ${field.@type}')
 					}
 				}
 			}
@@ -102,13 +100,23 @@ pub fn (db Connection) @select(config orm.SelectConfig, data orm.QueryData, wher
 
 	stmt.bind_result_buffer()!
 	stmt.store_result()!
+
 	for {
 		status = stmt.fetch_stmt()!
 
 		if status == 1 || status == 100 {
 			break
 		}
-		row++
+
+		for index, mut bind in string_binds_map {
+			string_length := lengths[index] + 1
+			dataptr[index] = unsafe { malloc(string_length) }
+			bind.buffer = dataptr[index]
+			bind.buffer_length = string_length
+			bind.length = unsafe { nil }
+
+			stmt.fetch_column(bind, index)!
+		}
 
 		data_list := buffer_to_primitive(dataptr, types, field_types)!
 		ret << data_list

vlib/db/mysql/stmt.c.v

@@ -61,6 +61,7 @@ fn C.mysql_stmt_bind_result(&C.MYSQL_STMT, &C.MYSQL_BIND) bool
 fn C.mysql_stmt_fetch(&C.MYSQL_STMT) int
 fn C.mysql_stmt_next_result(&C.MYSQL_STMT) int
 fn C.mysql_stmt_store_result(&C.MYSQL_STMT) int
+fn C.mysql_stmt_fetch_column(&C.MYSQL_STMT, &C.MYSQL_BIND, u32, u64) int
 
 pub struct Stmt {
 	stmt  &C.MYSQL_STMT = &C.MYSQL_STMT(unsafe { nil })
@@ -248,14 +249,12 @@ pub fn (mut stmt Stmt) bind(typ int, buffer voidptr, buf_len u32) {
 }
 
 // bind_res will store one result in the statement `stmt`
-pub fn (mut stmt Stmt) bind_res(fields &C.MYSQL_FIELD, dataptr []&u8, lens []u32, num_fields int) {
+pub fn (mut stmt Stmt) bind_res(fields &C.MYSQL_FIELD, dataptr []&u8, lengths []u32, num_fields int) {
 	for i in 0 .. num_fields {
-		len := unsafe { FieldType(fields[i].@type).get_len() }
 		stmt.res << C.MYSQL_BIND{
 			buffer_type: unsafe { fields[i].@type }
 			buffer: dataptr[i]
-			length: &lens[i]
-			buffer_length: len
+			length: &lengths[i]
 		}
 	}
 }
@@ -283,3 +282,15 @@ pub fn (mut stmt Stmt) store_result() ! {
 		return stmt.error(res)
 	}
 }
+
+// fetch_column fetches one column from the current result set row.
+// `bind` provides the buffer where data should be placed.
+// It should be set up the same way as for `mysql_stmt_bind_result()`.
+// `column` indicates which column to fetch. The first column is numbered 0.
+pub fn (mut stmt Stmt) fetch_column(bind &C.MYSQL_BIND, column int) ! {
+	result := C.mysql_stmt_fetch_column(stmt.stmt, bind, column, 0)
+
+	if result != 0 && stmt.get_error_msg() != '' {
+		return stmt.error(result)
+	}
+}