orm: add null handling and option fields (#19379)

* orm: added is none and !is none handling

* orm: added NullType, support option fields and deprecate [nonull]

Nullable DB fields are now determined by corresponding option struct field.  The
[nonull] attribute is deprecated and fields are all NOT NULL now, unless they
are option fields. New orm primitive, NullType, added to support passing none
values to db backends, which have been updated to support it.  Also, empty
string and 0 numberic values are no longer skipped during insert (since they may
be valid values).

* orm: fix [nonull] deprecation warning

* orm: add null handling to update and select

also, improved formatting for orm cgen, and removed optimised operand handling
of orm `is` and `!is` operators

* sqlite: read/report NULLs using new orm NullType

* postgres: returning data primitives now returns new orm.NullType

* orm: initialise NullType Primitives properly

* orm: do not smart cast operands inside sql

* orm: fix bad setting of option value

* orm: improve orm_null_test.v, adding/fixing selects

* orm: cleanup: rename NullType->Null, use serial const, cgen output

* orm: handle automatically generated fields more explicitly

During insert, fields which are
* [sql: serial]
* [default: whatever]
and where the data is a default value (e.g., 0, ""), those fields are not sent
to the db, so that the db can generate auto-increment or default values.  (This
was previously done only for [primary] fields, and not in all circumstances, but
that is not correct -- primary and serial/auto-increment fields are differnet.)

* orm: udpated README

* orm: select cgen fixes: read from uninit res; fail to init res

* orm: udpated tests

* orm: fix option sub-struct fields

* orm: fixed joins to option structs

Changed orm.write_orm_select() so that you pass to it the name of a resut
variable which it populates with the result (or not) and changed use of it in
sql_select_expr() and calls in write_orm_select() to populate substructs.

* orm: fix pg driver handling of NULL results

* orm: move runtime checks to comptime checker; cache checked tables

* orm: vfmt :(

* orm: markdown formatting

* orm: renamed orm.time_ and orm.enum_; updated db drivers

* checker: updated orm tests

* orm: fix issue setting up ast option values as orm primitives

* checker: ORM use of none/options and operations (added tests)

* orm: fixed tests

* db: clean code

* examples: remove orm nonull attributes

* orm: skip test memory santisation for orm_null_test.v

* orm: make the type-to-primitive converstion fns not public

* orm: mv object var c-code from checker->cgen; fix memory corruption

Code in checker/orm.v used the SqlStmtLine object field name to store c-specific
referenecs to option and array fields (for arrays of children).  I moved this
logic to cgen.  And fixed an issue introduced with option fields, where an array
of children was unpacked into a non-array result which could corrupt memory.

* orm: fixed vast error

* orm: skip 2 tests on ubuntu-musl which require sqlite3.h

* cgen: prevent casting a struct (string)

* v fmt orm_fkey_attribute.vv, orm_multidim_array.vv, orm_table_attributes.vv; run `VAUTOFIX=1 ./v vlib/v/compiler_errors_test.v`

cmd/tools/vast/vast.v

@@ -1770,7 +1770,7 @@ fn (t Tree) sql_stmt_line(node ast.SqlStmtLine) &Node {
 	obj.add_terse('ast_type', t.string_node('SqlStmtLine'))
 	obj.add_terse('kind', t.enum_node(node.kind))
 	obj.add_terse('table_expr', t.type_expr(node.table_expr))
-	obj.add_terse('object_var_name', t.string_node(node.object_var_name))
+	obj.add_terse('object_var', t.string_node(node.object_var))
 	obj.add_terse('where_expr', t.expr(node.where_expr))
 	obj.add_terse('fields', t.array_node_struct_field(node.fields))
 	obj.add_terse('updated_columns', t.array_node_string(node.updated_columns))

cmd/tools/vtest-self.v

@@ -132,6 +132,7 @@ const (
 		'vlib/orm/orm_string_interpolation_in_where_test.v',
 		'vlib/orm/orm_interface_test.v',
 		'vlib/orm/orm_mut_db_test.v',
+		'vlib/orm/orm_null_test.v',
 		'vlib/orm/orm_result_test.v',
 		'vlib/orm/orm_custom_operators_test.v',
 		'vlib/orm/orm_fk_test.v',
@@ -214,6 +215,7 @@ const (
 		'vlib/orm/orm_insert_test.v',
 		'vlib/orm/orm_insert_reserved_name_test.v',
 		'vlib/orm/orm_fn_calls_test.v',
+		'vlib/orm/orm_null_test.v',
 		'vlib/orm/orm_last_id_test.v',
 		'vlib/orm/orm_string_interpolation_in_where_test.v',
 		'vlib/orm/orm_interface_test.v',

examples/vweb_fullstack/src/product_entities.v

@@ -4,6 +4,6 @@ module main
 struct Product {
 	id         int    [primary; sql: serial]
 	user_id    int
-	name       string [nonull; sql_type: 'TEXT']
+	name       string [sql_type: 'TEXT']
 	created_at string [default: 'CURRENT_TIMESTAMP']
 }

examples/vweb_fullstack/src/user_entities.v

@@ -4,8 +4,8 @@ module main
 pub struct User {
 mut:
 	id       int       [primary; sql: serial]
-	username string    [nonull; sql_type: 'TEXT'; unique]
-	password string    [nonull; sql_type: 'TEXT']
+	username string    [sql_type: 'TEXT'; unique]
+	password string    [sql_type: 'TEXT']
 	active   bool
 	products []Product [fkey: 'user_id']
 }

vlib/db/mysql/orm.v

@@ -75,7 +75,7 @@ pub fn (db DB) @select(config orm.SelectConfig, data orm.QueryData, where orm.Qu
 			orm.type_string {
 				string_binds_map[i] = mysql_bind
 			}
-			orm.time {
+			orm.time_ {
 				match field_type {
 					.type_long {
 						mysql_bind.buffer_type = C.MYSQL_TYPE_LONG
@@ -250,6 +250,9 @@ fn stmt_bind_primitive(mut stmt Stmt, data orm.Primitive) {
 		orm.InfixType {
 			stmt_bind_primitive(mut stmt, data.right)
 		}
+		orm.Null {
+			stmt.bind_null()
+		}
 	}
 }
 
@@ -297,7 +300,7 @@ fn data_pointers_to_primitives(data_pointers []&u8, types []int, field_types []F
 			orm.type_string {
 				primitive = unsafe { cstring_to_vstring(&char(data)) }
 			}
-			orm.time {
+			orm.time_ {
 				match field_types[i] {
 					.type_long {
 						timestamp := *(unsafe { &int(data) })
@@ -310,6 +313,9 @@ fn data_pointers_to_primitives(data_pointers []&u8, types []int, field_types []F
 					else {}
 				}
 			}
+			orm.enum_ {
+				primitive = *(unsafe { &i64(data) })
+			}
 			else {
 				return error('Unknown type ${types[i]}')
 			}
@@ -329,10 +335,10 @@ fn mysql_type_from_v(typ int) !string {
 		orm.type_idx['i16'], orm.type_idx['u16'] {
 			'SMALLINT'
 		}
-		orm.type_idx['int'], orm.type_idx['u32'], orm.time {
+		orm.type_idx['int'], orm.type_idx['u32'], orm.time_ {
 			'INT'
 		}
-		orm.type_idx['i64'], orm.type_idx['u64'] {
+		orm.type_idx['i64'], orm.type_idx['u64'], orm.enum_ {
 			'BIGINT'
 		}
 		orm.type_idx['f32'] {

vlib/db/mysql/stmt.c.v

@@ -247,6 +247,14 @@ pub fn (mut stmt Stmt) bind_text(b string) {
 	stmt.bind(mysql.mysql_type_string, b.str, u32(b.len))
 }
 
+// bind_null binds a single NULL value to the statement `stmt`
+pub fn (mut stmt Stmt) bind_null() {
+	stmt.binds << C.MYSQL_BIND{
+		buffer_type: mysql.mysql_type_null
+		length: 0
+	}
+}
+
 // bind binds a single value pointed by `buffer`, to the statement `stmt`. The buffer length must be passed as well in `buf_len`.
 // Note: it is more convenient to use one of the other bind_XYZ methods.
 pub fn (mut stmt Stmt) bind(typ int, buffer voidptr, buf_len u32) {

vlib/db/pg/orm.v

@@ -10,18 +10,17 @@ import net.conv
 pub fn (db DB) @select(config orm.SelectConfig, data orm.QueryData, where orm.QueryData) ![][]orm.Primitive {
 	query := orm.orm_select_gen(config, '"', true, '$', 1, where)
 
-	res := pg_stmt_worker(db, query, where, data)!
+	rows := pg_stmt_worker(db, query, where, data)!
 
 	mut ret := [][]orm.Primitive{}
 
 	if config.is_count {
 	}
 
-	for row in res {
+	for row in rows {
 		mut row_data := []orm.Primitive{}
 		for i, val in row.vals {
-			field := str_to_primitive(val, config.types[i])!
-			row_data << field
+			row_data << val_to_primitive(val, config.types[i])!
 		}
 		ret << row_data
 	}
@@ -189,6 +188,12 @@ fn pg_stmt_match(mut types []u32, mut vals []&char, mut lens []int, mut formats
 		orm.InfixType {
 			pg_stmt_match(mut types, mut vals, mut lens, mut formats, data.right)
 		}
+		orm.Null {
+			types << u32(0) // we do not know col type, let server infer
+			vals << &char(0) // NULL pointer indicates NULL
+			lens << int(0) // ignored
+			formats << 0 // ignored
+		}
 	}
 }
 
@@ -203,9 +208,12 @@ fn pg_type_from_v(typ int) !string {
 		orm.type_idx['int'], orm.type_idx['u32'] {
 			'INT'
 		}
-		orm.time {
+		orm.time_ {
 			'TIMESTAMP'
 		}
+		orm.enum_ {
+			'BIGINT'
+		}
 		orm.type_idx['i64'], orm.type_idx['u64'] {
 			'BIGINT'
 		}
@@ -231,69 +239,76 @@ fn pg_type_from_v(typ int) !string {
 	return str
 }
 
-fn str_to_primitive(str string, typ int) !orm.Primitive {
-	match typ {
-		// bool
-		orm.type_idx['bool'] {
-			return orm.Primitive(str == 't')
-		}
-		// i8
-		orm.type_idx['i8'] {
-			return orm.Primitive(str.i8())
-		}
-		// i16
-		orm.type_idx['i16'] {
-			return orm.Primitive(str.i16())
-		}
-		// int
-		orm.type_idx['int'] {
-			return orm.Primitive(str.int())
-		}
-		// i64
-		orm.type_idx['i64'] {
-			return orm.Primitive(str.i64())
-		}
-		// u8
-		orm.type_idx['u8'] {
-			data := str.i8()
-			return orm.Primitive(*unsafe { &u8(&data) })
-		}
-		// u16
-		orm.type_idx['u16'] {
-			data := str.i16()
-			return orm.Primitive(*unsafe { &u16(&data) })
-		}
-		// u32
-		orm.type_idx['u32'] {
-			data := str.int()
-			return orm.Primitive(*unsafe { &u32(&data) })
-		}
-		// u64
-		orm.type_idx['u64'] {
-			data := str.i64()
-			return orm.Primitive(*unsafe { &u64(&data) })
-		}
-		// f32
-		orm.type_idx['f32'] {
-			return orm.Primitive(str.f32())
-		}
-		// f64
-		orm.type_idx['f64'] {
-			return orm.Primitive(str.f64())
-		}
-		orm.type_string {
-			return orm.Primitive(str)
-		}
-		orm.time {
-			if str.contains_any(' /:-') {
-				date_time_str := time.parse(str)!
-				return orm.Primitive(date_time_str)
+fn val_to_primitive(val ?string, typ int) !orm.Primitive {
+	if str := val {
+		match typ {
+			// bool
+			orm.type_idx['bool'] {
+				return orm.Primitive(str == 't')
+			}
+			// i8
+			orm.type_idx['i8'] {
+				return orm.Primitive(str.i8())
+			}
+			// i16
+			orm.type_idx['i16'] {
+				return orm.Primitive(str.i16())
+			}
+			// int
+			orm.type_idx['int'] {
+				return orm.Primitive(str.int())
+			}
+			// i64
+			orm.type_idx['i64'] {
+				return orm.Primitive(str.i64())
+			}
+			// u8
+			orm.type_idx['u8'] {
+				data := str.i8()
+				return orm.Primitive(*unsafe { &u8(&data) })
+			}
+			// u16
+			orm.type_idx['u16'] {
+				data := str.i16()
+				return orm.Primitive(*unsafe { &u16(&data) })
 			}
+			// u32
+			orm.type_idx['u32'] {
+				data := str.int()
+				return orm.Primitive(*unsafe { &u32(&data) })
+			}
+			// u64
+			orm.type_idx['u64'] {
+				data := str.i64()
+				return orm.Primitive(*unsafe { &u64(&data) })
+			}
+			// f32
+			orm.type_idx['f32'] {
+				return orm.Primitive(str.f32())
+			}
+			// f64
+			orm.type_idx['f64'] {
+				return orm.Primitive(str.f64())
+			}
+			orm.type_string {
+				return orm.Primitive(str)
+			}
+			orm.time_ {
+				if str.contains_any(' /:-') {
+					date_time_str := time.parse(str)!
+					return orm.Primitive(date_time_str)
+				}
 
-			timestamp := str.int()
-			return orm.Primitive(time.unix(timestamp))
+				timestamp := str.int()
+				return orm.Primitive(time.unix(timestamp))
+			}
+			orm.enum_ {
+				return orm.Primitive(str.i64())
+			}
+			else {}
 		}
-		else {}
+		return error('Unknown field type ${typ}')
+	} else {
+		return orm.Null{}
 	}
-	return error('Unknown field type ${typ}')
 }

vlib/db/pg/pg.v

@@ -43,7 +43,7 @@ mut:
 
 pub struct Row {
 pub mut:
-	vals []string
+	vals []?string
 }
 
 pub struct Config {
@@ -110,6 +110,8 @@ fn C.PQexec(res &C.PGconn, const_query &char) &C.PGresult
 
 //
 
+fn C.PQgetisnull(const_res &C.PGresult, int, int) int
+
 fn C.PQgetvalue(const_res &C.PGresult, int, int) &char
 
 fn C.PQresultStatus(const_res &C.PGresult) int
@@ -179,9 +181,13 @@ fn res_to_rows(res voidptr) []Row {
 	for i in 0 .. nr_rows {
 		mut row := Row{}
 		for j in 0 .. nr_cols {
-			val := C.PQgetvalue(res, i, j)
-			sval := unsafe { val.vstring() }
-			row.vals << sval
+			if C.PQgetisnull(res, i, j) != 0 {
+				row.vals << none
+			} else {
+				val := C.PQgetvalue(res, i, j)
+				sval := unsafe { val.vstring() }
+				row.vals << sval
+			}
 		}
 		rows << row
 	}
@@ -209,7 +215,7 @@ pub fn (db DB) q_int(query string) !int {
 		return 0
 	}
 	val := row.vals[0]
-	return val.int()
+	return val or { '0' }.int()
 }
 
 // q_string submit a command to the database server and
@@ -226,7 +232,7 @@ pub fn (db DB) q_string(query string) !string {
 		return ''
 	}
 	val := row.vals[0]
-	return val
+	return val or { '' }
 }
 
 // q_strings submit a command to the database server and