Merge pull request #2267 from johnelse/cp-12398

CP-12398 - Add Intel GVT-g support

ocaml/idl/datamodel.ml

@@ -18,7 +18,7 @@ open Datamodel_types
 (* IMPORTANT: Please bump schema vsn if you change/add/remove a _field_.
               You do not have to bump vsn if you change/add/remove a message *)
 let schema_major_vsn = 5
-let schema_minor_vsn = 85
+let schema_minor_vsn = 86
 
 (* Historical schema versions just in case this is useful later *)
 let rio_schema_major_vsn = 5
@@ -67,7 +67,7 @@ let cream_release_schema_major_vsn = 5
 let cream_release_schema_minor_vsn = 73
 
 let dundee_release_schema_major_vsn = 5
-let dundee_release_schema_minor_vsn = 85
+let dundee_release_schema_minor_vsn = 86
 
 (* the schema vsn of the last release: used to determine whether we can upgrade or not.. *)
 let last_release_schema_major_vsn = creedence_release_schema_major_vsn
@@ -8262,6 +8262,18 @@ let vgpu =
 		~allowed_roles:_R_POOL_OP
 		()
 	in
+	let atomic_set_resident_on = call
+		~name:"atomic_set_resident_on"
+		~lifecycle:[Published, rel_dundee, ""]
+		~params:[
+			Ref _vgpu, "self", "The vGPU to modify";
+			Ref _pgpu, "value", "The pGPU on which the vGPU is running";
+		]
+		~allowed_roles:_R_LOCAL_ROOT_ONLY
+		~hide_from_docs:true
+		~pool_internal:true
+		()
+	in
 	create_obj
 		~name:_vgpu
 		~descr:"A virtual GPU (vGPU)"
@@ -8270,7 +8282,7 @@ let vgpu =
 		~gen_events:true
 		~in_db:true
 		~lifecycle:[Published, rel_boston, ""]
-		~messages:[create; destroy]
+		~messages:[create; destroy; atomic_set_resident_on]
 		~messages_default_allowed_roles:_R_POOL_OP
 		~persist:PersistEverything
 		~in_oss_since:None
@@ -8283,11 +8295,19 @@ let vgpu =
 			field ~qualifier:RW ~ty:(Map (String,String)) ~lifecycle:[Published, rel_boston, ""] "other_config" "Additional configuration" ~default_value:(Some (VMap []));
 			field ~qualifier:DynamicRO ~ty:(Ref _vgpu_type) ~lifecycle:[Published, rel_vgpu_tech_preview, ""] "type" "Preset type for this VGPU" ~default_value:(Some (VRef (Ref.string_of Ref.null)));
 			field ~qualifier:DynamicRO ~ty:(Ref _pgpu) ~lifecycle:[Published, rel_vgpu_tech_preview, ""] "resident_on" "The PGPU on which this VGPU is running" ~default_value:(Some (VRef (Ref.string_of Ref.null)));
+			field ~qualifier:DynamicRO ~ty:(Ref _pgpu) ~lifecycle:[Published, rel_dundee, ""] ~internal_only:true "scheduled_to_be_resident_on" "The PGPU on which this VGPU is scheduled to run" ~default_value:(Some (VRef (Ref.string_of Ref.null)));
 			]
 		()
 
 (** Virtual GPU types (i.e. preset sizes) *)
 
+let vgpu_type_implementation =
+	Enum ("vgpu_type_implementation", [
+		"passthrough", "Pass through an entire physical GPU to a guest";
+		"nvidia", "vGPU using NVIDIA hardware";
+		"gvt_g", "vGPU using Intel GVT-g";
+	])
+
 let vgpu_type =
 	create_obj
 		~name:_vgpu_type
@@ -8316,6 +8336,7 @@ let vgpu_type =
 			field ~qualifier:StaticRO ~ty:(Map (String, String)) ~lifecycle:[Published, rel_vgpu_tech_preview, ""] ~default_value:(Some (VMap [])) ~internal_only:true "internal_config" "Extra configuration information for internal use.";
 			field ~qualifier:DynamicRO ~ty:(Set (Ref _gpu_group)) ~lifecycle:[Published, rel_vgpu_productisation, ""] "supported_on_GPU_groups" "List of GPU groups in which at least one PGPU supports this VGPU type";
 			field ~qualifier:DynamicRO ~ty:(Set (Ref _gpu_group)) ~lifecycle:[Published, rel_vgpu_productisation, ""] "enabled_on_GPU_groups" "List of GPU groups in which at least one have this VGPU type enabled";
+			field ~qualifier:StaticRO ~ty:vgpu_type_implementation ~lifecycle:[Published, rel_dundee, ""] ~default_value:(Some (VEnum "passthrough")) "implementation" "The internal implementation of this VGPU type";
 		]
 	()
 

ocaml/test/test_common.ml

@@ -215,16 +215,17 @@ let make_gpu_group ~__context ?(ref=Ref.make ()) ?(uuid=make_uuid ())
 
 let make_vgpu ~__context ?(ref=Ref.make ()) ?(uuid=make_uuid ()) ?(vM=Ref.null)
 		?(gPU_group=Ref.null) ?(device="0") ?(currently_attached=false)
-		?(other_config=[]) ?(_type=Ref.null) ?(resident_on=Ref.null) () =
+		?(other_config=[]) ?(_type=Ref.null) ?(resident_on=Ref.null)
+		?(scheduled_to_be_resident_on=Ref.null) () =
 	Db.VGPU.create ~__context ~ref ~uuid ~vM ~gPU_group ~device ~currently_attached
-		~other_config ~_type ~resident_on;
+		~other_config ~_type ~resident_on ~scheduled_to_be_resident_on;
 	ref
 
 let make_vgpu_type ~__context ?(ref=Ref.make ()) ?(uuid=make_uuid ())
 		?(vendor_name="") ?(model_name="") ?(framebuffer_size=0L) ?(max_heads=0L)
 		?(max_resolution_x=0L) ?(max_resolution_y=0L) ?(size=0L)
-		?(internal_config=[]) () =
+		?(internal_config=[]) ?(implementation=`passthrough) () =
 	Db.VGPU_type.create ~__context ~ref ~uuid ~vendor_name ~model_name
 		~framebuffer_size ~max_heads ~max_resolution_x ~max_resolution_y ~size
-		~internal_config;
+		~internal_config ~implementation;
 	ref

ocaml/test/test_pgpu_helpers.ml

@@ -49,24 +49,24 @@ module GetRemainingCapacity = Generic.Make(Generic.EncapsulateState(struct
 		 * GPU type. *)
 		(default_k1, k100), 8L;
 		(default_k1, k140q), 4L;
-		(default_k1, entire_gpu), 1L;
+		(default_k1, passthrough_gpu), 1L;
 		(default_k2, k200), 8L;
 		(default_k2, k240q), 4L;
 		(default_k2, k260q), 2L;
-		(default_k2, entire_gpu), 1L;
+		(default_k2, passthrough_gpu), 1L;
 		(* Test that we can't mix VGPU types. *)
-		({default_k1 with resident_VGPU_types = [entire_gpu]}, k100), 0L;
+		({default_k1 with resident_VGPU_types = [passthrough_gpu]}, k100), 0L;
 		({default_k1 with resident_VGPU_types = [k100]}, k140q), 0L;
-		({default_k2 with resident_VGPU_types = [entire_gpu]}, k200), 0L;
+		({default_k2 with resident_VGPU_types = [passthrough_gpu]}, k200), 0L;
 		({default_k2 with resident_VGPU_types = [k260q]}, k200), 0L;
 		(* Test that remaining capacity values in other situations are correct. *)
 		({default_k1 with resident_VGPU_types = [k100; k100]}, k100), 6L;
 		({default_k1 with resident_VGPU_types = [k140q; k140q]}, k140q), 2L;
-		({default_k1 with resident_VGPU_types = [entire_gpu]}, entire_gpu), 0L;
+		({default_k1 with resident_VGPU_types = [passthrough_gpu]}, passthrough_gpu), 0L;
 		({default_k2 with resident_VGPU_types = [k200]}, k200), 7L;
 		({default_k2 with resident_VGPU_types = [k240q; k240q; k240q]}, k240q), 1L;
 		({default_k2 with resident_VGPU_types = [k260q]}, k260q), 1L;
-		({default_k2 with resident_VGPU_types = [entire_gpu]}, entire_gpu), 0L;
+		({default_k2 with resident_VGPU_types = [passthrough_gpu]}, passthrough_gpu), 0L;
 	]
 end))
 

ocaml/test/test_vgpu_common.ml

@@ -23,6 +23,7 @@ let k100 = {
 	max_resolution_y = 1200L;
 	size = Int64.div Constants.pgpu_default_size 8L;
 	internal_config = [];
+	implementation = `nvidia;
 }
 
 let k140q = {
@@ -34,6 +35,7 @@ let k140q = {
 	max_resolution_y = 1600L;
 	size = Int64.div Constants.pgpu_default_size 4L;
 	internal_config = [];
+	implementation = `nvidia;
 }
 
 let k200 = {
@@ -45,6 +47,7 @@ let k200 = {
 	max_resolution_y = 1200L;
 	size = Int64.div Constants.pgpu_default_size 8L;
 	internal_config = [];
+	implementation = `nvidia;
 }
 
 let k240q = {
@@ -56,6 +59,7 @@ let k240q = {
 	max_resolution_y = 1600L;
 	size = Int64.div Constants.pgpu_default_size 4L;
 	internal_config = [];
+	implementation = `nvidia;
 }
 
 let k260q = {
@@ -67,19 +71,20 @@ let k260q = {
 	max_resolution_y = 1600L;
 	size = Int64.div Constants.pgpu_default_size 2L;
 	internal_config = [];
+	implementation = `nvidia;
 }
 
 let k1_vgpu_types = [
 	k100;
 	k140q;
-	entire_gpu;
+	passthrough_gpu;
 ]
 
 let k2_vgpu_types = [
 	k200;
 	k240q;
 	k260q;
-	entire_gpu;
+	passthrough_gpu;
 ]
 
 (* Represents the state of a PGPU, its supported and enabled VGPU types, and

ocaml/test/test_vgpu_type.ml

@@ -19,6 +19,7 @@ open Test_vgpu_common
 open Xapi_vgpu_type
 
 let string_of_vgpu_conf conf =
+	let open Nvidia in
 	Printf.sprintf "%04x %s %04x %04x %Ld"
 		conf.pdev_id
 		(match conf.psubdev_id with
@@ -34,18 +35,18 @@ let print_vgpu_conf conf =
 module OfConfFile = Generic.Make(struct
 	module Io = struct
 		type input_t = string
-		type output_t = vgpu_conf
+		type output_t = Nvidia.vgpu_conf
 
 		let string_of_input_t x = x
 		let string_of_output_t = string_of_vgpu_conf
 	end
 
-	let transform = of_conf_file
+	let transform = Nvidia.of_conf_file
 
 	let tests = [
 		"ocaml/test/data/test_vgpu_subdevid.conf",
 		{
-			pdev_id = 0x3333;
+			Nvidia.pdev_id = 0x3333;
 			psubdev_id = Some 0x4444;
 			vdev_id = 0x1111;
 			vsubdev_id = 0x2222;
@@ -58,7 +59,7 @@ module OfConfFile = Generic.Make(struct
 		};
 		"ocaml/test/data/test_vgpu_nosubdevid.conf",
 		{
-			pdev_id = 0x3333;
+			Nvidia.pdev_id = 0x3333;
 			psubdev_id = None;
 			vdev_id = 0x1111;
 			vsubdev_id = 0x2222;
@@ -78,6 +79,7 @@ let skip = true
 let print_nv_types () =
 	skip_if skip "Generates print...";
 	try
+		let open Nvidia in
 		if (Sys.file_exists nvidia_conf_dir
 			&& Sys.is_directory nvidia_conf_dir) then
 			begin

ocaml/test/test_vm_helpers.ml

@@ -80,7 +80,7 @@ let test_gpus_available_fails_no_capacity () =
 		let pgpus = Db.GPU_group.get_PGPUs ~__context ~self:group in
 		(* Fill up all the PGPUs *)
 		List.iter (fun p ->
-			ignore (make_vgpu ~__context p Xapi_vgpu_type.entire_gpu))
+			ignore (make_vgpu ~__context p Xapi_vgpu_type.passthrough_gpu))
 			pgpus;
 		let vm = make_vm_with_vgpu_in_group ~__context k100 group in
 		assert_raises_api_error Api_errors.vm_requires_gpu

ocaml/test/test_xenopsd_metadata.ml

@@ -38,11 +38,13 @@ let load_vm_config __context conf =
 		()
 	in ()
 
-let run_builder_of_vm __context =
+let run_create_metadata ~__context =
+	let localhost_uuid = Helpers.get_localhost_uuid () in
+	let host = make_host ~__context ~uuid:localhost_uuid () in
+	let (_: API.ref_pool) = make_pool ~__context ~master:host () in
 	let vms = Db.VM.get_by_name_label ~__context ~label:test_vm_name in
 	let vm = List.nth vms 0 in
-	let vm_record = Db.VM.get_record ~__context ~self:vm in
-	Xapi_xenops.builder_of_vm ~__context (vm, vm_record) "0" false
+	Xapi_xenops.create_metadata ~__context ~upgrade:false ~self:vm
 
 (* Test the behaviour of the "hvm_serial" other_config/platform key. *)
 module HVMSerial = Generic.Make(Generic.EncapsulateState(struct
@@ -61,7 +63,8 @@ module HVMSerial = Generic.Make(Generic.EncapsulateState(struct
 	let load_input = load_vm_config
 
 	let extract_output __context _ =
-		match run_builder_of_vm __context with
+		let metadata = run_create_metadata ~__context in
+		match metadata.Metadata.vm.Vm.ty with
 		| Vm.HVM {Vm.serial = serial} -> serial
 		| _ -> failwith "expected HVM metadata"
 
@@ -112,10 +115,12 @@ module HVMSerial = Generic.Make(Generic.EncapsulateState(struct
 		]
 end))
 
-let vgpu_pci_id = "vgpu_pci_id", "0000:0a:00.0"
-let vgpu_config = "vgpu_config", "/usr/share/nvidia/vgx/grid_k100.conf"
+let vgpu_manual_setup = Xapi_globs.vgpu_manual_setup_key, "true"
+let vgpu_pci_id = Xapi_globs.vgpu_pci_key, "0000:0a:00.0"
+let vgpu_config =
+	Xapi_globs.vgpu_config_key, "/usr/share/nvidia/vgx/grid_k100.conf"
 
-let vgpu_platform_data = [vgpu_pci_id; vgpu_config]
+let vgpu_platform_data = [vgpu_manual_setup; vgpu_pci_id; vgpu_config]
 
 module VideoMode = Generic.Make(Generic.EncapsulateState(struct
 	module Io = struct
@@ -135,7 +140,8 @@ module VideoMode = Generic.Make(Generic.EncapsulateState(struct
 	let load_input = load_vm_config
 
 	let extract_output __context _ =
-		match run_builder_of_vm __context with
+		let metadata = run_create_metadata ~__context in
+		match metadata.Metadata.vm.Vm.ty with
 		| Vm.HVM {Vm.video = video_mode} -> video_mode
 		| _ -> failwith "expected HVM metadata"
 
@@ -158,20 +164,32 @@ module VideoMode = Generic.Make(Generic.EncapsulateState(struct
 			oc=[];
 			platform=["igd_passthrough", "true"; "vga", "std"]
 		}, Vm.(IGD_passthrough GVT_d);
-		(* We should be able to enable vGPU mode. *)
+		(* We should be able to enable vGPU via the manual setup mode. *)
 		{oc=[]; platform=vgpu_platform_data}, Vm.Vgpu;
 		(* vGPU mode should override whatever's set for the "vga" key. *)
 		{oc=[]; platform=["vga", "cirrus"] @ vgpu_platform_data}, Vm.Vgpu;
 		{oc=[]; platform=["vga", "std"] @ vgpu_platform_data}, Vm.Vgpu;
 		(* If somehow only one of the vGPU keys is set, this shouldn't
 		 * trigger vGPU mode. This should only ever happen if a user is
-		 * experimenting with vgpu_manual_setup. *)
-		{oc=[]; platform=[vgpu_pci_id]}, Vm.Cirrus;
-		{oc=[]; platform=["vga", "cirrus"; vgpu_pci_id]}, Vm.Cirrus;
-		{oc=[]; platform=["vga", "std"; vgpu_pci_id]}, Vm.Standard_VGA;
-		{oc=[]; platform=[vgpu_config]}, Vm.Cirrus;
-		{oc=[]; platform=["vga", "cirrus"; vgpu_config]}, Vm.Cirrus;
-		{oc=[]; platform=["vga", "std"; vgpu_config]}, Vm.Standard_VGA;
+		 * experimenting with vgpu_manual_setup and has got things wrong. *)
+		{oc=[]; platform=[vgpu_manual_setup; vgpu_pci_id]}, Vm.Cirrus;
+		{
+			oc=[];
+			platform=["vga", "cirrus"; vgpu_manual_setup; vgpu_pci_id]
+		}, Vm.Cirrus;
+		{
+			oc=[];
+			platform=["vga", "std"; vgpu_manual_setup; vgpu_pci_id]
+		}, Vm.Standard_VGA;
+		{oc=[]; platform=[vgpu_manual_setup; vgpu_config]}, Vm.Cirrus;
+		{
+			oc=[];
+			platform=["vga", "cirrus"; vgpu_manual_setup; vgpu_config]
+		}, Vm.Cirrus;
+		{
+			oc=[];
+			platform=["vga", "std"; vgpu_manual_setup; vgpu_config]
+		}, Vm.Standard_VGA;
 	]
 end))
 
@@ -189,7 +207,8 @@ module VideoRam = Generic.Make(Generic.EncapsulateState(struct
 	let load_input = load_vm_config
 
 	let extract_output __context _ =
-		match run_builder_of_vm __context with
+		let metadata = run_create_metadata ~__context in
+		match metadata.Metadata.vm.Vm.ty with
 		| Vm.HVM {Vm.video_mib = video_mib} -> video_mib
 		| _ -> failwith "expected HVM metadata"
 

ocaml/xapi/import.ml

@@ -1062,7 +1062,8 @@ module VGPUType : HandlerTools = struct
 							~max_resolution_x:value.API.vGPU_type_max_resolution_x
 							~max_resolution_y:value.API.vGPU_type_max_resolution_y
 							~size:0L
-							~internal_config:[])
+							~internal_config:[]
+							~implementation:value.API.vGPU_type_implementation)
 					vgpu_type_record
 			in
 			state.cleanup <- (fun __context rpc session_id -> Db.VGPU_type.destroy __context vgpu_type) :: state.cleanup;

ocaml/xapi/message_forwarding.ml

@@ -3847,6 +3847,11 @@ module Forward = functor(Local: Custom_actions.CUSTOM_ACTIONS) -> struct
 		let destroy ~__context ~self =
 			info "VGPU.destroy: VGPU = '%s'" (vgpu_uuid ~__context self);
 			Local.VGPU.destroy ~__context ~self
+
+		let atomic_set_resident_on ~__context ~self ~value =
+			info "VGPU.atomic_set_resident_on: VGPU = '%s'; PGPU = '%s'"
+				(vgpu_uuid ~__context self) (pgpu_uuid ~__context value);
+			Local.VGPU.atomic_set_resident_on ~__context ~self ~value
 	end
 
 	module VGPU_type = struct end

ocaml/xapi/vgpuops.ml

@@ -142,27 +142,15 @@ let create_virtual_vgpu ~__context vm vgpu =
 				Ref.string_of vgpu.type_ref
 			]))
 		| Some pgpu ->
-			Db.VGPU.set_resident_on ~__context ~self:vgpu.vgpu_ref ~value:pgpu;
-			Db.PGPU.get_PCI ~__context ~self:pgpu
+			Db.VGPU.set_scheduled_to_be_resident_on ~__context
+				~self:vgpu.vgpu_ref ~value:pgpu;
 	)
 
 let add_vgpus_to_vm ~__context vm vgpus =
-	(* Update VM platform for xenops to use *)
-	List.iter
-		(fun key ->
-			try Db.VM.remove_from_platform ~__context ~self:vm ~key with _ -> ())
-		[Xapi_globs.vgpu_pci_key; Xapi_globs.vgpu_config_key];
 	(* Only support a maximum of one virtual GPU per VM for now. *)
 	match vgpus with
 	| [] -> ()
-	| vgpu :: _ ->
-		let vgpu_type = Db.VGPU_type.get_record_internal ~__context ~self:vgpu.type_ref in
-		let internal_config = vgpu_type.Db_actions.vGPU_type_internal_config in
-		let config_path = List.assoc Xapi_globs.vgpu_config_key internal_config in
-		let vgpu_pci = create_virtual_vgpu ~__context vm vgpu in
-		let pci_id = Db.PCI.get_pci_id ~__context ~self:vgpu_pci in
-		Db.VM.add_to_platform ~__context ~self:vm ~key:Xapi_globs.vgpu_config_key ~value:config_path;
-		Db.VM.add_to_platform ~__context ~self:vm ~key:Xapi_globs.vgpu_pci_key ~value:pci_id
+	| vgpu :: _ -> create_virtual_vgpu ~__context vm vgpu
 
 let vgpu_manual_setup_of_vm vm_r =
 	List.mem_assoc Xapi_globs.vgpu_manual_setup_key vm_r.API.vM_platform &&