From f73518de906c1b0dfed5fa30d4d988a30bea684e Mon Sep 17 00:00:00 2001 From: Serban Iorga Date: Thu, 11 Apr 2019 12:24:40 +0300 Subject: [PATCH 1/6] AMD: process leaf 0x7 only if index == 0 per AMD EPYC PPR specifications Signed-off-by: Serban Iorga --- cpuid/src/transformer/amd.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/cpuid/src/transformer/amd.rs b/cpuid/src/transformer/amd.rs index f9225d1ac91..bc59549dc0a 100644 --- a/cpuid/src/transformer/amd.rs +++ b/cpuid/src/transformer/amd.rs @@ -16,8 +16,12 @@ pub fn update_structured_extended_entry( ) -> Result<(), Error> { use cpu_leaf::leaf_0x7::index0::*; - // KVM sets this bit no matter what but this feature is not supported by hardware - entry.edx.write_bit(edx::ARCH_CAPABILITIES_BITINDEX, false); + // according to the EPYC PPR, only the leaf 0x7 with index 0 contains the + // structured extended feature identifiers + if entry.index == 0 { + // KVM sets this bit no matter what but this feature is not supported by hardware + entry.edx.write_bit(edx::ARCH_CAPABILITIES_BITINDEX, false); + } Ok(()) } @@ -83,6 +87,7 @@ mod test { fn test_update_structured_extended_entry() { use cpu_leaf::leaf_0x7::index0::*; + // Check that if index == 0 the entry is processed let vm_spec = VmSpec::new(VENDOR_ID_AMD, 0, 1, false); let mut entry = &mut kvm_cpuid_entry2 { function: leaf_0x7::LEAF_NUM, @@ -94,9 +99,13 @@ mod test { edx: *(0 as u32).write_bit(edx::ARCH_CAPABILITIES_BITINDEX, true), padding: [0, 0, 0], }; - assert!(update_structured_extended_entry(&mut entry, &vm_spec).is_ok()); - assert_eq!(entry.edx.read_bit(edx::ARCH_CAPABILITIES_BITINDEX), false); + + // Check that if index != 0 the entry is not processed + entry.index = 1; + entry.edx.write_bit(edx::ARCH_CAPABILITIES_BITINDEX, true); + assert!(update_structured_extended_entry(&mut entry, &vm_spec).is_ok()); + assert_eq!(entry.edx.read_bit(edx::ARCH_CAPABILITIES_BITINDEX), true); } } From 8bee3a1acbecd9e6a585a0c382e4fdae428719e4 Mon Sep 17 00:00:00 2001 From: Serban Iorga Date: Mon, 8 Apr 2019 18:35:08 +0300 Subject: [PATCH 2/6] AMD: fix the value of the largest extended fn KVM sets the largest extended function to 0x80000000. We have to change it to 0x8000001f since we also use the leaf 0x8000001d. Signed-off-by: Serban Iorga --- cpuid/src/cpu_leaf.rs | 14 +++++-- cpuid/src/transformer/amd.rs | 79 ++++++++++++++++++++++++++++-------- 2 files changed, 72 insertions(+), 21 deletions(-) diff --git a/cpuid/src/cpu_leaf.rs b/cpuid/src/cpu_leaf.rs index 5350e5aaccd..328d0d65c1c 100644 --- a/cpuid/src/cpu_leaf.rs +++ b/cpuid/src/cpu_leaf.rs @@ -173,10 +173,6 @@ pub mod leaf_0x7 { } } -pub mod leaf_0x80000000 { - pub const LEAF_NUM: u32 = 0x8000_0000; -} - pub mod leaf_0x80000001 { pub const LEAF_NUM: u32 = 0x8000_0001; @@ -225,3 +221,13 @@ pub mod leaf_0xb { pub const LEVEL_NUMBER_BITRANGE: BitRange = bit_range!(7, 0); } } + +pub mod leaf_0x80000000 { + pub const LEAF_NUM: u32 = 0x8000_0000; + + pub mod eax { + use bit_helper::BitRange; + + pub const LARGEST_EXTENDED_FN_BITRANGE: BitRange = bit_range!(31, 0); + } +} diff --git a/cpuid/src/transformer/amd.rs b/cpuid/src/transformer/amd.rs index bc59549dc0a..8c980f77ce2 100644 --- a/cpuid/src/transformer/amd.rs +++ b/cpuid/src/transformer/amd.rs @@ -10,6 +10,9 @@ use bit_helper::BitHelper; use cpu_leaf::*; use transformer::common::use_host_cpuid_function; +// Largest extended function. It has to be larger then 0x8000001d (Extended Cache Topology). +const LARGEST_EXTENDED_FN: u32 = 0x8000_001f; + pub fn update_structured_extended_entry( entry: &mut kvm_cpuid_entry2, _vm_spec: &VmSpec, @@ -26,6 +29,21 @@ pub fn update_structured_extended_entry( Ok(()) } +pub fn update_largest_extended_fn_entry( + entry: &mut kvm_cpuid_entry2, + _vm_spec: &VmSpec, +) -> Result<(), Error> { + use cpu_leaf::leaf_0x80000000::*; + + // KVM sets the largest extended function to 0x80000000. Change it to 0x8000001f + // Since we also use the leaf 0x8000001d (Extended Cache Topology). + entry + .eax + .write_bits_in_range(&eax::LARGEST_EXTENDED_FN_BITRANGE, LARGEST_EXTENDED_FN); + + Ok(()) +} + pub fn update_extended_cache_topology_entry( entry: &mut kvm_cpuid_entry2, vm_spec: &VmSpec, @@ -46,6 +64,7 @@ impl CpuidTransformer for AmdCpuidTransformer { let maybe_transformer_fn: Option = match entry.function { leaf_0x1::LEAF_NUM => Some(common::update_feature_info_entry), leaf_0x7::LEAF_NUM => Some(amd::update_structured_extended_entry), + leaf_0x80000000::LEAF_NUM => Some(amd::update_largest_extended_fn_entry), leaf_0x8000001d::LEAF_NUM => Some(amd::update_extended_cache_topology_entry), 0x8000_0002..=0x8000_0004 => Some(common::update_brand_string_entry), _ => None, @@ -65,10 +84,38 @@ mod test { use common::VENDOR_ID_AMD; #[test] - fn test_update_extended_cache_topology_entry() { + fn test_update_structured_extended_entry() { + use cpu_leaf::leaf_0x7::index0::*; + + // Check that if index == 0 the entry is processed let vm_spec = VmSpec::new(VENDOR_ID_AMD, 0, 1, false); let mut entry = &mut kvm_cpuid_entry2 { - function: leaf_0x8000001d::LEAF_NUM, + function: leaf_0x7::LEAF_NUM, + index: 0, + flags: 0, + eax: 0, + ebx: 0, + ecx: 0, + edx: *(0 as u32).write_bit(edx::ARCH_CAPABILITIES_BITINDEX, true), + padding: [0, 0, 0], + }; + assert!(update_structured_extended_entry(&mut entry, &vm_spec).is_ok()); + assert_eq!(entry.edx.read_bit(edx::ARCH_CAPABILITIES_BITINDEX), false); + + // Check that if index != 0 the entry is not processed + entry.index = 1; + entry.edx.write_bit(edx::ARCH_CAPABILITIES_BITINDEX, true); + assert!(update_structured_extended_entry(&mut entry, &vm_spec).is_ok()); + assert_eq!(entry.edx.read_bit(edx::ARCH_CAPABILITIES_BITINDEX), true); + } + + #[test] + fn test_update_largest_extended_fn_entry() { + use cpu_leaf::leaf_0x80000000::*; + + let vm_spec = VmSpec::new(VENDOR_ID_AMD, 0, 1, false); + let mut entry = &mut kvm_cpuid_entry2 { + function: LEAF_NUM, index: 0, flags: 0, eax: 0, @@ -78,34 +125,32 @@ mod test { padding: [0, 0, 0], }; - assert!(update_extended_cache_topology_entry(&mut entry, &vm_spec).is_ok()); + assert!(update_largest_extended_fn_entry(&mut entry, &vm_spec).is_ok()); - assert_eq!(entry.flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX, 1); + assert_eq!( + entry + .eax + .read_bits_in_range(&eax::LARGEST_EXTENDED_FN_BITRANGE), + LARGEST_EXTENDED_FN + ); } #[test] - fn test_update_structured_extended_entry() { - use cpu_leaf::leaf_0x7::index0::*; - - // Check that if index == 0 the entry is processed + fn test_update_extended_cache_topology_entry() { let vm_spec = VmSpec::new(VENDOR_ID_AMD, 0, 1, false); let mut entry = &mut kvm_cpuid_entry2 { - function: leaf_0x7::LEAF_NUM, + function: leaf_0x8000001d::LEAF_NUM, index: 0, flags: 0, eax: 0, ebx: 0, ecx: 0, - edx: *(0 as u32).write_bit(edx::ARCH_CAPABILITIES_BITINDEX, true), + edx: 0, padding: [0, 0, 0], }; - assert!(update_structured_extended_entry(&mut entry, &vm_spec).is_ok()); - assert_eq!(entry.edx.read_bit(edx::ARCH_CAPABILITIES_BITINDEX), false); - // Check that if index != 0 the entry is not processed - entry.index = 1; - entry.edx.write_bit(edx::ARCH_CAPABILITIES_BITINDEX, true); - assert!(update_structured_extended_entry(&mut entry, &vm_spec).is_ok()); - assert_eq!(entry.edx.read_bit(edx::ARCH_CAPABILITIES_BITINDEX), true); + assert!(update_extended_cache_topology_entry(&mut entry, &vm_spec).is_ok()); + + assert_eq!(entry.flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX, 1); } } From 6188faa7c179f3ce8cdf8b2accc55f4aa590637d Mon Sep 17 00:00:00 2001 From: Serban Iorga Date: Mon, 8 Apr 2019 18:38:15 +0300 Subject: [PATCH 3/6] AMD: enable topology extension support We need to enable it since we use the Extended Cache Topology leaf. Signed-off-by: Serban Iorga --- cpuid/src/cpu_leaf.rs | 27 ++++++++++++++------------- cpuid/src/transformer/amd.rs | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 13 deletions(-) diff --git a/cpuid/src/cpu_leaf.rs b/cpuid/src/cpu_leaf.rs index 328d0d65c1c..60494791de8 100644 --- a/cpuid/src/cpu_leaf.rs +++ b/cpuid/src/cpu_leaf.rs @@ -173,19 +173,6 @@ pub mod leaf_0x7 { } } -pub mod leaf_0x80000001 { - pub const LEAF_NUM: u32 = 0x8000_0001; - - pub mod ecx { - pub const PREFETCH_SHIFT: u32 = 8; // 3DNow! PREFETCH/PREFETCHW instructions - pub const LZCNT_SHIFT: u32 = 5; // advanced bit manipulation - } - - pub mod edx { - pub const PDPE1GB_SHIFT: u32 = 26; // 1-GByte pages are available if 1. - } -} - pub mod leaf_0xa { pub const LEAF_NUM: u32 = 0xa; } @@ -231,3 +218,17 @@ pub mod leaf_0x80000000 { pub const LARGEST_EXTENDED_FN_BITRANGE: BitRange = bit_range!(31, 0); } } + +pub mod leaf_0x80000001 { + pub const LEAF_NUM: u32 = 0x8000_0001; + + pub mod ecx { + pub const LZCNT_SHIFT: u32 = 5; // advanced bit manipulation + pub const PREFETCH_SHIFT: u32 = 8; // 3DNow! PREFETCH/PREFETCHW instructions + pub const TOPOEXT_INDEX: u32 = 22; + } + + pub mod edx { + pub const PDPE1GB_SHIFT: u32 = 26; // 1-GByte pages are available if 1. + } +} diff --git a/cpuid/src/transformer/amd.rs b/cpuid/src/transformer/amd.rs index 8c980f77ce2..87274afa275 100644 --- a/cpuid/src/transformer/amd.rs +++ b/cpuid/src/transformer/amd.rs @@ -44,6 +44,18 @@ pub fn update_largest_extended_fn_entry( Ok(()) } +pub fn update_extended_feature_info_entry( + entry: &mut kvm_cpuid_entry2, + _vm_spec: &VmSpec, +) -> Result<(), Error> { + use cpu_leaf::leaf_0x80000001::*; + + // set the Topology Extension bit since we use the Extended Cache Topology leaf + entry.ecx.write_bit(ecx::TOPOEXT_INDEX, true); + + Ok(()) +} + pub fn update_extended_cache_topology_entry( entry: &mut kvm_cpuid_entry2, vm_spec: &VmSpec, @@ -65,6 +77,7 @@ impl CpuidTransformer for AmdCpuidTransformer { leaf_0x1::LEAF_NUM => Some(common::update_feature_info_entry), leaf_0x7::LEAF_NUM => Some(amd::update_structured_extended_entry), leaf_0x80000000::LEAF_NUM => Some(amd::update_largest_extended_fn_entry), + leaf_0x80000001::LEAF_NUM => Some(amd::update_extended_feature_info_entry), leaf_0x8000001d::LEAF_NUM => Some(amd::update_extended_cache_topology_entry), 0x8000_0002..=0x8000_0004 => Some(common::update_brand_string_entry), _ => None, @@ -135,6 +148,27 @@ mod test { ); } + #[test] + fn test_update_extended_feature_info_entry() { + use cpu_leaf::leaf_0x80000001::*; + + let vm_spec = VmSpec::new(VENDOR_ID_AMD, 0, 1, false); + let mut entry = &mut kvm_cpuid_entry2 { + function: LEAF_NUM, + index: 0, + flags: 0, + eax: 0, + ebx: 0, + ecx: 0, + edx: 0, + padding: [0, 0, 0], + }; + + assert!(update_extended_feature_info_entry(&mut entry, &vm_spec).is_ok()); + + assert_eq!(entry.ecx.read_bit(ecx::TOPOEXT_INDEX), true); + } + #[test] fn test_update_extended_cache_topology_entry() { let vm_spec = VmSpec::new(VENDOR_ID_AMD, 0, 1, false); From bdac38916de53b74ca553a68e9cdc5c93feea654 Mon Sep 17 00:00:00 2001 From: Serban Iorga Date: Mon, 8 Apr 2019 18:41:49 +0300 Subject: [PATCH 4/6] AMD: set the total number of threads We don't support more then 64 threads right now. It's safe to put them all on the same processor. Signed-off-by: Serban Iorga --- cpuid/src/cpu_leaf.rs | 18 ++++++++++ cpuid/src/transformer/amd.rs | 67 ++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/cpuid/src/cpu_leaf.rs b/cpuid/src/cpu_leaf.rs index 60494791de8..f4d8c578f8c 100644 --- a/cpuid/src/cpu_leaf.rs +++ b/cpuid/src/cpu_leaf.rs @@ -232,3 +232,21 @@ pub mod leaf_0x80000001 { pub const PDPE1GB_SHIFT: u32 = 26; // 1-GByte pages are available if 1. } } + +pub mod leaf_0x80000008 { + pub const LEAF_NUM: u32 = 0x8000_0008; + + pub mod ecx { + use bit_helper::BitRange; + + // The number of bits in the initial ApicId value that indicate thread ID within a package + // Possible values: + // 0-3 -> Reserved + // 4 -> 1 Die, up to 16 threads + // 5 -> 2 Die, up to 32 threads + // 6 -> 3,4 Die, up to 64 threads + pub const THREAD_ID_SIZE_BITRANGE: BitRange = bit_range!(15, 12); + // The number of threads in the package - 1 + pub const NUM_THREADS_BITRANGE: BitRange = bit_range!(7, 0); + } +} diff --git a/cpuid/src/transformer/amd.rs b/cpuid/src/transformer/amd.rs index 87274afa275..20c6a061621 100644 --- a/cpuid/src/transformer/amd.rs +++ b/cpuid/src/transformer/amd.rs @@ -12,6 +12,9 @@ use transformer::common::use_host_cpuid_function; // Largest extended function. It has to be larger then 0x8000001d (Extended Cache Topology). const LARGEST_EXTENDED_FN: u32 = 0x8000_001f; +// This value allows at most 64 logical threads within a package. +// See also the documentation for leaf_0x80000008::ecx::THREAD_ID_SIZE_BITRANGE +const THREAD_ID_MAX_SIZE: u32 = 6; pub fn update_structured_extended_entry( entry: &mut kvm_cpuid_entry2, @@ -56,6 +59,22 @@ pub fn update_extended_feature_info_entry( Ok(()) } +pub fn update_amd_features_entry( + entry: &mut kvm_cpuid_entry2, + vm_spec: &VmSpec, +) -> Result<(), Error> { + use cpu_leaf::leaf_0x80000008::*; + + // We don't support more then 64 threads right now. + // It's safe to put them all on the same processor. + entry + .ecx + .write_bits_in_range(&ecx::THREAD_ID_SIZE_BITRANGE, THREAD_ID_MAX_SIZE) + .write_bits_in_range(&ecx::NUM_THREADS_BITRANGE, u32::from(vm_spec.cpu_count - 1)); + + Ok(()) +} + pub fn update_extended_cache_topology_entry( entry: &mut kvm_cpuid_entry2, vm_spec: &VmSpec, @@ -78,6 +97,7 @@ impl CpuidTransformer for AmdCpuidTransformer { leaf_0x7::LEAF_NUM => Some(amd::update_structured_extended_entry), leaf_0x80000000::LEAF_NUM => Some(amd::update_largest_extended_fn_entry), leaf_0x80000001::LEAF_NUM => Some(amd::update_extended_feature_info_entry), + leaf_0x80000008::LEAF_NUM => Some(amd::update_amd_features_entry), leaf_0x8000001d::LEAF_NUM => Some(amd::update_extended_cache_topology_entry), 0x8000_0002..=0x8000_0004 => Some(common::update_brand_string_entry), _ => None, @@ -169,6 +189,33 @@ mod test { assert_eq!(entry.ecx.read_bit(ecx::TOPOEXT_INDEX), true); } + fn check_update_amd_features_entry(cpu_count: u8, ht_enabled: bool) { + use cpu_leaf::leaf_0x80000008::*; + + let vm_spec = VmSpec::new(VENDOR_ID_AMD, 0, cpu_count, ht_enabled); + let mut entry = &mut kvm_cpuid_entry2 { + function: LEAF_NUM, + index: 0, + flags: 0, + eax: 0, + ebx: 0, + ecx: 0, + edx: 0, + padding: [0, 0, 0], + }; + + assert!(update_amd_features_entry(&mut entry, &vm_spec).is_ok()); + + assert_eq!( + entry.ecx.read_bits_in_range(&ecx::NUM_THREADS_BITRANGE), + u32::from(cpu_count - 1) + ); + assert_eq!( + entry.ecx.read_bits_in_range(&ecx::THREAD_ID_SIZE_BITRANGE), + THREAD_ID_MAX_SIZE + ); + } + #[test] fn test_update_extended_cache_topology_entry() { let vm_spec = VmSpec::new(VENDOR_ID_AMD, 0, 1, false); @@ -187,4 +234,24 @@ mod test { assert_eq!(entry.flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX, 1); } + + #[test] + fn test_1vcpu_ht_off() { + check_update_amd_features_entry(1, false); + } + + #[test] + fn test_1vcpu_ht_on() { + check_update_amd_features_entry(1, true); + } + + #[test] + fn test_2vcpu_ht_off() { + check_update_amd_features_entry(2, false); + } + + #[test] + fn test_2vcpu_ht_on() { + check_update_amd_features_entry(2, true); + } } From 76b79eefc0cb753d2fbe9577cf827873b42c7f1b Mon Sep 17 00:00:00 2001 From: Serban Iorga Date: Mon, 8 Apr 2019 18:46:22 +0300 Subject: [PATCH 5/6] AMD cpuid improvements - set the extended apic id - fix the number of threads per core - set the number of nodes per processor Signed-off-by: Serban Iorga --- cpuid/src/cpu_leaf.rs | 47 +++++++++++--- cpuid/src/transformer/amd.rs | 107 +++++++++++++++++++++++++++++++- cpuid/src/transformer/common.rs | 40 ++++++++++-- 3 files changed, 181 insertions(+), 13 deletions(-) diff --git a/cpuid/src/cpu_leaf.rs b/cpuid/src/cpu_leaf.rs index f4d8c578f8c..fdff902e6e0 100644 --- a/cpuid/src/cpu_leaf.rs +++ b/cpuid/src/cpu_leaf.rs @@ -88,14 +88,6 @@ pub mod leaf_0x4 { } } -// Extended Cache Topology Leaf -pub mod leaf_0x8000001d { - pub const LEAF_NUM: u32 = 0x8000_001d; - - // inherit eax from leaf_cache_parameters - pub use cpu_leaf::leaf_cache_parameters::eax; -} - // Thermal and Power Management Leaf pub mod leaf_0x6 { pub const LEAF_NUM: u32 = 0x6; @@ -250,3 +242,42 @@ pub mod leaf_0x80000008 { pub const NUM_THREADS_BITRANGE: BitRange = bit_range!(7, 0); } } + +// Extended Cache Topology Leaf +pub mod leaf_0x8000001d { + pub const LEAF_NUM: u32 = 0x8000_001d; + + // inherit eax from leaf_cache_parameters + pub use cpu_leaf::leaf_cache_parameters::eax; +} + +// Extended APIC ID Leaf +pub mod leaf_0x8000001e { + pub const LEAF_NUM: u32 = 0x8000_001e; + + pub mod eax { + use bit_helper::BitRange; + + pub const EXTENDED_APIC_ID_BITRANGE: BitRange = bit_range!(31, 0); + } + + pub mod ebx { + use bit_helper::BitRange; + + // The number of threads per core - 1 + pub const THREADS_PER_CORE_BITRANGE: BitRange = bit_range!(15, 8); + pub const CORE_ID_BITRANGE: BitRange = bit_range!(7, 0); + } + + pub mod ecx { + use bit_helper::BitRange; + + // The number of nodes per processor. Possible values: + // 0 -> 1 node per processor + // 1 -> 2 nodes per processor + // 2 -> Reserved + // 3 -> 4 nodes per processor + pub const NODES_PER_PROCESSOR_BITRANGE: BitRange = bit_range!(10, 8); + pub const NODE_ID_BITRANGE: BitRange = bit_range!(7, 0); + } +} diff --git a/cpuid/src/transformer/amd.rs b/cpuid/src/transformer/amd.rs index 20c6a061621..56a6aef3744 100644 --- a/cpuid/src/transformer/amd.rs +++ b/cpuid/src/transformer/amd.rs @@ -15,6 +15,9 @@ const LARGEST_EXTENDED_FN: u32 = 0x8000_001f; // This value allows at most 64 logical threads within a package. // See also the documentation for leaf_0x80000008::ecx::THREAD_ID_SIZE_BITRANGE const THREAD_ID_MAX_SIZE: u32 = 6; +// This value means there is 1 node per processor. +// See also the documentation for leaf_0x8000001e::ecx::NODES_PER_PROCESSOR_BITRANGE. +const NODES_PER_PROCESSOR: u32 = 0; pub fn update_structured_extended_entry( entry: &mut kvm_cpuid_entry2, @@ -84,11 +87,52 @@ pub fn update_extended_cache_topology_entry( common::update_cache_parameters_entry(entry, vm_spec) } +pub fn update_extended_apic_id_entry( + entry: &mut kvm_cpuid_entry2, + vm_spec: &VmSpec, +) -> Result<(), Error> { + use cpu_leaf::leaf_0x8000001e::*; + + let mut core_id = u32::from(vm_spec.cpu_id); + // When hyper-threading is enabled each pair of 2 consecutive logical CPUs + // will have the same core id since they represent 2 threads in the same core. + // For Example: + // logical CPU 0 -> core id: 0 + // logical CPU 1 -> core id: 0 + // logical CPU 2 -> core id: 1 + // logical CPU 3 -> core id: 1 + if vm_spec.ht_enabled { + core_id /= 2; + } + + entry + .eax + // the Extended APIC ID is the id of the current logical CPU + .write_bits_in_range(&eax::EXTENDED_APIC_ID_BITRANGE, u32::from(vm_spec.cpu_id)); + + entry + .ebx + .write_bits_in_range(&ebx::CORE_ID_BITRANGE, core_id) + .write_bits_in_range( + &ebx::THREADS_PER_CORE_BITRANGE, + u32::from(vm_spec.ht_enabled), + ); + + entry + .ecx + .write_bits_in_range(&ecx::NODES_PER_PROCESSOR_BITRANGE, NODES_PER_PROCESSOR) + // Put all the cpus in the same node. + .write_bits_in_range(&ecx::NODE_ID_BITRANGE, 0); + + Ok(()) +} + pub struct AmdCpuidTransformer {} impl CpuidTransformer for AmdCpuidTransformer { fn preprocess_cpuid(&self, cpuid: &mut CpuId) -> Result<(), Error> { - use_host_cpuid_function(cpuid, leaf_0x8000001d::LEAF_NUM) + use_host_cpuid_function(cpuid, leaf_0x8000001e::LEAF_NUM, false)?; + use_host_cpuid_function(cpuid, leaf_0x8000001d::LEAF_NUM, true) } fn transform_entry(&self, entry: &mut kvm_cpuid_entry2, vm_spec: &VmSpec) -> Result<(), Error> { @@ -99,6 +143,7 @@ impl CpuidTransformer for AmdCpuidTransformer { leaf_0x80000001::LEAF_NUM => Some(amd::update_extended_feature_info_entry), leaf_0x80000008::LEAF_NUM => Some(amd::update_amd_features_entry), leaf_0x8000001d::LEAF_NUM => Some(amd::update_extended_cache_topology_entry), + leaf_0x8000001e::LEAF_NUM => Some(amd::update_extended_apic_id_entry), 0x8000_0002..=0x8000_0004 => Some(common::update_brand_string_entry), _ => None, }; @@ -216,6 +261,56 @@ mod test { ); } + fn check_update_extended_apic_id_entry( + cpu_id: u8, + cpu_count: u8, + ht_enabled: bool, + expected_core_id: u32, + expected_threads_per_core: u32, + ) { + use cpu_leaf::leaf_0x8000001e::*; + + let vm_spec = VmSpec::new(VENDOR_ID_AMD, cpu_id, cpu_count, ht_enabled); + let mut entry = &mut kvm_cpuid_entry2 { + function: LEAF_NUM, + index: 0, + flags: 0, + eax: 0, + ebx: 0, + ecx: 0, + edx: 0, + padding: [0, 0, 0], + }; + + assert!(update_extended_apic_id_entry(&mut entry, &vm_spec).is_ok()); + + assert_eq!( + entry + .eax + .read_bits_in_range(&eax::EXTENDED_APIC_ID_BITRANGE), + u32::from(cpu_id) + ); + + assert_eq!( + entry.ebx.read_bits_in_range(&ebx::CORE_ID_BITRANGE), + expected_core_id + ); + assert_eq!( + entry + .ebx + .read_bits_in_range(&ebx::THREADS_PER_CORE_BITRANGE), + expected_threads_per_core + ); + + assert_eq!( + entry + .ecx + .read_bits_in_range(&ecx::NODES_PER_PROCESSOR_BITRANGE), + NODES_PER_PROCESSOR + ); + assert_eq!(entry.ecx.read_bits_in_range(&ecx::NODE_ID_BITRANGE), 0); + } + #[test] fn test_update_extended_cache_topology_entry() { let vm_spec = VmSpec::new(VENDOR_ID_AMD, 0, 1, false); @@ -238,20 +333,30 @@ mod test { #[test] fn test_1vcpu_ht_off() { check_update_amd_features_entry(1, false); + + check_update_extended_apic_id_entry(0, 1, false, 0, 0); } #[test] fn test_1vcpu_ht_on() { check_update_amd_features_entry(1, true); + + check_update_extended_apic_id_entry(0, 1, true, 0, 1); } #[test] fn test_2vcpu_ht_off() { check_update_amd_features_entry(2, false); + + check_update_extended_apic_id_entry(0, 2, false, 0, 0); + check_update_extended_apic_id_entry(1, 2, false, 1, 0); } #[test] fn test_2vcpu_ht_on() { check_update_amd_features_entry(2, true); + + check_update_extended_apic_id_entry(0, 2, true, 0, 1); + check_update_extended_apic_id_entry(1, 2, true, 0, 1); } } diff --git a/cpuid/src/transformer/common.rs b/cpuid/src/transformer/common.rs index cd3c3518bee..3b5dd0a71c4 100644 --- a/cpuid/src/transformer/common.rs +++ b/cpuid/src/transformer/common.rs @@ -96,7 +96,11 @@ pub fn update_cache_parameters_entry( /// Replaces the `cpuid` entries corresponding to `function` with the entries from the host's cpuid. /// -pub fn use_host_cpuid_function(cpuid: &mut CpuId, function: u32) -> Result<(), Error> { +pub fn use_host_cpuid_function( + cpuid: &mut CpuId, + function: u32, + use_count: bool, +) -> Result<(), Error> { // copy all the CpuId entries, except for the ones with the provided function let mut entries: Vec = Vec::new(); for entry in cpuid.mut_entries_slice().iter() { @@ -108,6 +112,9 @@ pub fn use_host_cpuid_function(cpuid: &mut CpuId, function: u32) -> Result<(), E // add all the host leaves with the provided function let mut count: u32 = 0; while let Ok(entry) = get_cpuid(function, count) { + if count > 0 && !use_count { + break; + } // check if there's enough space to add a new entry to the cpuid if entries.len() == MAX_KVM_CPUID_ENTRIES { return Err(Error::SizeLimitExceeded); @@ -254,14 +261,14 @@ mod test { #[test] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - fn use_host_cpuid_function_test() { + fn test_use_host_cpuid_function_with_count() { // try to emulate the extended cache topology leaves let topoext_fn = get_topoext_fn(); // check that it behaves correctly for TOPOEXT function let mut cpuid = CpuId::new(1); cpuid.mut_entries_slice()[0].function = topoext_fn; - assert!(use_host_cpuid_function(&mut cpuid, topoext_fn).is_ok()); + assert!(use_host_cpuid_function(&mut cpuid, topoext_fn, true).is_ok()); let entries = cpuid.mut_entries_slice(); assert!(entries.len() > 1); let mut count = 0; @@ -271,10 +278,35 @@ mod test { assert!(entry.eax != 0); count = count + 1; } + } + #[test] + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + fn test_use_host_cpuid_function_without_count() { + use cpu_leaf::leaf_0x1::*; + // try to emulate the extended cache topology leaves + let feature_info_fn = LEAF_NUM; + + // check that it behaves correctly for TOPOEXT function + let mut cpuid = CpuId::new(1); + cpuid.mut_entries_slice()[0].function = feature_info_fn; + assert!(use_host_cpuid_function(&mut cpuid, feature_info_fn, false).is_ok()); + let entries = cpuid.mut_entries_slice(); + assert!(entries.len() == 1); + let entry = entries[0]; + + assert!(entry.function == feature_info_fn); + assert!(entry.index == 0); + assert!(entry.eax != 0); + } + + #[test] + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + fn test_use_host_cpuid_function_err() { + let topoext_fn = get_topoext_fn(); // check that it returns Err when there are too many entriesentry.function == topoext_fn let mut cpuid = CpuId::new(MAX_KVM_CPUID_ENTRIES); - match use_host_cpuid_function(&mut cpuid, topoext_fn) { + match use_host_cpuid_function(&mut cpuid, topoext_fn, true) { Err(Error::SizeLimitExceeded) => {} _ => panic!("Wrong behavior"), } From b8550cc749227565ad1e13ae4b99821c98239e94 Mon Sep 17 00:00:00 2001 From: Serban Iorga Date: Mon, 8 Apr 2019 18:47:03 +0300 Subject: [PATCH 6/6] add cpuid integration tests Signed-off-by: Serban Iorga --- .../integration_tests/build/test_coverage.py | 2 +- .../functional/test_cpu_features.py | 260 +++++++++++++----- 2 files changed, 199 insertions(+), 63 deletions(-) diff --git a/tests/integration_tests/build/test_coverage.py b/tests/integration_tests/build/test_coverage.py index 3634de0b6c7..cc5b178ffef 100644 --- a/tests/integration_tests/build/test_coverage.py +++ b/tests/integration_tests/build/test_coverage.py @@ -18,7 +18,7 @@ import host_tools.cargo_build as host # pylint: disable=import-error -COVERAGE_TARGET_PCT = 83.9 +COVERAGE_TARGET_PCT = 84.1 COVERAGE_MAX_DELTA = 0.01 CARGO_KCOV_REL_PATH = os.path.join(host.CARGO_BUILD_REL_PATH, 'kcov') diff --git a/tests/integration_tests/functional/test_cpu_features.py b/tests/integration_tests/functional/test_cpu_features.py index bfe607a0803..cbfb632805a 100644 --- a/tests/integration_tests/functional/test_cpu_features.py +++ b/tests/integration_tests/functional/test_cpu_features.py @@ -4,78 +4,216 @@ import re +from enum import Enum, auto + import host_tools.network as net_tools # pylint: disable=import-error -def test_1vcpu(test_microvm_with_ssh, network_config): - """Test CPU feature emulation with 1 vCPU.""" - test_microvm = test_microvm_with_ssh - test_microvm.spawn() +class CpuVendor(Enum): + """CPU vendors enum.""" - # Set up the microVM with 1 vCPUs, 256 MiB of RAM, no network ifaces, and - # a root file system with the rw permission. The network interfaces is - # added after we get a unique MAC and IP. - test_microvm.basic_config(vcpu_count=1) + AMD = auto() + INTEL = auto() - _tap, _, _ = test_microvm.ssh_network_config(network_config, '1') - test_microvm.start() + +def _get_cpu_vendor(): + cif = open('/proc/cpuinfo', 'r') + host_vendor_id = None + while True: + line = cif.readline() + if line == '': + break + mo = re.search("^vendor_id\\s+:\\s+(.+)$", line) + if mo: + host_vendor_id = mo.group(1) + cif.close() + assert host_vendor_id is not None + + if host_vendor_id == "AuthenticAMD": + return CpuVendor.AMD + return CpuVendor.INTEL + + +def _check_guest_cmd_output(test_microvm, guest_cmd, expected_header, + expected_separator, + expected_key_value_store): + ssh_connection = net_tools.SSHConnection(test_microvm.ssh_config) + _, stdout, stderr = ssh_connection.execute_command(guest_cmd) + + assert stderr.read().decode("utf-8") == '' + while True: + line = stdout.readline().decode('utf-8') + if line != '': + # All the keys have been matched. Stop. + if not expected_key_value_store: + break + + # Try to match the header if needed. + if expected_header not in (None, ''): + if line.strip() == expected_header: + expected_header = None + continue + else: + continue + + # See if any key matches. + # We Use a try-catch block here since line.split() may fail. + try: + [key, value] = list( + map(lambda x: x.strip(), line.split(expected_separator))) + except ValueError: + continue + + if key in expected_key_value_store.keys(): + assert value == expected_key_value_store[key], \ + "%s does not have the expected value" % key + del expected_key_value_store[key] + + else: + break + + assert not expected_key_value_store, \ + "some keys in dictionary have not been found in the output: %s" \ + % expected_key_value_store + + +def _check_cpu_topology(test_microvm, expected_cpu_count, + expected_threads_per_core, + expected_cpus_list): expected_cpu_topology = { - "CPU(s)": "1", - "On-line CPU(s) list": "0", - "Thread(s) per core": "1", - "Core(s) per socket": "1", + "CPU(s)": str(expected_cpu_count), + "On-line CPU(s) list": expected_cpus_list, + "Thread(s) per core": str(expected_threads_per_core), + "Core(s) per socket": str( + int(expected_cpu_count / expected_threads_per_core)), "Socket(s)": "1", "NUMA node(s)": "1" } - _check_cpu_topology(test_microvm, expected_cpu_topology) + + _check_guest_cmd_output(test_microvm, "lscpu", None, ':', + expected_cpu_topology) + + +def _check_cpu_features(test_microvm, expected_cpu_count, expected_htt): + expected_cpu_features = { + "cpu count": '{} ({})'.format(hex(expected_cpu_count), + expected_cpu_count), + "CLFLUSH line size": "0x8 (8)", + "hypervisor guest status": "true", + "hyper-threading / multi-core supported": expected_htt + } + + _check_guest_cmd_output(test_microvm, "cpuid -1", None, '=', + expected_cpu_features) + + +def _check_cache_topology(test_microvm, num_vcpus_on_lvl_1_cache, + num_vcpus_on_lvl_3_cache): + expected_lvl_1_str = '{} ({})'.format(hex(num_vcpus_on_lvl_1_cache), + num_vcpus_on_lvl_1_cache) + expected_lvl_3_str = '{} ({})'.format(hex(num_vcpus_on_lvl_3_cache), + num_vcpus_on_lvl_3_cache) + + cpu_vendor = _get_cpu_vendor() + if cpu_vendor == CpuVendor.AMD: + expected_level_1_topology = { + "level": '0x1 (1)', + "extra cores sharing this cache": expected_lvl_1_str + } + expected_level_3_topology = { + "level": '0x3 (3)', + "extra cores sharing this cache": expected_lvl_3_str + } + elif cpu_vendor == CpuVendor.INTEL: + expected_level_1_topology = { + "cache level": '0x1 (1)', + "extra threads sharing this cache": expected_lvl_1_str, + } + expected_level_3_topology = { + "cache level": '0x3 (3)', + "extra threads sharing this cache": expected_lvl_3_str, + } + + _check_guest_cmd_output(test_microvm, "cpuid -1", "--- cache 0 ---", '=', + expected_level_1_topology) + _check_guest_cmd_output(test_microvm, "cpuid -1", "--- cache 1 ---", '=', + expected_level_1_topology) + _check_guest_cmd_output(test_microvm, "cpuid -1", "--- cache 2 ---", '=', + expected_level_1_topology) + _check_guest_cmd_output(test_microvm, "cpuid -1", "--- cache 3 ---", '=', + expected_level_3_topology) + + +def test_1vcpu_ht_disabled(test_microvm_with_ssh, network_config): + """Check the CPUID for a microvm with the specified config.""" + test_microvm_with_ssh.spawn() + test_microvm_with_ssh.basic_config(vcpu_count=1, ht_enabled=False) + _tap, _, _ = test_microvm_with_ssh.ssh_network_config(network_config, '1') + test_microvm_with_ssh.start() + + _check_cpu_topology(test_microvm_with_ssh, 1, 1, "0") + _check_cpu_features(test_microvm_with_ssh, 1, "false") + _check_cache_topology(test_microvm_with_ssh, 0, 0) + + +def test_1vcpu_ht_enabled(test_microvm_with_ssh, network_config): + """Check the CPUID for a microvm with the specified config.""" + test_microvm_with_ssh.spawn() + test_microvm_with_ssh.basic_config(vcpu_count=1, ht_enabled=True) + _tap, _, _ = test_microvm_with_ssh.ssh_network_config(network_config, '1') + test_microvm_with_ssh.start() + + _check_cpu_topology(test_microvm_with_ssh, 1, 1, "0") + _check_cpu_features(test_microvm_with_ssh, 1, "false") + _check_cache_topology(test_microvm_with_ssh, 0, 0) def test_2vcpu_ht_disabled(test_microvm_with_ssh, network_config): - """Test CPU feature emulation with 2 vCPUs, and no hyperthreading.""" - test_microvm = test_microvm_with_ssh - test_microvm.spawn() + """Check the CPUID for a microvm with the specified config.""" + test_microvm_with_ssh.spawn() + test_microvm_with_ssh.basic_config(vcpu_count=2, ht_enabled=False) + _tap, _, _ = test_microvm_with_ssh.ssh_network_config(network_config, '1') + test_microvm_with_ssh.start() - # Set up the microVM with 2 vCPUs, 256 MiB of RAM, 0 network ifaces, and - # a root file system with the rw permission. The network interfaces is - # added after we get a unique MAC and IP. - test_microvm.basic_config(vcpu_count=2, ht_enabled=False) + _check_cpu_topology(test_microvm_with_ssh, 2, 1, "0,1") + _check_cpu_features(test_microvm_with_ssh, 2, "true") + _check_cache_topology(test_microvm_with_ssh, 0, 1) - _tap, _, _ = test_microvm.ssh_network_config(network_config, '1') - test_microvm.start() +def test_2vcpu_ht_enabled(test_microvm_with_ssh, network_config): + """Check the CPUID for a microvm with the specified config.""" + test_microvm_with_ssh.spawn() + test_microvm_with_ssh.basic_config(vcpu_count=2, ht_enabled=True) + _tap, _, _ = test_microvm_with_ssh.ssh_network_config(network_config, '1') + test_microvm_with_ssh.start() - expected_cpu_topology = { - "CPU(s)": "2", - "On-line CPU(s) list": "0,1", - "Thread(s) per core": "1", - "Core(s) per socket": "2", - "Socket(s)": "1", - "NUMA node(s)": "1" - } - _check_cpu_topology(test_microvm, expected_cpu_topology) + _check_cpu_topology(test_microvm_with_ssh, 2, 2, "0,1") + _check_cpu_features(test_microvm_with_ssh, 2, "true") + _check_cache_topology(test_microvm_with_ssh, 1, 1) -def _check_cpu_topology(test_microvm, expected_cpu_topology): - """Perform common microvm setup for different CPU topology tests. +def test_16vcpu_ht_disabled(test_microvm_with_ssh, network_config): + """Check the CPUID for a microvm with the specified config.""" + test_microvm_with_ssh.spawn() + test_microvm_with_ssh.basic_config(vcpu_count=16, ht_enabled=False) + _tap, _, _ = test_microvm_with_ssh.ssh_network_config(network_config, '1') + test_microvm_with_ssh.start() - This is a wrapper function for calling lscpu and checking if the - command returns the expected cpu topology. - """ - ssh_connection = net_tools.SSHConnection(test_microvm.ssh_config) + _check_cpu_topology(test_microvm_with_ssh, 16, 1, "0-15") + _check_cpu_features(test_microvm_with_ssh, 16, "true") + _check_cache_topology(test_microvm_with_ssh, 0, 15) - # Execute the lscpu command to check the guest topology - _, stdout, stderr = ssh_connection.execute_command("lscpu") - assert stderr.read().decode("utf-8") == '' - # Read the stdout of lscpu line by line to check the relevant information. - while True: - line = stdout.readline().decode('utf-8') - if line != '': - [key, value] = list(map(lambda x: x.strip(), line.split(':'))) - if key in expected_cpu_topology.keys(): - assert value == expected_cpu_topology[key],\ - "%s does not have the expected value" % key - else: - break + +def test_16vcpu_ht_enabled(test_microvm_with_ssh, network_config): + """Check the CPUID for a microvm with the specified config.""" + test_microvm_with_ssh.spawn() + test_microvm_with_ssh.basic_config(vcpu_count=16, ht_enabled=True) + _tap, _, _ = test_microvm_with_ssh.ssh_network_config(network_config, '1') + test_microvm_with_ssh.start() + + _check_cpu_topology(test_microvm_with_ssh, 16, 2, "0-15") + _check_cpu_features(test_microvm_with_ssh, 16, "true") + _check_cache_topology(test_microvm_with_ssh, 1, 15) def test_brand_string(test_microvm_with_ssh, network_config): @@ -85,24 +223,21 @@ def test_brand_string(test_microvm_with_ssh, network_config): Intel(R) Xeon(R) Processor @ {host frequency} where {host frequency} is the frequency reported by the host CPUID (e.g. 4.01GHz) - * For non-Intel CPUs, the guest brand string should be: - Intel(R) Xeon(R) Processor + * For AMD CPUs, the guest brand string should be: + AMD EPYC + * For other CPUs, the guest brand string should be: + "" """ cif = open('/proc/cpuinfo', 'r') host_brand_string = None - host_vendor_id = None while True: line = cif.readline() if line == '': break - mo = re.search("^vendor_id\\s+:\\s+(.+)$", line) - if mo: - host_vendor_id = mo.group(1) mo = re.search("^model name\\s+:\\s+(.+)$", line) if mo: host_brand_string = mo.group(1) cif.close() - assert host_vendor_id is not None assert host_brand_string is not None test_microvm = test_microvm_with_ssh @@ -124,13 +259,14 @@ def test_brand_string(test_microvm_with_ssh, network_config): guest_brand_string = mo.group(1) assert guest_brand_string + cpu_vendor = _get_cpu_vendor() expected_guest_brand_string = "" - if host_vendor_id == "GenuineIntel": + if cpu_vendor == CpuVendor.AMD: + expected_guest_brand_string += "AMD EPYC" + elif cpu_vendor == CpuVendor.INTEL: expected_guest_brand_string = "Intel(R) Xeon(R) Processor" mo = re.search("[.0-9]+[MG]Hz", host_brand_string) if mo: expected_guest_brand_string += " @ " + mo.group(0) - elif host_vendor_id == "AuthenticAMD": - expected_guest_brand_string += "AMD EPYC" assert guest_brand_string == expected_guest_brand_string