From ecd6663d95c94e1f1b736eab4254c2debd94edc1 Mon Sep 17 00:00:00 2001 From: Matthew Rosato Date: Fri, 27 Sep 2019 09:21:01 -0400 Subject: [PATCH 01/12] MAINTAINERS: Update S390 PCI Maintainer As discussed previously with Collin, I will take over maintaining s390 pci. Signed-off-by: Matthew Rosato Message-Id: <1569590461-12562-1-git-send-email-mjrosato@linux.ibm.com> Acked-by: Collin Walling Signed-off-by: Christian Borntraeger --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index bd7ee2310184..21264eae9c43 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1206,7 +1206,7 @@ T: git https://github.com/borntraeger/qemu.git s390-next L: qemu-s390x@nongnu.org S390 PCI -M: Collin Walling +M: Matthew Rosato S: Supported F: hw/s390x/s390-pci* L: qemu-s390x@nongnu.org From 7df1dac5f1c85312474df9cb3a8fcae72303da62 Mon Sep 17 00:00:00 2001 From: Matthew Rosato Date: Thu, 26 Sep 2019 10:10:36 -0400 Subject: [PATCH 02/12] s390: PCI: fix IOMMU region init The fix in dbe9cf606c shrinks the IOMMU memory region to a size that seems reasonable on the surface, however is actually too small as it is based against a 0-mapped address space. This causes breakage with small guests as they can overrun the IOMMU window. Let's go back to the prior method of initializing iommu for now. Fixes: dbe9cf606c ("s390x/pci: Set the iommu region size mpcifc request") Cc: qemu-stable@nongnu.org Reviewed-by: Pierre Morel Reported-by: Boris Fiuczynski Tested-by: Boris Fiuczynski Reported-by: Stefan Zimmerman Signed-off-by: Matthew Rosato Message-Id: <1569507036-15314-1-git-send-email-mjrosato@linux.ibm.com> Signed-off-by: Christian Borntraeger --- hw/s390x/s390-pci-bus.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c index 963a41c7f532..2d2f4a7c419c 100644 --- a/hw/s390x/s390-pci-bus.c +++ b/hw/s390x/s390-pci-bus.c @@ -695,10 +695,15 @@ static const MemoryRegionOps s390_msi_ctrl_ops = { void s390_pci_iommu_enable(S390PCIIOMMU *iommu) { + /* + * The iommu region is initialized against a 0-mapped address space, + * so the smallest IOMMU region we can define runs from 0 to the end + * of the PCI address space. + */ char *name = g_strdup_printf("iommu-s390-%04x", iommu->pbdev->uid); memory_region_init_iommu(&iommu->iommu_mr, sizeof(iommu->iommu_mr), TYPE_S390_IOMMU_MEMORY_REGION, OBJECT(&iommu->mr), - name, iommu->pal - iommu->pba + 1); + name, iommu->pal + 1); iommu->enabled = true; memory_region_add_subregion(&iommu->mr, 0, MEMORY_REGION(&iommu->iommu_mr)); g_free(name); From 679b844756e4eaa1b94bb82f52ec9f999dc4f851 Mon Sep 17 00:00:00 2001 From: Janosch Frank Date: Fri, 27 Sep 2019 15:33:20 +0200 Subject: [PATCH 03/12] s390x: sclp: refactor invalid command check Invalid command checking has to be done before the boundary check, refactoring it now allows to insert the boundary check at the correct place later. Signed-off-by: Janosch Frank Reviewed-by: Jason J. Herne Message-Id: <1569591203-15258-2-git-send-email-imbrenda@linux.ibm.com> Reviewed-by: David Hildenbrand Signed-off-by: Christian Borntraeger --- hw/s390x/event-facility.c | 3 --- hw/s390x/sclp.c | 17 ++++++++++++++++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/hw/s390x/event-facility.c b/hw/s390x/event-facility.c index 797ecbb7a9c8..66205697ae75 100644 --- a/hw/s390x/event-facility.c +++ b/hw/s390x/event-facility.c @@ -377,9 +377,6 @@ static void command_handler(SCLPEventFacility *ef, SCCB *sccb, uint64_t code) case SCLP_CMD_WRITE_EVENT_MASK: write_event_mask(ef, sccb); break; - default: - sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_SCLP_COMMAND); - break; } } diff --git a/hw/s390x/sclp.c b/hw/s390x/sclp.c index fac7c3bb6c02..95ebfe7bd2f1 100644 --- a/hw/s390x/sclp.c +++ b/hw/s390x/sclp.c @@ -219,8 +219,23 @@ int sclp_service_call(CPUS390XState *env, uint64_t sccb, uint32_t code) goto out; } - sclp_c->execute(sclp, &work_sccb, code); + switch (code & SCLP_CMD_CODE_MASK) { + case SCLP_CMDW_READ_SCP_INFO: + case SCLP_CMDW_READ_SCP_INFO_FORCED: + case SCLP_CMDW_READ_CPU_INFO: + case SCLP_CMDW_CONFIGURE_IOA: + case SCLP_CMDW_DECONFIGURE_IOA: + case SCLP_CMD_READ_EVENT_DATA: + case SCLP_CMD_WRITE_EVENT_DATA: + case SCLP_CMD_WRITE_EVENT_MASK: + break; + default: + work_sccb.h.response_code = cpu_to_be16(SCLP_RC_INVALID_SCLP_COMMAND); + goto out_write; + } + sclp_c->execute(sclp, &work_sccb, code); +out_write: cpu_physical_memory_write(sccb, &work_sccb, be16_to_cpu(work_sccb.h.length)); From 6f6c9333efaf43d5a6f435266444aa2c51ac11b2 Mon Sep 17 00:00:00 2001 From: Janosch Frank Date: Fri, 27 Sep 2019 15:33:21 +0200 Subject: [PATCH 04/12] s390x: sclp: boundary check All sclp codes need to be checked for page boundary violations. Signed-off-by: Janosch Frank Reviewed-by: Jason J. Herne Message-Id: <1569591203-15258-3-git-send-email-imbrenda@linux.ibm.com> Reviewed-by: David Hildenbrand Signed-off-by: Christian Borntraeger --- hw/s390x/sclp.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hw/s390x/sclp.c b/hw/s390x/sclp.c index 95ebfe7bd2f1..73244c938b10 100644 --- a/hw/s390x/sclp.c +++ b/hw/s390x/sclp.c @@ -234,6 +234,11 @@ int sclp_service_call(CPUS390XState *env, uint64_t sccb, uint32_t code) goto out_write; } + if ((sccb + be16_to_cpu(work_sccb.h.length)) > ((sccb & PAGE_MASK) + PAGE_SIZE)) { + work_sccb.h.response_code = cpu_to_be16(SCLP_RC_SCCB_BOUNDARY_VIOLATION); + goto out_write; + } + sclp_c->execute(sclp, &work_sccb, code); out_write: cpu_physical_memory_write(sccb, &work_sccb, From d959f6cf502f37ddda91140a2e7a2e2b72de397e Mon Sep 17 00:00:00 2001 From: Janosch Frank Date: Fri, 27 Sep 2019 15:33:22 +0200 Subject: [PATCH 05/12] s390x: sclp: fix error handling for oversize control blocks Requests over 4k are not a spec exception. Signed-off-by: Janosch Frank Reviewed-by: Jason J. Herne Message-Id: <1569591203-15258-4-git-send-email-imbrenda@linux.ibm.com> Acked-by: David Hildenbrand Signed-off-by: Christian Borntraeger --- hw/s390x/sclp.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hw/s390x/sclp.c b/hw/s390x/sclp.c index 73244c938b10..abb6e5011f9c 100644 --- a/hw/s390x/sclp.c +++ b/hw/s390x/sclp.c @@ -213,8 +213,7 @@ int sclp_service_call(CPUS390XState *env, uint64_t sccb, uint32_t code) cpu_physical_memory_read(sccb, &work_sccb, sccb_len); /* Valid sccb sizes */ - if (be16_to_cpu(work_sccb.h.length) < sizeof(SCCBHeader) || - be16_to_cpu(work_sccb.h.length) > SCCB_SIZE) { + if (be16_to_cpu(work_sccb.h.length) < sizeof(SCCBHeader)) { r = -PGM_SPECIFICATION; goto out; } From 832be0d8a3bb7b54d64730f7a37d674f30ca0427 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Fri, 27 Sep 2019 15:33:23 +0200 Subject: [PATCH 06/12] s390x: sclp: Report insufficient SCCB length Return the correct error code when the SCCB buffer is too small to contain all of the output, for the Read SCP Information and Read CPU Information commands. Signed-off-by: Claudio Imbrenda Reviewed-by: Jason J. Herne Message-Id: <1569591203-15258-5-git-send-email-imbrenda@linux.ibm.com> Reviewed-by: David Hildenbrand Signed-off-by: Christian Borntraeger --- hw/s390x/sclp.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/hw/s390x/sclp.c b/hw/s390x/sclp.c index abb6e5011f9c..f57ce7b73943 100644 --- a/hw/s390x/sclp.c +++ b/hw/s390x/sclp.c @@ -68,6 +68,12 @@ static void read_SCP_info(SCLPDevice *sclp, SCCB *sccb) read_info->ibc_val = cpu_to_be32(s390_get_ibc_val()); + if (be16_to_cpu(sccb->h.length) < + (sizeof(ReadInfo) + cpu_count * sizeof(CPUEntry))) { + sccb->h.response_code = cpu_to_be16(SCLP_RC_INSUFFICIENT_SCCB_LENGTH); + return; + } + /* Configuration Characteristic (Extension) */ s390_get_feat_block(S390_FEAT_TYPE_SCLP_CONF_CHAR, read_info->conf_char); @@ -118,6 +124,12 @@ static void sclp_read_cpu_info(SCLPDevice *sclp, SCCB *sccb) cpu_info->offset_configured = cpu_to_be16(offsetof(ReadCpuInfo, entries)); cpu_info->nr_standby = cpu_to_be16(0); + if (be16_to_cpu(sccb->h.length) < + (sizeof(ReadCpuInfo) + cpu_count * sizeof(CPUEntry))) { + sccb->h.response_code = cpu_to_be16(SCLP_RC_INSUFFICIENT_SCCB_LENGTH); + return; + } + /* The standby offset is 16-byte for each CPU */ cpu_info->offset_standby = cpu_to_be16(cpu_info->offset_configured + cpu_info->nr_configured*sizeof(CPUEntry)); From ee35e9684c334523e2864b1f9c31f0838e834343 Mon Sep 17 00:00:00 2001 From: Thomas Huth Date: Sat, 28 Sep 2019 21:03:34 +0200 Subject: [PATCH 07/12] configure: Remove s390 (31-bit mode) from the list of supported CPUs On IBM Z, KVM in the kernel is only implemented for 64-bit mode, and with regards to TCG, we also only support 64-bit host CPUs (see the check at the beginning of tcg/s390/tcg-target.inc.c), so we should remove s390 (without "x", i.e. the old 31-bit mode CPUs) from the list of supported CPUs. Signed-off-by: Thomas Huth Message-Id: <20190928190334.6897-1-thuth@redhat.com> Reviewed-by: David Hildenbrand Signed-off-by: Christian Borntraeger --- configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure b/configure index 542f6aea3f61..8f8446f52b92 100755 --- a/configure +++ b/configure @@ -728,7 +728,7 @@ ARCH= # Normalise host CPU name and set ARCH. # Note that this case should only have supported host CPUs, not guests. case "$cpu" in - ppc|ppc64|s390|s390x|sparc64|x32|riscv32|riscv64) + ppc|ppc64|s390x|sparc64|x32|riscv32|riscv64) supported_cpu="yes" ;; ppc64le) From 4222147dfb7229de36d8cc8c9fa3543b43347241 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 24 Sep 2019 10:47:48 -0400 Subject: [PATCH 08/12] kvm: extract kvm_log_clear_one_slot We may need to clear the dirty bitmap for more than one KVM memslot. First do some code movement with no semantic change. Signed-off-by: Paolo Bonzini Signed-off-by: Igor Mammedov Reviewed-by: Peter Xu Message-Id: <20190924144751.24149-2-imammedo@redhat.com> Acked-by: Paolo Bonzini Signed-off-by: Christian Borntraeger [fixup line break] --- accel/kvm/kvm-all.c | 103 ++++++++++++++++++++++++-------------------- 1 file changed, 57 insertions(+), 46 deletions(-) diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index b09bad08048d..a85ec09486dd 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -575,55 +575,14 @@ static int kvm_physical_sync_dirty_bitmap(KVMMemoryListener *kml, #define KVM_CLEAR_LOG_ALIGN (qemu_real_host_page_size << KVM_CLEAR_LOG_SHIFT) #define KVM_CLEAR_LOG_MASK (-KVM_CLEAR_LOG_ALIGN) -/** - * kvm_physical_log_clear - Clear the kernel's dirty bitmap for range - * - * NOTE: this will be a no-op if we haven't enabled manual dirty log - * protection in the host kernel because in that case this operation - * will be done within log_sync(). - * - * @kml: the kvm memory listener - * @section: the memory range to clear dirty bitmap - */ -static int kvm_physical_log_clear(KVMMemoryListener *kml, - MemoryRegionSection *section) +static int kvm_log_clear_one_slot(KVMSlot *mem, int as_id, uint64_t start, + uint64_t size) { KVMState *s = kvm_state; + uint64_t end, bmap_start, start_delta, bmap_npages; struct kvm_clear_dirty_log d; - uint64_t start, end, bmap_start, start_delta, bmap_npages, size; unsigned long *bmap_clear = NULL, psize = qemu_real_host_page_size; - KVMSlot *mem = NULL; - int ret, i; - - if (!s->manual_dirty_log_protect) { - /* No need to do explicit clear */ - return 0; - } - - start = section->offset_within_address_space; - size = int128_get64(section->size); - - if (!size) { - /* Nothing more we can do... */ - return 0; - } - - kvm_slots_lock(kml); - - /* Find any possible slot that covers the section */ - for (i = 0; i < s->nr_slots; i++) { - mem = &kml->slots[i]; - if (mem->start_addr <= start && - start + size <= mem->start_addr + mem->memory_size) { - break; - } - } - - /* - * We should always find one memslot until this point, otherwise - * there could be something wrong from the upper layer - */ - assert(mem && i != s->nr_slots); + int ret; /* * We need to extend either the start or the size or both to @@ -694,7 +653,7 @@ static int kvm_physical_log_clear(KVMMemoryListener *kml, /* It should never overflow. If it happens, say something */ assert(bmap_npages <= UINT32_MAX); d.num_pages = bmap_npages; - d.slot = mem->slot | (kml->as_id << 16); + d.slot = mem->slot | (as_id << 16); if (kvm_vm_ioctl(s, KVM_CLEAR_DIRTY_LOG, &d) == -1) { ret = -errno; @@ -717,6 +676,58 @@ static int kvm_physical_log_clear(KVMMemoryListener *kml, size / psize); /* This handles the NULL case well */ g_free(bmap_clear); + return ret; +} + + +/** + * kvm_physical_log_clear - Clear the kernel's dirty bitmap for range + * + * NOTE: this will be a no-op if we haven't enabled manual dirty log + * protection in the host kernel because in that case this operation + * will be done within log_sync(). + * + * @kml: the kvm memory listener + * @section: the memory range to clear dirty bitmap + */ +static int kvm_physical_log_clear(KVMMemoryListener *kml, + MemoryRegionSection *section) +{ + KVMState *s = kvm_state; + uint64_t start, size; + KVMSlot *mem = NULL; + int ret, i; + + if (!s->manual_dirty_log_protect) { + /* No need to do explicit clear */ + return 0; + } + + start = section->offset_within_address_space; + size = int128_get64(section->size); + + if (!size) { + /* Nothing more we can do... */ + return 0; + } + + kvm_slots_lock(kml); + + /* Find any possible slot that covers the section */ + for (i = 0; i < s->nr_slots; i++) { + mem = &kml->slots[i]; + if (mem->start_addr <= start && + start + size <= mem->start_addr + mem->memory_size) { + break; + } + } + + /* + * We should always find one memslot until this point, otherwise + * there could be something wrong from the upper layer + */ + assert(mem && i != s->nr_slots); + ret = kvm_log_clear_one_slot(mem, kml->as_id, start, size); kvm_slots_unlock(kml); From 84516e5b8db66b8a169a8fdd4c25568ed0b67a3a Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 24 Sep 2019 10:47:49 -0400 Subject: [PATCH 09/12] kvm: clear dirty bitmaps from all overlapping memslots Currently MemoryRegionSection has 1:1 mapping to KVMSlot. However next patch will allow splitting MemoryRegionSection into several KVMSlot-s, make sure that kvm_physical_log_slot_clear() is able to handle such 1:N mapping. Signed-off-by: Paolo Bonzini Signed-off-by: Igor Mammedov Reviewed-by: Peter Xu Message-Id: <20190924144751.24149-3-imammedo@redhat.com> Acked-by: Paolo Bonzini Signed-off-by: Christian Borntraeger --- accel/kvm/kvm-all.c | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index a85ec09486dd..ff9b95c0d103 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -589,8 +589,8 @@ static int kvm_log_clear_one_slot(KVMSlot *mem, int as_id, uint64_t start, * satisfy the KVM interface requirement. Firstly, do the start * page alignment on 64 host pages */ - bmap_start = (start - mem->start_addr) & KVM_CLEAR_LOG_MASK; - start_delta = start - mem->start_addr - bmap_start; + bmap_start = start & KVM_CLEAR_LOG_MASK; + start_delta = start - bmap_start; bmap_start /= psize; /* @@ -694,8 +694,8 @@ static int kvm_physical_log_clear(KVMMemoryListener *kml, MemoryRegionSection *section) { KVMState *s = kvm_state; - uint64_t start, size; - KVMSlot *mem = NULL; + uint64_t start, size, offset, count; + KVMSlot *mem; int ret, i; if (!s->manual_dirty_log_protect) { @@ -713,22 +713,30 @@ static int kvm_physical_log_clear(KVMMemoryListener *kml, kvm_slots_lock(kml); - /* Find any possible slot that covers the section */ for (i = 0; i < s->nr_slots; i++) { mem = &kml->slots[i]; - if (mem->start_addr <= start && - start + size <= mem->start_addr + mem->memory_size) { + /* Discard slots that are empty or do not overlap the section */ + if (!mem->memory_size || + mem->start_addr > start + size - 1 || + start > mem->start_addr + mem->memory_size - 1) { + continue; + } + + if (start >= mem->start_addr) { + /* The slot starts before section or is aligned to it. */ + offset = start - mem->start_addr; + count = MIN(mem->memory_size - offset, size); + } else { + /* The slot starts after section. */ + offset = 0; + count = MIN(mem->memory_size, size - (mem->start_addr - start)); + } + ret = kvm_log_clear_one_slot(mem, kml->as_id, offset, count); + if (ret < 0) { break; } } - /* - * We should always find one memslot until this point, otherwise - * there could be something wrong from the upper layer - */ - assert(mem && i != s->nr_slots); - ret = kvm_log_clear_one_slot(mem, kml->as_id, start, size); - kvm_slots_unlock(kml); return ret; From 023ae9a88a7cfbdf6f23c3b78ccfcb9b1e26da98 Mon Sep 17 00:00:00 2001 From: Igor Mammedov Date: Tue, 24 Sep 2019 10:47:50 -0400 Subject: [PATCH 10/12] kvm: split too big memory section on several memslots Max memslot size supported by kvm on s390 is 8Tb, move logic of splitting RAM in chunks upto 8T to KVM code. This way it will hide KVM specific restrictions in KVM code and won't affect board level design decisions. Which would allow us to avoid misusing memory_region_allocate_system_memory() API and eventually use a single hostmem backend for guest RAM. Signed-off-by: Igor Mammedov Message-Id: <20190924144751.24149-4-imammedo@redhat.com> Reviewed-by: Peter Xu Acked-by: Paolo Bonzini Signed-off-by: Christian Borntraeger --- accel/kvm/kvm-all.c | 124 +++++++++++++++++++++++++-------------- include/sysemu/kvm_int.h | 1 + 2 files changed, 81 insertions(+), 44 deletions(-) diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index ff9b95c0d103..aabe097c410f 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -140,6 +140,7 @@ bool kvm_direct_msi_allowed; bool kvm_ioeventfd_any_length_allowed; bool kvm_msi_use_devid; static bool kvm_immediate_exit; +static hwaddr kvm_max_slot_size = ~0; static const KVMCapabilityInfo kvm_required_capabilites[] = { KVM_CAP_INFO(USER_MEMORY), @@ -437,7 +438,7 @@ static int kvm_slot_update_flags(KVMMemoryListener *kml, KVMSlot *mem, static int kvm_section_update_flags(KVMMemoryListener *kml, MemoryRegionSection *section) { - hwaddr start_addr, size; + hwaddr start_addr, size, slot_size; KVMSlot *mem; int ret = 0; @@ -448,13 +449,18 @@ static int kvm_section_update_flags(KVMMemoryListener *kml, kvm_slots_lock(kml); - mem = kvm_lookup_matching_slot(kml, start_addr, size); - if (!mem) { - /* We don't have a slot if we want to trap every access. */ - goto out; - } + while (size && !ret) { + slot_size = MIN(kvm_max_slot_size, size); + mem = kvm_lookup_matching_slot(kml, start_addr, slot_size); + if (!mem) { + /* We don't have a slot if we want to trap every access. */ + goto out; + } - ret = kvm_slot_update_flags(kml, mem, section->mr); + ret = kvm_slot_update_flags(kml, mem, section->mr); + start_addr += slot_size; + size -= slot_size; + } out: kvm_slots_unlock(kml); @@ -527,11 +533,15 @@ static int kvm_physical_sync_dirty_bitmap(KVMMemoryListener *kml, struct kvm_dirty_log d = {}; KVMSlot *mem; hwaddr start_addr, size; + hwaddr slot_size, slot_offset = 0; int ret = 0; size = kvm_align_section(section, &start_addr); - if (size) { - mem = kvm_lookup_matching_slot(kml, start_addr, size); + while (size) { + MemoryRegionSection subsection = *section; + + slot_size = MIN(kvm_max_slot_size, size); + mem = kvm_lookup_matching_slot(kml, start_addr, slot_size); if (!mem) { /* We don't have a slot if we want to trap every access. */ goto out; @@ -549,11 +559,11 @@ static int kvm_physical_sync_dirty_bitmap(KVMMemoryListener *kml, * So for now, let's align to 64 instead of HOST_LONG_BITS here, in * a hope that sizeof(long) won't become >8 any time soon. */ - size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS), - /*HOST_LONG_BITS*/ 64) / 8; if (!mem->dirty_bmap) { + hwaddr bitmap_size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS), + /*HOST_LONG_BITS*/ 64) / 8; /* Allocate on the first log_sync, once and for all */ - mem->dirty_bmap = g_malloc0(size); + mem->dirty_bmap = g_malloc0(bitmap_size); } d.dirty_bitmap = mem->dirty_bmap; @@ -564,7 +574,13 @@ static int kvm_physical_sync_dirty_bitmap(KVMMemoryListener *kml, goto out; } - kvm_get_dirty_pages_log_range(section, d.dirty_bitmap); + subsection.offset_within_region += slot_offset; + subsection.size = int128_make64(slot_size); + kvm_get_dirty_pages_log_range(&subsection, d.dirty_bitmap); + + slot_offset += slot_size; + start_addr += slot_size; + size -= slot_size; } out: return ret; @@ -972,6 +988,14 @@ kvm_check_extension_list(KVMState *s, const KVMCapabilityInfo *list) return NULL; } +void kvm_set_max_memslot_size(hwaddr max_slot_size) +{ + g_assert( + ROUND_UP(max_slot_size, qemu_real_host_page_size) == max_slot_size + ); + kvm_max_slot_size = max_slot_size; +} + static void kvm_set_phys_mem(KVMMemoryListener *kml, MemoryRegionSection *section, bool add) { @@ -979,7 +1003,7 @@ static void kvm_set_phys_mem(KVMMemoryListener *kml, int err; MemoryRegion *mr = section->mr; bool writeable = !mr->readonly && !mr->rom_device; - hwaddr start_addr, size; + hwaddr start_addr, size, slot_size; void *ram; if (!memory_region_is_ram(mr)) { @@ -1004,41 +1028,52 @@ static void kvm_set_phys_mem(KVMMemoryListener *kml, kvm_slots_lock(kml); if (!add) { - mem = kvm_lookup_matching_slot(kml, start_addr, size); - if (!mem) { - goto out; - } - if (mem->flags & KVM_MEM_LOG_DIRTY_PAGES) { - kvm_physical_sync_dirty_bitmap(kml, section); - } + do { + slot_size = MIN(kvm_max_slot_size, size); + mem = kvm_lookup_matching_slot(kml, start_addr, slot_size); + if (!mem) { + goto out; + } + if (mem->flags & KVM_MEM_LOG_DIRTY_PAGES) { + kvm_physical_sync_dirty_bitmap(kml, section); + } - /* unregister the slot */ - g_free(mem->dirty_bmap); - mem->dirty_bmap = NULL; - mem->memory_size = 0; - mem->flags = 0; - err = kvm_set_user_memory_region(kml, mem, false); - if (err) { - fprintf(stderr, "%s: error unregistering slot: %s\n", - __func__, strerror(-err)); - abort(); - } + /* unregister the slot */ + g_free(mem->dirty_bmap); + mem->dirty_bmap = NULL; + mem->memory_size = 0; + mem->flags = 0; + err = kvm_set_user_memory_region(kml, mem, false); + if (err) { + fprintf(stderr, "%s: error unregistering slot: %s\n", + __func__, strerror(-err)); + abort(); + } + start_addr += slot_size; + size -= slot_size; + } while (size); goto out; } /* register the new slot */ - mem = kvm_alloc_slot(kml); - mem->memory_size = size; - mem->start_addr = start_addr; - mem->ram = ram; - mem->flags = kvm_mem_flags(mr); - - err = kvm_set_user_memory_region(kml, mem, true); - if (err) { - fprintf(stderr, "%s: error registering slot: %s\n", __func__, - strerror(-err)); - abort(); - } + do { + slot_size = MIN(kvm_max_slot_size, size); + mem = kvm_alloc_slot(kml); + mem->memory_size = slot_size; + mem->start_addr = start_addr; + mem->ram = ram; + mem->flags = kvm_mem_flags(mr); + + err = kvm_set_user_memory_region(kml, mem, true); + if (err) { + fprintf(stderr, "%s: error registering slot: %s\n", __func__, + strerror(-err)); + abort(); + } + start_addr += slot_size; + ram += slot_size; + size -= slot_size; + } while (size); out: kvm_slots_unlock(kml); @@ -2878,6 +2913,7 @@ static bool kvm_accel_has_memory(MachineState *ms, AddressSpace *as, for (i = 0; i < kvm->nr_as; ++i) { if (kvm->as[i].as == as && kvm->as[i].ml) { + size = MIN(kvm_max_slot_size, size); return NULL != kvm_lookup_matching_slot(kvm->as[i].ml, start_addr, size); } diff --git a/include/sysemu/kvm_int.h b/include/sysemu/kvm_int.h index 72b2d1b3aea5..ac2d1f8b5682 100644 --- a/include/sysemu/kvm_int.h +++ b/include/sysemu/kvm_int.h @@ -41,4 +41,5 @@ typedef struct KVMMemoryListener { void kvm_memory_listener_register(KVMState *s, KVMMemoryListener *kml, AddressSpace *as, int as_id); +void kvm_set_max_memslot_size(hwaddr max_slot_size); #endif From fb1fc5a82b836fe1e0d6f5dd57164a82e674ea8a Mon Sep 17 00:00:00 2001 From: Igor Mammedov Date: Tue, 24 Sep 2019 10:47:51 -0400 Subject: [PATCH 11/12] s390: do not call memory_region_allocate_system_memory() multiple times s390 was trying to solve limited KVM memslot size issue by abusing memory_region_allocate_system_memory(), which breaks API contract where the function might be called only once. Beside an invalid use of API, the approach also introduced migration issue, since RAM chunks for each KVM_SLOT_MAX_BYTES are transferred in migration stream as separate RAMBlocks. After discussion [1], it was agreed to break migration from older QEMU for guest with RAM >8Tb (as it was relatively new (since 2.12) and considered to be not actually used downstream). Migration should keep working for guests with less than 8TB and for more than 8TB with QEMU 4.2 and newer binary. In case user tries to migrate more than 8TB guest, between incompatible QEMU versions, migration should fail gracefully due to non-exiting RAMBlock ID or RAMBlock size mismatch. Taking in account above and that now KVM code is able to split too big MemorySection into several memslots, partially revert commit (bb223055b s390-ccw-virtio: allow for systems larger that 7.999TB) and use kvm_set_max_memslot_size() to set KVMSlot size to KVM_SLOT_MAX_BYTES. 1) [PATCH RFC v2 4/4] s390: do not call memory_region_allocate_system_memory() multiple times Signed-off-by: Igor Mammedov Message-Id: <20190924144751.24149-5-imammedo@redhat.com> Acked-by: Peter Xu Signed-off-by: Christian Borntraeger --- hw/s390x/s390-virtio-ccw.c | 30 +++--------------------------- target/s390x/kvm.c | 11 +++++++++++ 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c index 8bfb6684cb72..18ad279a00a3 100644 --- a/hw/s390x/s390-virtio-ccw.c +++ b/hw/s390x/s390-virtio-ccw.c @@ -154,39 +154,15 @@ static void virtio_ccw_register_hcalls(void) virtio_ccw_hcall_early_printk); } -/* - * KVM does only support memory slots up to KVM_MEM_MAX_NR_PAGES pages - * as the dirty bitmap must be managed by bitops that take an int as - * position indicator. If we have a guest beyond that we will split off - * new subregions. The split must happen on a segment boundary (1MB). - */ -#define KVM_MEM_MAX_NR_PAGES ((1ULL << 31) - 1) -#define SEG_MSK (~0xfffffULL) -#define KVM_SLOT_MAX_BYTES ((KVM_MEM_MAX_NR_PAGES * TARGET_PAGE_SIZE) & SEG_MSK) static void s390_memory_init(ram_addr_t mem_size) { MemoryRegion *sysmem = get_system_memory(); - ram_addr_t chunk, offset = 0; - unsigned int number = 0; + MemoryRegion *ram = g_new(MemoryRegion, 1); Error *local_err = NULL; - gchar *name; /* allocate RAM for core */ - name = g_strdup_printf("s390.ram"); - while (mem_size) { - MemoryRegion *ram = g_new(MemoryRegion, 1); - uint64_t size = mem_size; - - /* KVM does not allow memslots >= 8 TB */ - chunk = MIN(size, KVM_SLOT_MAX_BYTES); - memory_region_allocate_system_memory(ram, NULL, name, chunk); - memory_region_add_subregion(sysmem, offset, ram); - mem_size -= chunk; - offset += chunk; - g_free(name); - name = g_strdup_printf("s390.ram.%u", ++number); - } - g_free(name); + memory_region_allocate_system_memory(ram, NULL, "s390.ram", mem_size); + memory_region_add_subregion(sysmem, 0, ram); /* * Configure the maximum page size. As no memory devices were created diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c index 97a662ad0ebf..54864c259c5e 100644 --- a/target/s390x/kvm.c +++ b/target/s390x/kvm.c @@ -28,6 +28,7 @@ #include "cpu.h" #include "internal.h" #include "kvm_s390x.h" +#include "sysemu/kvm_int.h" #include "qapi/error.h" #include "qemu/error-report.h" #include "qemu/timer.h" @@ -122,6 +123,15 @@ */ #define VCPU_IRQ_BUF_SIZE(max_cpus) (sizeof(struct kvm_s390_irq) * \ (max_cpus + NR_LOCAL_IRQS)) +/* + * KVM does only support memory slots up to KVM_MEM_MAX_NR_PAGES pages + * as the dirty bitmap must be managed by bitops that take an int as + * position indicator. If we have a guest beyond that we will split off + * new subregions. The split must happen on a segment boundary (1MB). + */ +#define KVM_MEM_MAX_NR_PAGES ((1ULL << 31) - 1) +#define SEG_MSK (~0xfffffULL) +#define KVM_SLOT_MAX_BYTES ((KVM_MEM_MAX_NR_PAGES * TARGET_PAGE_SIZE) & SEG_MSK) static CPUWatchpoint hw_watchpoint; /* @@ -355,6 +365,7 @@ int kvm_arch_init(MachineState *ms, KVMState *s) */ /* kvm_vm_enable_cap(s, KVM_CAP_S390_AIS, 0); */ + kvm_set_max_memslot_size(KVM_SLOT_MAX_BYTES); return 0; } From c5b9ce518c0551d0198bcddadc82e03de9ac8de9 Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Wed, 25 Sep 2019 09:45:50 +0200 Subject: [PATCH 12/12] s390/kvm: split kvm mem slots at 4TB Instead of splitting at an unaligned address, we can simply split at 4TB. Signed-off-by: Christian Borntraeger Acked-by: Igor Mammedov --- target/s390x/kvm.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c index 54864c259c5e..c24c869e7703 100644 --- a/target/s390x/kvm.c +++ b/target/s390x/kvm.c @@ -126,12 +126,11 @@ /* * KVM does only support memory slots up to KVM_MEM_MAX_NR_PAGES pages * as the dirty bitmap must be managed by bitops that take an int as - * position indicator. If we have a guest beyond that we will split off - * new subregions. The split must happen on a segment boundary (1MB). + * position indicator. This would end at an unaligned address + * (0x7fffff00000). As future variants might provide larger pages + * and to make all addresses properly aligned, let us split at 4TB. */ -#define KVM_MEM_MAX_NR_PAGES ((1ULL << 31) - 1) -#define SEG_MSK (~0xfffffULL) -#define KVM_SLOT_MAX_BYTES ((KVM_MEM_MAX_NR_PAGES * TARGET_PAGE_SIZE) & SEG_MSK) +#define KVM_SLOT_MAX_BYTES (4UL * TiB) static CPUWatchpoint hw_watchpoint; /*