Skip to content

Commit a889ea5

Browse files
Ben Gardonbonzini
Ben Gardon
authored and
bonzini
committed
KVM: x86/mmu: Ensure TDP MMU roots are freed after yield
Many TDP MMU functions which need to perform some action on all TDP MMU roots hold a reference on that root so that they can safely drop the MMU lock in order to yield to other threads. However, when releasing the reference on the root, there is a bug: the root will not be freed even if its reference count (root_count) is reduced to 0. To simplify acquiring and releasing references on TDP MMU root pages, and to ensure that these roots are properly freed, move the get/put operations into another TDP MMU root iterator macro. Moving the get/put operations into an iterator macro also helps simplify control flow when a root does need to be freed. Note that using the list_for_each_entry_safe macro would not have been appropriate in this situation because it could keep a pointer to the next root across an MMU lock release + reacquire, during which time that root could be freed. Reported-by: Maciej S. Szmigiero <maciej.szmigiero@oracle.com> Suggested-by: Paolo Bonzini <pbonzini@redhat.com> Fixes: faaf05b ("kvm: x86/mmu: Support zapping SPTEs in the TDP MMU") Fixes: 063afac ("kvm: x86/mmu: Support invalidate range MMU notifier for TDP MMU") Fixes: a6a0b05 ("kvm: x86/mmu: Support dirty logging for the TDP MMU") Fixes: 1488199 ("kvm: x86/mmu: Support disabling dirty logging for the tdp MMU") Signed-off-by: Ben Gardon <bgardon@google.com> Message-Id: <20210107001935.3732070-1-bgardon@google.com> Cc: stable@vger.kernel.org Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
1 parent 88bf56d commit a889ea5

File tree

1 file changed

+48
-56
lines changed

1 file changed

+48
-56
lines changed

‎arch/x86/kvm/mmu/tdp_mmu.c‎

Lines changed: 48 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,48 @@ void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm)
4444
WARN_ON(!list_empty(&kvm->arch.tdp_mmu_roots));
4545
}
4646

47-
#define for_each_tdp_mmu_root(_kvm, _root) \
47+
static void tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root)
48+
{
49+
if (kvm_mmu_put_root(kvm, root))
50+
kvm_tdp_mmu_free_root(kvm, root);
51+
}
52+
53+
static inline bool tdp_mmu_next_root_valid(struct kvm *kvm,
54+
struct kvm_mmu_page *root)
55+
{
56+
lockdep_assert_held(&kvm->mmu_lock);
57+
58+
if (list_entry_is_head(root, &kvm->arch.tdp_mmu_roots, link))
59+
return false;
60+
61+
kvm_mmu_get_root(kvm, root);
62+
return true;
63+
64+
}
65+
66+
static inline struct kvm_mmu_page *tdp_mmu_next_root(struct kvm *kvm,
67+
struct kvm_mmu_page *root)
68+
{
69+
struct kvm_mmu_page *next_root;
70+
71+
next_root = list_next_entry(root, link);
72+
tdp_mmu_put_root(kvm, root);
73+
return next_root;
74+
}
75+
76+
/*
77+
* Note: this iterator gets and puts references to the roots it iterates over.
78+
* This makes it safe to release the MMU lock and yield within the loop, but
79+
* if exiting the loop early, the caller must drop the reference to the most
80+
* recent root. (Unless keeping a live reference is desirable.)
81+
*/
82+
#define for_each_tdp_mmu_root_yield_safe(_kvm, _root) \
83+
for (_root = list_first_entry(&_kvm->arch.tdp_mmu_roots, \
84+
typeof(*_root), link); \
85+
tdp_mmu_next_root_valid(_kvm, _root); \
86+
_root = tdp_mmu_next_root(_kvm, _root))
87+
88+
#define for_each_tdp_mmu_root(_kvm, _root) \
4889
list_for_each_entry(_root, &_kvm->arch.tdp_mmu_roots, link)
4990

5091
bool is_tdp_mmu_root(struct kvm *kvm, hpa_t hpa)
@@ -447,18 +488,9 @@ bool kvm_tdp_mmu_zap_gfn_range(struct kvm *kvm, gfn_t start, gfn_t end)
447488
struct kvm_mmu_page *root;
448489
bool flush = false;
449490

450-
for_each_tdp_mmu_root(kvm, root) {
451-
/*
452-
* Take a reference on the root so that it cannot be freed if
453-
* this thread releases the MMU lock and yields in this loop.
454-
*/
455-
kvm_mmu_get_root(kvm, root);
456-
491+
for_each_tdp_mmu_root_yield_safe(kvm, root)
457492
flush |= zap_gfn_range(kvm, root, start, end, true);
458493

459-
kvm_mmu_put_root(kvm, root);
460-
}
461-
462494
return flush;
463495
}
464496

@@ -619,13 +651,7 @@ static int kvm_tdp_mmu_handle_hva_range(struct kvm *kvm, unsigned long start,
619651
int ret = 0;
620652
int as_id;
621653

622-
for_each_tdp_mmu_root(kvm, root) {
623-
/*
624-
* Take a reference on the root so that it cannot be freed if
625-
* this thread releases the MMU lock and yields in this loop.
626-
*/
627-
kvm_mmu_get_root(kvm, root);
628-
654+
for_each_tdp_mmu_root_yield_safe(kvm, root) {
629655
as_id = kvm_mmu_page_as_id(root);
630656
slots = __kvm_memslots(kvm, as_id);
631657
kvm_for_each_memslot(memslot, slots) {
@@ -647,8 +673,6 @@ static int kvm_tdp_mmu_handle_hva_range(struct kvm *kvm, unsigned long start,
647673
ret |= handler(kvm, memslot, root, gfn_start,
648674
gfn_end, data);
649675
}
650-
651-
kvm_mmu_put_root(kvm, root);
652676
}
653677

654678
return ret;
@@ -838,21 +862,13 @@ bool kvm_tdp_mmu_wrprot_slot(struct kvm *kvm, struct kvm_memory_slot *slot,
838862
int root_as_id;
839863
bool spte_set = false;
840864

841-
for_each_tdp_mmu_root(kvm, root) {
865+
for_each_tdp_mmu_root_yield_safe(kvm, root) {
842866
root_as_id = kvm_mmu_page_as_id(root);
843867
if (root_as_id != slot->as_id)
844868
continue;
845869

846-
/*
847-
* Take a reference on the root so that it cannot be freed if
848-
* this thread releases the MMU lock and yields in this loop.
849-
*/
850-
kvm_mmu_get_root(kvm, root);
851-
852870
spte_set |= wrprot_gfn_range(kvm, root, slot->base_gfn,
853871
slot->base_gfn + slot->npages, min_level);
854-
855-
kvm_mmu_put_root(kvm, root);
856872
}
857873

858874
return spte_set;
@@ -906,21 +922,13 @@ bool kvm_tdp_mmu_clear_dirty_slot(struct kvm *kvm, struct kvm_memory_slot *slot)
906922
int root_as_id;
907923
bool spte_set = false;
908924

909-
for_each_tdp_mmu_root(kvm, root) {
925+
for_each_tdp_mmu_root_yield_safe(kvm, root) {
910926
root_as_id = kvm_mmu_page_as_id(root);
911927
if (root_as_id != slot->as_id)
912928
continue;
913929

914-
/*
915-
* Take a reference on the root so that it cannot be freed if
916-
* this thread releases the MMU lock and yields in this loop.
917-
*/
918-
kvm_mmu_get_root(kvm, root);
919-
920930
spte_set |= clear_dirty_gfn_range(kvm, root, slot->base_gfn,
921931
slot->base_gfn + slot->npages);
922-
923-
kvm_mmu_put_root(kvm, root);
924932
}
925933

926934
return spte_set;
@@ -1029,21 +1037,13 @@ bool kvm_tdp_mmu_slot_set_dirty(struct kvm *kvm, struct kvm_memory_slot *slot)
10291037
int root_as_id;
10301038
bool spte_set = false;
10311039

1032-
for_each_tdp_mmu_root(kvm, root) {
1040+
for_each_tdp_mmu_root_yield_safe(kvm, root) {
10331041
root_as_id = kvm_mmu_page_as_id(root);
10341042
if (root_as_id != slot->as_id)
10351043
continue;
10361044

1037-
/*
1038-
* Take a reference on the root so that it cannot be freed if
1039-
* this thread releases the MMU lock and yields in this loop.
1040-
*/
1041-
kvm_mmu_get_root(kvm, root);
1042-
10431045
spte_set |= set_dirty_gfn_range(kvm, root, slot->base_gfn,
10441046
slot->base_gfn + slot->npages);
1045-
1046-
kvm_mmu_put_root(kvm, root);
10471047
}
10481048
return spte_set;
10491049
}
@@ -1089,21 +1089,13 @@ void kvm_tdp_mmu_zap_collapsible_sptes(struct kvm *kvm,
10891089
struct kvm_mmu_page *root;
10901090
int root_as_id;
10911091

1092-
for_each_tdp_mmu_root(kvm, root) {
1092+
for_each_tdp_mmu_root_yield_safe(kvm, root) {
10931093
root_as_id = kvm_mmu_page_as_id(root);
10941094
if (root_as_id != slot->as_id)
10951095
continue;
10961096

1097-
/*
1098-
* Take a reference on the root so that it cannot be freed if
1099-
* this thread releases the MMU lock and yields in this loop.
1100-
*/
1101-
kvm_mmu_get_root(kvm, root);
1102-
11031097
zap_collapsible_spte_range(kvm, root, slot->base_gfn,
11041098
slot->base_gfn + slot->npages);
1105-
1106-
kvm_mmu_put_root(kvm, root);
11071099
}
11081100
}
11091101

0 commit comments

Comments
 (0)