From 7c95a219c04c1fc1148a3ae62bffd2026cff30d4 Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 22 May 2025 18:15:11 +0100 Subject: [PATCH] ANDROID: KVM: arm64: Add hyp request SPLIT This request asks the host to split its order 9 kvm_pinned_page. Once done it also issues a host_split_guest HVC. This intends to be used on the back of a guest relinquish memory when a stage-2 block mapping is hit. Bug: 419548963 Bug: 278011447 Change-Id: I64b18131b42a09c8f34127a1db802753a5aae8b5 Signed-off-by: Vincent Donnefort --- android/abi_gki_aarch64.stg | 14 +- android/abi_gki_aarch64.stg.allowed_breaks | 5 + arch/arm64/include/asm/kvm_host.h | 12 +- arch/arm64/include/asm/kvm_mmu.h | 1 + arch/arm64/kvm/handle_exit.c | 8 ++ arch/arm64/kvm/mmu.c | 156 +++++++++++++++++++++ arch/arm64/kvm/pkvm.c | 4 +- 7 files changed, 193 insertions(+), 7 deletions(-) diff --git a/android/abi_gki_aarch64.stg b/android/abi_gki_aarch64.stg index 6b0b35306559..f3b75195a4e3 100644 --- a/android/abi_gki_aarch64.stg +++ b/android/abi_gki_aarch64.stg @@ -42943,8 +42943,8 @@ member { offset: 704 } member { - id: 0x335fcb96 - type_id: 0x42514dcd + id: 0x335fcf90 + type_id: 0x42515dd6 offset: 64 } member { @@ -189320,6 +189320,11 @@ member { type_id: 0x0f07cf2c offset: 1408 } +member { + id: 0x406e59a2 + name: "split" + type_id: 0x3721bb90 +} member { id: 0x406ecc6d name: "split" @@ -224306,12 +224311,13 @@ struct_union { } } struct_union { - id: 0x42514dcd + id: 0x42515dd6 kind: UNION definition { bytesize: 16 member_id: 0x52238160 member_id: 0x8dc1500c + member_id: 0x406e59a2 } } struct_union { @@ -253209,7 +253215,7 @@ struct_union { definition { bytesize: 24 member_id: 0x5c7f890c - member_id: 0x335fcb96 + member_id: 0x335fcf90 } } struct_union { diff --git a/android/abi_gki_aarch64.stg.allowed_breaks b/android/abi_gki_aarch64.stg.allowed_breaks index efce896d03cc..ca4b24e9dc52 100644 --- a/android/abi_gki_aarch64.stg.allowed_breaks +++ b/android/abi_gki_aarch64.stg.allowed_breaks @@ -136,3 +136,8 @@ type 'struct kvm_protected_vm' changed member 'struct maple_tree pinned_pages' was removed member 'union { struct rb_root_cached pinned_pages; struct { struct maple_tree __unused; }; union { }; }' was added +type 'struct kvm_hyp_req' changed + member changed from 'union { struct { u8 dest; int nr_pages; int sz_alloc; } mem; struct { unsigned long guest_ipa; size_t size; } map; }' to 'union { struct { u8 dest; int nr_pages; int sz_alloc; } mem; struct { unsigned long guest_ipa; size_t size; } map; struct { unsigned long guest_ipa; size_t size; } split; }' + type changed from 'union { struct { u8 dest; int nr_pages; int sz_alloc; } mem; struct { unsigned long guest_ipa; size_t size; } map; }' to 'union { struct { u8 dest; int nr_pages; int sz_alloc; } mem; struct { unsigned long guest_ipa; size_t size; } map; struct { unsigned long guest_ipa; size_t size; } split; }' + member 'struct { unsigned long guest_ipa; size_t size; } split' was added + diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 68998b23221d..e4504048783c 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -224,7 +224,10 @@ struct kvm_smccc_features { }; struct kvm_pinned_page { - struct rb_node node; + union { + struct rb_node node; + struct list_head list_node; + }; struct page *page; u64 ipa; u64 __subtree_last; @@ -538,6 +541,7 @@ struct kvm_hyp_req { #define KVM_HYP_LAST_REQ 0 #define KVM_HYP_REQ_TYPE_MEM 1 #define KVM_HYP_REQ_TYPE_MAP 2 +#define KVM_HYP_REQ_TYPE_SPLIT 3 u8 type; union { struct { @@ -552,6 +556,12 @@ struct kvm_hyp_req { unsigned long guest_ipa; size_t size; } map; +#ifndef __GENKSYMS__ + struct { + unsigned long guest_ipa; + size_t size; + } split; +#endif }; }; diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h index 03122dbd5ac2..a174ca1b9e66 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -184,6 +184,7 @@ int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa, int kvm_handle_guest_abort(struct kvm_vcpu *vcpu); int pkvm_mem_abort_range(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, size_t size); +int __pkvm_pgtable_stage2_split(struct kvm_vcpu *vcpu, phys_addr_t ipa, size_t size); phys_addr_t kvm_mmu_get_httbr(void); phys_addr_t kvm_get_idmap_vector(void); diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c index 92d82b2b4bc7..0179275ac299 100644 --- a/arch/arm64/kvm/handle_exit.c +++ b/arch/arm64/kvm/handle_exit.c @@ -363,6 +363,11 @@ static int handle_hyp_req_map(struct kvm_vcpu *vcpu, return pkvm_mem_abort_range(vcpu, req->map.guest_ipa, req->map.size); } +static int handle_hyp_req_split(struct kvm_vcpu *vcpu, struct kvm_hyp_req *req) +{ + return __pkvm_pgtable_stage2_split(vcpu, req->split.guest_ipa, req->split.size); +} + static int handle_hyp_req(struct kvm_vcpu *vcpu) { struct kvm_hyp_req *hyp_req = vcpu->arch.hyp_reqs; @@ -379,6 +384,9 @@ static int handle_hyp_req(struct kvm_vcpu *vcpu) case KVM_HYP_REQ_TYPE_MAP: ret = handle_hyp_req_map(vcpu, hyp_req); break; + case KVM_HYP_REQ_TYPE_SPLIT: + ret = handle_hyp_req_split(vcpu, hyp_req); + break; default: pr_warn("Unknown kvm_hyp_req type: %d\n", hyp_req->type); ret = -EINVAL; diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 213957709010..38d25bab1057 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1851,6 +1851,162 @@ end: return err; } +static int __pkvm_pin_user_pages(struct kvm *kvm, struct kvm_memory_slot *memslot, + u64 gfn, u64 nr_pages, struct page ***__pages) +{ + unsigned long hva = gfn_to_hva_memslot_prot(memslot, gfn, NULL); + unsigned int flags = FOLL_HWPOISON | FOLL_LONGTERM | FOLL_WRITE; + struct mm_struct *mm = current->mm; + struct page **pages; + long ret; + int p; + + pages = kmalloc_array(nr_pages, sizeof(*pages), GFP_KERNEL); + if (!pages) + return -ENOMEM; + + mmap_read_lock(mm); + ret = pin_user_pages(hva, nr_pages, flags, pages); + mmap_read_unlock(mm); + + if (ret == -EHWPOISON) { + kvm_send_hwpoison_signal(hva, PAGE_SHIFT); + goto err_free_pages; + } else if (ret == -EFAULT) { + /* Will try MMIO map */ + ret = -EREMOTEIO; + goto err_free_pages; + } else if (ret < 0) { + ret = -EFAULT; + goto err_free_pages; + } else if (ret != nr_pages) { + nr_pages = ret; + ret = -EFAULT; + goto err_unpin_pages; + } + + /* See PageSwapBacked() in pkvm_mem_abort() */ + for (p = 0; p < nr_pages; p++) { + if (!folio_test_swapbacked(page_folio(pages[p]))) { + ret = -EIO; + goto err_unpin_pages; + } + } + + *__pages = pages; + return 0; + +err_unpin_pages: + unpin_user_pages(pages, nr_pages); +err_free_pages: + kfree(pages); + return ret; +} + +/* + * Splitting is only expected on the back of a relinquish guest HVC in the pKVM case, while + * pkvm_pgtable_stage2_split() can be called with dirty logging. + */ +int __pkvm_pgtable_stage2_split(struct kvm_vcpu *vcpu, phys_addr_t ipa, size_t size) +{ + struct list_head ppage_prealloc = LIST_HEAD_INIT(ppage_prealloc); + struct kvm_hyp_memcache *hyp_memcache = &vcpu->arch.stage2_mc; + struct kvm_pinned_page *ppage, *tmp; + struct kvm_memory_slot *memslot; + struct kvm *kvm = vcpu->kvm; + int idx, p, ret, nr_pages; + struct page **pages; + kvm_pfn_t pfn; + gfn_t gfn; + + if (!IS_ALIGNED(ipa, PMD_SIZE) || size != PMD_SIZE) + return -EINVAL; + + if (!hyp_memcache->nr_pages) { + ret = topup_hyp_memcache(hyp_memcache, 1, 0); + if (ret) + return -ENOMEM; + + atomic64_add(PAGE_SIZE, &kvm->stat.protected_hyp_mem); + atomic64_add(PAGE_SIZE, &kvm->stat.protected_pgtable_mem); + } + + /* We already have 1 pin on the Huge Page */ + nr_pages = (size >> PAGE_SHIFT) - 1; + gfn = (ipa >> PAGE_SHIFT) + 1; + + /* Pre-allocate kvm_pinned_page before acquiring the mmu_lock */ + for (p = 0; p < nr_pages; p++) { + ppage = kzalloc(sizeof(*ppage), GFP_KERNEL_ACCOUNT); + if (!ppage) { + ret = -ENOMEM; + goto free_pinned_pages; + } + list_add(&ppage->list_node, &ppage_prealloc); + } + + idx = srcu_read_lock(&vcpu->kvm->srcu); + memslot = gfn_to_memslot(vcpu->kvm, gfn); + ret = __pkvm_pin_user_pages(kvm, memslot, gfn, nr_pages, &pages); + if (ret) + goto unlock_srcu; + + write_lock(&kvm->mmu_lock); + + ppage = find_ppage(kvm, ipa); + if (!ppage) { + ret = -EPERM; + goto end; + } else if (!ppage->order) { + ret = 0; + goto end; + } + + ret = kvm_call_hyp_nvhe(__pkvm_host_split_guest, page_to_pfn(ppage->page), + ipa >> PAGE_SHIFT, size); + if (ret) + goto end; + + ppage->order = 0; + ppage->pins = 1; + + pfn = page_to_pfn(ppage->page) + 1; + ipa = ipa + PAGE_SIZE; + while (nr_pages--) { + /* Pop a ppage from the pre-allocated list */ + ppage = list_first_entry(&ppage_prealloc, struct kvm_pinned_page, list_node); + list_del_init(&ppage->list_node); + + ppage->page = pfn_to_page(pfn); + ppage->ipa = ipa; + ppage->order = 0; + ppage->pins = 1; + kvm_pinned_pages_insert(ppage, &kvm->arch.pkvm.pinned_pages); + + pfn += 1; + ipa += PAGE_SIZE; + } + +end: + write_unlock(&kvm->mmu_lock); + + if (ret) + unpin_user_pages(pages, nr_pages); + kfree(pages); + +unlock_srcu: + srcu_read_unlock(&vcpu->kvm->srcu, idx); + +free_pinned_pages: + /* Free unused pre-allocated kvm_pinned_page */ + list_for_each_entry_safe(ppage, tmp, &ppage_prealloc, list_node) { + list_del(&ppage->list_node); + kfree(ppage); + } + + return ret; +} + static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, struct kvm_memory_slot *memslot, unsigned long hva, unsigned long fault_status) diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 22f31d53c3e0..4523cc6f2725 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -540,10 +540,10 @@ void pkvm_host_reclaim_page(struct kvm *host_kvm, phys_addr_t ipa) ppage = kvm_pinned_pages_iter_first(&host_kvm->arch.pkvm.pinned_pages, ipa, ipa + PAGE_SIZE - 1); if (ppage) { + WARN_ON_ONCE(ppage->pins != 1); + if (ppage->pins) ppage->pins--; - else - WARN_ON(1); pins = ppage->pins; if (!pins)