From 390699f93dd8f52b35cac6b113651f10318c5fba Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 22 May 2025 18:08:25 +0100 Subject: [PATCH] ANDROID: KVM: arm64: Add host_split_guest for pKVM In preparation for supporting relinquish of memory mapped at the stage-2 by a huge mapping, add a HVC to break a block mapping into last level mappings. Bug: 419548963 Bug: 278011447 Change-Id: If2c4e375b26d8015b770f7710ecccbb7c6ed284a Signed-off-by: Vincent Donnefort --- arch/arm64/include/asm/kvm_asm.h | 1 + arch/arm64/include/asm/kvm_pgtable.h | 3 +- arch/arm64/kvm/hyp/include/nvhe/mem_protect.h | 1 + arch/arm64/kvm/hyp/nvhe/hyp-main.c | 22 ++++++++++ arch/arm64/kvm/hyp/nvhe/mem_protect.c | 24 ++++++++++ arch/arm64/kvm/hyp/pgtable.c | 44 +++++++++++++++++-- 6 files changed, 89 insertions(+), 6 deletions(-) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 34a2e60525e6..9cc0033513a9 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -83,6 +83,7 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___pkvm_relax_perms, __KVM_HOST_SMCCC_FUNC___pkvm_wrprotect, __KVM_HOST_SMCCC_FUNC___pkvm_dirty_log, + __KVM_HOST_SMCCC_FUNC___pkvm_host_split_guest, __KVM_HOST_SMCCC_FUNC___pkvm_tlb_flush_vmid, __KVM_HOST_SMCCC_FUNC___kvm_adjust_pc, __KVM_HOST_SMCCC_FUNC___kvm_vcpu_run, diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h index 4c521003ad22..90c647d4b329 100644 --- a/arch/arm64/include/asm/kvm_pgtable.h +++ b/arch/arm64/include/asm/kvm_pgtable.h @@ -862,8 +862,7 @@ int kvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size); * kvm_pgtable_stage2_split() is best effort: it tries to break as many * blocks in the input range as allowed by @mc_capacity. */ -int kvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size, - struct kvm_mmu_memory_cache *mc); +int kvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size, void *mc); /** * kvm_pgtable_walk() - Walk a page-table. diff --git a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h index 40d9b7341097..5c02f8c8fb06 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h @@ -63,6 +63,7 @@ int __pkvm_host_unuse_dma(u64 phys_addr, size_t size); int __pkvm_guest_stage2_snapshot(struct kvm_pgtable_snapshot *snap, struct pkvm_hyp_vm *vm); int __pkvm_host_stage2_snapshot(struct kvm_pgtable_snapshot *snap); int __pkvm_host_lazy_pte(u64 pfn, u64 nr_pages, bool enable); +int __pkvm_host_split_guest(u64 pfn, u64 gfn, u64 size, struct pkvm_hyp_vcpu *vcpu); bool addr_is_memory(phys_addr_t phys); int host_stage2_idmap_locked(phys_addr_t addr, u64 size, diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index e8c484dcc769..a5ffc4cd3f70 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -1073,6 +1073,27 @@ out: cpu_reg(host_ctxt, 1) = ret; } +static void handle___pkvm_host_split_guest(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(u64, pfn, host_ctxt, 1); + DECLARE_REG(u64, gfn, host_ctxt, 2); + DECLARE_REG(u64, size, host_ctxt, 3); + struct pkvm_hyp_vcpu *hyp_vcpu; + int ret = -EINVAL; + + if (!is_protected_kvm_enabled()) + goto out; + + hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); + if (!hyp_vcpu) + goto out; + + ret = __pkvm_host_split_guest(pfn, gfn, size, hyp_vcpu); + +out: + cpu_reg(host_ctxt, 1) = ret; +} + static void handle___kvm_adjust_pc(struct kvm_cpu_context *host_ctxt) { struct pkvm_hyp_vcpu *hyp_vcpu; @@ -1618,6 +1639,7 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__pkvm_relax_perms), HANDLE_FUNC(__pkvm_wrprotect), HANDLE_FUNC(__pkvm_dirty_log), + HANDLE_FUNC(__pkvm_host_split_guest), HANDLE_FUNC(__pkvm_tlb_flush_vmid), HANDLE_FUNC(__kvm_adjust_pc), HANDLE_FUNC(__kvm_vcpu_run), diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index 1981ce61acd4..994609939db6 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -2764,6 +2764,30 @@ unlock: } +int __pkvm_host_split_guest(u64 pfn, u64 gfn, u64 size, struct pkvm_hyp_vcpu *vcpu) +{ + struct kvm_hyp_memcache *mc = &vcpu->vcpu.arch.stage2_mc; + struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu); + u64 ipa = hyp_pfn_to_phys(gfn); + int ret; + + if (size != PMD_SIZE) + return -EINVAL; + + guest_lock_component(vm); + + /* + * stage2_split() already checks the existing mapping is valid and PMD-level. + * No other check is necessary. + */ + + ret = kvm_pgtable_stage2_split(&vm->pgt, ipa, size, mc); + + guest_unlock_component(vm); + + return ret; +} + int __pkvm_host_donate_guest(struct pkvm_hyp_vcpu *vcpu, u64 pfn, u64 gfn, u64 nr_pages) { diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index d337e9349a65..3dab9deb3415 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -1769,13 +1769,49 @@ static int stage2_split_walker(const struct kvm_pgtable_visit_ctx *ctx, return 0; } -int kvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size, - struct kvm_mmu_memory_cache *mc) +static int pkvm_stage2_split_walker(const struct kvm_pgtable_visit_ctx *ctx, + enum kvm_pgtable_walk_flags visit) { + struct stage2_map_data *data = ctx->arg; + struct kvm_pgtable *pgt = data->mmu->pgt; + struct kvm_hyp_memcache *mc = data->memcache; + enum kvm_pgtable_prot prot; + kvm_pte_t pte = ctx->old; + kvm_pte_t *childp; + + if (ctx->level == KVM_PGTABLE_MAX_LEVELS - 1) + return 0; + + /* We can only split PMD-level blocks */ + if (!kvm_pte_valid(pte) || ctx->level != KVM_PGTABLE_MAX_LEVELS - 2) + return -EINVAL; + + prot = kvm_pgtable_stage2_pte_prot(pte); + childp = kvm_pgtable_stage2_create_unlinked(pgt, kvm_pte_to_phys(pte), + ctx->level, prot, mc, true); + if (IS_ERR(childp)) + return PTR_ERR(childp); + + WARN_ON(!stage2_try_break_pte(ctx, data->mmu)); + + stage2_make_pte(ctx, kvm_init_table_pte(childp, ctx->mm_ops)); + dsb(ishst); + + return 0; +} + +int kvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size, void *mc) +{ + struct stage2_map_data data = { + .mmu = pgt->mmu, + .memcache = mc, + }; struct kvm_pgtable_walker walker = { - .cb = stage2_split_walker, + .cb = static_branch_unlikely(&kvm_protected_mode_initialized) ? + pkvm_stage2_split_walker : stage2_split_walker, + .arg = static_branch_unlikely(&kvm_protected_mode_initialized) ? + &data : mc, .flags = KVM_PGTABLE_WALK_LEAF, - .arg = mc, }; return kvm_pgtable_walk(pgt, addr, size, &walker);