197 lines
6.2 KiB
C
197 lines
6.2 KiB
C
/* SPDX-License-Identifier: GPL-2.0-only */
|
|
#ifndef IO_PGTABLE_H_
|
|
#define IO_PGTABLE_H_
|
|
|
|
#include <linux/io-pgtable.h>
|
|
|
|
extern bool selftest_running;
|
|
|
|
typedef u64 arm_lpae_iopte;
|
|
|
|
struct arm_lpae_io_pgtable {
|
|
struct io_pgtable iop;
|
|
|
|
int pgd_bits;
|
|
int start_level;
|
|
int bits_per_level;
|
|
|
|
void *pgd;
|
|
|
|
bool idmapped; /* Used by hypervisor */
|
|
};
|
|
|
|
/* Struct accessors */
|
|
#define io_pgtable_to_data(x) \
|
|
container_of((x), struct arm_lpae_io_pgtable, iop)
|
|
|
|
#define io_pgtable_ops_to_data(x) \
|
|
io_pgtable_to_data(io_pgtable_ops_to_pgtable(x))
|
|
|
|
/*
|
|
* Calculate the right shift amount to get to the portion describing level l
|
|
* in a virtual address mapped by the pagetable in d.
|
|
*/
|
|
#define ARM_LPAE_LVL_SHIFT(l,d) \
|
|
(((ARM_LPAE_MAX_LEVELS - (l)) * (d)->bits_per_level) + \
|
|
ilog2(sizeof(arm_lpae_iopte)))
|
|
|
|
#define ARM_LPAE_GRANULE(d) \
|
|
(sizeof(arm_lpae_iopte) << (d)->bits_per_level)
|
|
#define ARM_LPAE_PGD_SIZE(d) \
|
|
(sizeof(arm_lpae_iopte) << (d)->pgd_bits)
|
|
|
|
#define ARM_LPAE_PTES_PER_TABLE(d) \
|
|
(ARM_LPAE_GRANULE(d) >> ilog2(sizeof(arm_lpae_iopte)))
|
|
|
|
/*
|
|
* Calculate the index at level l used to map virtual address a using the
|
|
* pagetable in d.
|
|
*/
|
|
#define ARM_LPAE_PGD_IDX(l,d) \
|
|
((l) == (d)->start_level ? (d)->pgd_bits - (d)->bits_per_level : 0)
|
|
|
|
#define ARM_LPAE_LVL_IDX(a,l,d) \
|
|
(((u64)(a) >> ARM_LPAE_LVL_SHIFT(l,d)) & \
|
|
((1 << ((d)->bits_per_level + ARM_LPAE_PGD_IDX(l,d))) - 1))
|
|
|
|
/* Calculate the block/page mapping size at level l for pagetable in d. */
|
|
#define ARM_LPAE_BLOCK_SIZE(l,d) (1ULL << ARM_LPAE_LVL_SHIFT(l,d))
|
|
|
|
/* Page table bits */
|
|
#define ARM_LPAE_PTE_TYPE_SHIFT 0
|
|
#define ARM_LPAE_PTE_TYPE_MASK 0x3
|
|
|
|
#define ARM_LPAE_PTE_TYPE_BLOCK 1
|
|
#define ARM_LPAE_PTE_TYPE_TABLE 3
|
|
#define ARM_LPAE_PTE_TYPE_PAGE 3
|
|
|
|
#define ARM_LPAE_PTE_ADDR_MASK GENMASK_ULL(47,12)
|
|
|
|
#define ARM_LPAE_PTE_NSTABLE (((arm_lpae_iopte)1) << 63)
|
|
#define ARM_LPAE_PTE_XN (((arm_lpae_iopte)3) << 53)
|
|
#define ARM_LPAE_PTE_AF (((arm_lpae_iopte)1) << 10)
|
|
#define ARM_LPAE_PTE_SH_NS (((arm_lpae_iopte)0) << 8)
|
|
#define ARM_LPAE_PTE_SH_OS (((arm_lpae_iopte)2) << 8)
|
|
#define ARM_LPAE_PTE_SH_IS (((arm_lpae_iopte)3) << 8)
|
|
#define ARM_LPAE_PTE_NS (((arm_lpae_iopte)1) << 5)
|
|
#define ARM_LPAE_PTE_VALID (((arm_lpae_iopte)1) << 0)
|
|
|
|
#define ARM_LPAE_PTE_ATTR_LO_MASK (((arm_lpae_iopte)0x3ff) << 2)
|
|
/* Ignore the contiguous bit for block splitting */
|
|
#define ARM_LPAE_PTE_ATTR_HI_MASK (((arm_lpae_iopte)6) << 52)
|
|
#define ARM_LPAE_PTE_ATTR_MASK (ARM_LPAE_PTE_ATTR_LO_MASK | \
|
|
ARM_LPAE_PTE_ATTR_HI_MASK)
|
|
/* Software bit for solving coherency races */
|
|
#define ARM_LPAE_PTE_SW_SYNC (((arm_lpae_iopte)1) << 55)
|
|
|
|
/* Stage-1 PTE */
|
|
#define ARM_LPAE_PTE_AP_UNPRIV (((arm_lpae_iopte)1) << 6)
|
|
#define ARM_LPAE_PTE_AP_RDONLY (((arm_lpae_iopte)2) << 6)
|
|
#define ARM_LPAE_PTE_ATTRINDX_SHIFT 2
|
|
#define ARM_LPAE_PTE_nG (((arm_lpae_iopte)1) << 11)
|
|
|
|
/* Stage-2 PTE */
|
|
#define ARM_LPAE_PTE_HAP_FAULT (((arm_lpae_iopte)0) << 6)
|
|
#define ARM_LPAE_PTE_HAP_READ (((arm_lpae_iopte)1) << 6)
|
|
#define ARM_LPAE_PTE_HAP_WRITE (((arm_lpae_iopte)2) << 6)
|
|
#define ARM_LPAE_PTE_MEMATTR_OIWB (((arm_lpae_iopte)0xf) << 2)
|
|
#define ARM_LPAE_PTE_MEMATTR_NC (((arm_lpae_iopte)0x5) << 2)
|
|
#define ARM_LPAE_PTE_MEMATTR_DEV (((arm_lpae_iopte)0x1) << 2)
|
|
|
|
/* Register bits */
|
|
#define ARM_LPAE_VTCR_SL0_MASK 0x3
|
|
|
|
#define ARM_LPAE_TCR_T0SZ_SHIFT 0
|
|
|
|
#define ARM_LPAE_TCR_TG0_4K 0
|
|
#define ARM_LPAE_TCR_TG0_64K 1
|
|
#define ARM_LPAE_TCR_TG0_16K 2
|
|
|
|
#define ARM_LPAE_TCR_TG1_16K 1
|
|
#define ARM_LPAE_TCR_TG1_4K 2
|
|
#define ARM_LPAE_TCR_TG1_64K 3
|
|
|
|
#define ARM_LPAE_TCR_SH_NS 0
|
|
#define ARM_LPAE_TCR_SH_OS 2
|
|
#define ARM_LPAE_TCR_SH_IS 3
|
|
|
|
#define ARM_LPAE_TCR_RGN_NC 0
|
|
#define ARM_LPAE_TCR_RGN_WBWA 1
|
|
#define ARM_LPAE_TCR_RGN_WT 2
|
|
#define ARM_LPAE_TCR_RGN_WB 3
|
|
|
|
#define ARM_LPAE_TCR_PS_32_BIT 0x0ULL
|
|
#define ARM_LPAE_TCR_PS_36_BIT 0x1ULL
|
|
#define ARM_LPAE_TCR_PS_40_BIT 0x2ULL
|
|
#define ARM_LPAE_TCR_PS_42_BIT 0x3ULL
|
|
#define ARM_LPAE_TCR_PS_44_BIT 0x4ULL
|
|
#define ARM_LPAE_TCR_PS_48_BIT 0x5ULL
|
|
#define ARM_LPAE_TCR_PS_52_BIT 0x6ULL
|
|
|
|
#define ARM_LPAE_VTCR_PS_SHIFT 16
|
|
#define ARM_LPAE_VTCR_PS_MASK 0x7
|
|
|
|
#define ARM_LPAE_MAIR_ATTR_SHIFT(n) ((n) << 3)
|
|
#define ARM_LPAE_MAIR_ATTR_MASK 0xff
|
|
#define ARM_LPAE_MAIR_ATTR_DEVICE 0x04
|
|
#define ARM_LPAE_MAIR_ATTR_NC 0x44
|
|
#define ARM_LPAE_MAIR_ATTR_INC_OWBRWA 0xf4
|
|
#define ARM_LPAE_MAIR_ATTR_WBRWA 0xff
|
|
#define ARM_LPAE_MAIR_ATTR_IDX_NC 0
|
|
#define ARM_LPAE_MAIR_ATTR_IDX_CACHE 1
|
|
#define ARM_LPAE_MAIR_ATTR_IDX_DEV 2
|
|
#define ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE 3
|
|
|
|
#define ARM_MALI_LPAE_TTBR_ADRMODE_TABLE (3u << 0)
|
|
#define ARM_MALI_LPAE_TTBR_READ_INNER BIT(2)
|
|
#define ARM_MALI_LPAE_TTBR_SHARE_OUTER BIT(4)
|
|
|
|
#define ARM_MALI_LPAE_MEMATTR_IMP_DEF 0x88ULL
|
|
#define ARM_MALI_LPAE_MEMATTR_WRITE_ALLOC 0x8DULL
|
|
|
|
#define ARM_LPAE_MAX_LEVELS 4
|
|
|
|
#define iopte_type(pte) \
|
|
(((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK)
|
|
|
|
#define iopte_prot(pte) ((pte) & ARM_LPAE_PTE_ATTR_MASK)
|
|
|
|
static inline bool iopte_leaf(arm_lpae_iopte pte, int lvl,
|
|
enum io_pgtable_fmt fmt)
|
|
{
|
|
if (lvl == (ARM_LPAE_MAX_LEVELS - 1) && fmt != ARM_MALI_LPAE)
|
|
return iopte_type(pte) == ARM_LPAE_PTE_TYPE_PAGE;
|
|
|
|
return iopte_type(pte) == ARM_LPAE_PTE_TYPE_BLOCK;
|
|
}
|
|
|
|
#ifdef __KVM_NVHE_HYPERVISOR__
|
|
#include <nvhe/memory.h>
|
|
#define __arm_lpae_virt_to_phys hyp_virt_to_phys
|
|
#define __arm_lpae_phys_to_virt hyp_phys_to_virt
|
|
#else
|
|
#define __arm_lpae_virt_to_phys __pa
|
|
#define __arm_lpae_phys_to_virt __va
|
|
#endif
|
|
|
|
/* Generic functions */
|
|
void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
|
|
arm_lpae_iopte *ptep);
|
|
|
|
int arm_lpae_init_pgtable(struct io_pgtable_cfg *cfg,
|
|
struct arm_lpae_io_pgtable *data);
|
|
int arm_lpae_init_pgtable_s1(struct io_pgtable_cfg *cfg,
|
|
struct arm_lpae_io_pgtable *data);
|
|
int arm_lpae_init_pgtable_s2(struct io_pgtable_cfg *cfg,
|
|
struct arm_lpae_io_pgtable *data);
|
|
|
|
/* Host/hyp-specific functions */
|
|
void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, struct io_pgtable_cfg *cfg);
|
|
void __arm_lpae_free_pages(void *pages, size_t size, struct io_pgtable_cfg *cfg);
|
|
void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, int num_entries,
|
|
struct io_pgtable_cfg *cfg);
|
|
int arm_lpae_mapping_exists(struct arm_lpae_io_pgtable *data);
|
|
void arm_lpae_mapping_missing(struct arm_lpae_io_pgtable *data);
|
|
|
|
#endif /* IO_PGTABLE_H_ */
|