bdk: smmu: refactor and update driver

- Allow ASID to be configured
- Allow 34-bit PAs
- Use special type for setting PDE/PTE config
- Initialize all pages as non accessible
- Add function for mapping 4MB regions directly
- Add SMMU heap reset function
- Correct address load OP to 32-bit and remove alignment on SMMU enable payload
- Refactor all defines
This commit is contained in:
CTCaer 2024-03-14 09:21:06 +02:00
parent 0100c11757
commit 9e41aa7759
3 changed files with 197 additions and 98 deletions

View File

@ -18,6 +18,7 @@
#include <string.h>
#include <soc/bpmp.h>
#include <soc/ccplex.h>
#include <soc/timer.h>
#include <soc/t210.h>
@ -25,27 +26,39 @@
#include <mem/smmu.h>
#include <memory_map.h>
#define SMMU_PAGE_SHIFT 12
#define SMMU_PAGE_SIZE (1 << SMMU_PAGE_SHIFT)
/*! SMMU register defines */
#define SMMU_ASID(asid) (((asid) << 24u) | ((asid) << 16u) | ((asid) << 8u) | (asid))
#define SMMU_ENABLE BIT(31)
#define SMMU_TLB_ACTIVE_LINES(l) ((l) << 0u)
#define SMMU_TLB_RR_ARBITRATION BIT(28)
#define SMMU_TLB_HIT_UNDER_MISS BIT(29)
#define SMMU_TLB_STATS_ENABLE BIT(31)
#define SMUU_PTC_INDEX_MAP(m) ((m) << 0u)
#define SMUU_PTC_LINE_MASK(m) ((m) << 8u)
#define SMUU_PTC_REQ_LIMIT(l) ((l) << 24u)
#define SMUU_PTC_CACHE_ENABLE BIT(29)
#define SMUU_PTC_STATS_ENABLE BIT(31)
/*! Page table defines */
#define SMMU_4MB_REGION 0
#define SMMU_PAGE_TABLE 1
#define SMMU_PDIR_COUNT 1024
#define SMMU_PDIR_SIZE (sizeof(u32) * SMMU_PDIR_COUNT)
#define SMMU_PTBL_COUNT 1024
#define SMMU_PTBL_SIZE (sizeof(u32) * SMMU_PTBL_COUNT)
#define SMMU_PDIR_SHIFT 12
#define SMMU_PDE_SHIFT 12
#define SMMU_PTE_SHIFT 12
#define SMMU_PFN_MASK 0x000FFFFF
#define SMMU_ADDR_TO_PFN(addr) ((addr) >> 12)
#define SMMU_ADDR_TO_PDN(addr) ((addr) >> 22)
#define SMMU_PDN_TO_ADDR(addr) ((pdn) << 22)
#define SMMU_MK_PDIR(page, attr) (((page) >> SMMU_PDIR_SHIFT) | (attr))
#define SMMU_MK_PDE(page, attr) (((page) >> SMMU_PDE_SHIFT) | (attr))
#define SMMU_PAGE_SHIFT 12u
#define SMMU_PTN_SHIFT SMMU_PAGE_SHIFT
#define SMMU_PDN_SHIFT 22u
#define SMMU_ADDR_TO_PFN(addr) ((addr) >> SMMU_PAGE_SHIFT)
#define SMMU_ADDR_TO_PTN(addr) ((addr) >> SMMU_PTN_SHIFT)
#define SMMU_ADDR_TO_PDN(addr) ((addr) >> SMMU_PDN_SHIFT)
#define SMMU_PTN_TO_ADDR(ptn) ((ptn) << SMMU_PTN_SHIFT)
#define SMMU_PDN_TO_ADDR(pdn) ((pdn) << SMMU_PDN_SHIFT)
#define SMMU_PTB(page, attr) (((attr) << 29u) | ((page) >> SMMU_PAGE_SHIFT))
u8 *_pageheap = (u8 *)SMMU_HEAP_ADDR;
static void *smmu_heap = (void *)SMMU_HEAP_ADDR;
// Enabling SMMU requires a TZ secure write: MC(MC_SMMU_CONFIG) = 1;
u8 smmu_payload[] __attribute__((aligned(16))) = {
0xC1, 0x00, 0x00, 0x58, // 0x00: LDR X1, =0x70019010
// Enabling SMMU requires a TZ (EL3) secure write. MC(MC_SMMU_CONFIG) = 1;
static const u8 smmu_enable_payload[] = {
0xC1, 0x00, 0x00, 0x18, // 0x00: LDR W1, =0x70019010
0x20, 0x00, 0x80, 0xD2, // 0x04: MOV X0, #0x1
0x20, 0x00, 0x00, 0xB9, // 0x08: STR W0, [X1]
0x1F, 0x71, 0x08, 0xD5, // 0x0C: IC IALLUIS
@ -56,17 +69,22 @@ u8 smmu_payload[] __attribute__((aligned(16))) = {
void *smmu_page_zalloc(u32 num)
{
u8 *res = _pageheap;
_pageheap += SZ_PAGE * num;
memset(res, 0, SZ_PAGE * num);
return res;
void *page = smmu_heap;
memset(page, 0, SZ_PAGE * num);
smmu_heap += SZ_PAGE * num;
return page;
}
static u32 *_smmu_pdir_alloc()
static pde_t *_smmu_pdir_alloc()
{
u32 *pdir = (u32 *)smmu_page_zalloc(1);
for (int pdn = 0; pdn < SMMU_PDIR_COUNT; pdn++)
pdir[pdn] = _PDE_VACANT(pdn);
pde_t *pdir = (pde_t *)smmu_page_zalloc(1);
// Initialize pdes with no permissions.
for (u32 pdn = 0; pdn < SMMU_PDIR_COUNT; pdn++)
pdir[pdn].huge.page = pdn;
return pdir;
}
@ -77,9 +95,12 @@ static void _smmu_flush_regs()
void smmu_flush_all()
{
// Flush the entire page table cache.
MC(MC_SMMU_PTC_FLUSH) = 0;
_smmu_flush_regs();
// Flush the entire table.
MC(MC_SMMU_TLB_FLUSH) = 0;
_smmu_flush_regs();
}
@ -88,8 +109,8 @@ void smmu_init()
{
MC(MC_SMMU_PTB_ASID) = 0;
MC(MC_SMMU_PTB_DATA) = 0;
MC(MC_SMMU_TLB_CONFIG) = 0x30000030;
MC(MC_SMMU_PTC_CONFIG) = 0x28000F3F;
MC(MC_SMMU_TLB_CONFIG) = SMMU_TLB_HIT_UNDER_MISS | SMMU_TLB_RR_ARBITRATION | SMMU_TLB_ACTIVE_LINES(48);
MC(MC_SMMU_PTC_CONFIG) = SMUU_PTC_CACHE_ENABLE | SMUU_PTC_REQ_LIMIT(8) | SMUU_PTC_LINE_MASK(0xF) | SMUU_PTC_INDEX_MAP(0x3F);
MC(MC_SMMU_PTC_FLUSH) = 0;
MC(MC_SMMU_TLB_FLUSH) = 0;
}
@ -101,7 +122,8 @@ void smmu_enable()
if (enabled)
return;
ccplex_boot_cpu0((u32)smmu_payload, false);
// Launch payload on CCPLEX in order to set SMMU enable bit.
ccplex_boot_cpu0((u32)smmu_enable_payload, false);
msleep(100);
ccplex_powergate_cpu0();
@ -110,63 +132,114 @@ void smmu_enable()
enabled = true;
}
u32 *smmu_init_domain4(u32 dev_base, u32 asid)
void smmu_reset_heap()
{
u32 *pdir = _smmu_pdir_alloc();
smmu_heap = (void *)SMMU_HEAP_ADDR;
}
void *smmu_init_domain(u32 dev_base, u32 asid)
{
void *ptb = _smmu_pdir_alloc();
MC(MC_SMMU_PTB_ASID) = asid;
MC(MC_SMMU_PTB_DATA) = SMMU_MK_PDIR((u32)pdir, _PDIR_ATTR);
MC(MC_SMMU_PTB_DATA) = SMMU_PTB((u32)ptb, SMMU_ATTR_ALL);
_smmu_flush_regs();
MC(dev_base) = 0x80000000 | (asid << 24) | (asid << 16) | (asid << 8) | (asid);
// Use the same macro for both quad and single domains. Reserved bits are not set anyway.
MC(dev_base) = SMMU_ENABLE | SMMU_ASID(asid);
_smmu_flush_regs();
return pdir;
return ptb;
}
u32 *smmu_get_pte(u32 *pdir, u32 iova)
void smmu_deinit_domain(u32 dev_base, u32 asid)
{
u32 ptn = SMMU_ADDR_TO_PFN(iova);
u32 pdn = SMMU_ADDR_TO_PDN(iova);
u32 *ptbl;
MC(MC_SMMU_PTB_ASID) = asid;
MC(MC_SMMU_PTB_DATA) = 0;
MC(dev_base) = 0;
_smmu_flush_regs();
}
if (pdir[pdn] != _PDE_VACANT(pdn))
ptbl = (u32 *)((pdir[pdn] & SMMU_PFN_MASK) << SMMU_PDIR_SHIFT);
void smmu_domain_bypass(u32 dev_base, bool bypass)
{
if (bypass)
{
smmu_flush_all();
bpmp_mmu_maintenance(BPMP_MMU_MAINT_CLN_INV_WAY, false);
MC(dev_base) &= ~SMMU_ENABLE;
}
else
{
ptbl = (u32 *)smmu_page_zalloc(1);
u32 addr = SMMU_PDN_TO_ADDR(pdn);
for (int pn = 0; pn < SMMU_PTBL_COUNT; pn++, addr += SMMU_PAGE_SIZE)
ptbl[pn] = _PTE_VACANT(addr);
pdir[pdn] = SMMU_MK_PDE((u32)ptbl, _PDE_ATTR | _PDE_NEXT);
bpmp_mmu_maintenance(BPMP_MMU_MAINT_CLN_INV_WAY, false);
MC(dev_base) |= SMMU_ENABLE;
smmu_flush_all();
}
return &ptbl[ptn % SMMU_PTBL_COUNT];
}
void smmu_map(u32 *pdir, u32 addr, u32 page, int cnt, u32 attr)
{
for (int i = 0; i < cnt; i++)
{
u32 *pte = smmu_get_pte(pdir, addr);
*pte = SMMU_ADDR_TO_PFN(page) | attr;
addr += SZ_PAGE;
page += SZ_PAGE;
}
smmu_flush_all();
}
u32 *smmu_init_domain(u32 asid)
{
return smmu_init_domain4(asid, 1);
}
void smmu_deinit_domain(u32 asid)
{
MC(MC_SMMU_PTB_ASID) = 1;
MC(MC_SMMU_PTB_DATA) = 0;
MC(asid) = 0;
_smmu_flush_regs();
}
static pte_t *_smmu_get_pte(pde_t *pdir, u32 iova)
{
u32 pdn = SMMU_ADDR_TO_PDN(iova);
pte_t *ptbl;
// Get 4MB page table or initialize one.
if (pdir[pdn].tbl.attr)
ptbl = (pte_t *)(SMMU_PTN_TO_ADDR(pdir[pdn].tbl.table));
else
{
// Allocate page table.
ptbl = (pte_t *)smmu_page_zalloc(1);
// Get address.
u32 addr = SMMU_PDN_TO_ADDR(pdn);
// Initialize page table with no permissions.
for (u32 pn = 0; pn < SMMU_PTBL_COUNT; pn++, addr += SZ_PAGE)
ptbl[pn].page = SMMU_ADDR_TO_PFN(addr);
// Set page table to the page directory.
pdir[pdn].tbl.table = SMMU_ADDR_TO_PTN((u32)ptbl);
pdir[pdn].tbl.next = SMMU_PAGE_TABLE;
pdir[pdn].tbl.attr = SMMU_ATTR_ALL;
smmu_flush_all();
}
return &ptbl[SMMU_ADDR_TO_PTN(iova) % SMMU_PTBL_COUNT];
}
void smmu_map(void *ptb, u32 iova, u64 iopa, u32 pages, u32 attr)
{
// Map pages to page table entries. VA/PA should be aligned to 4KB.
for (u32 i = 0; i < pages; i++)
{
pte_t *pte = _smmu_get_pte((pde_t *)ptb, iova);
pte->page = SMMU_ADDR_TO_PFN(iopa);
pte->attr = attr;
iova += SZ_PAGE;
iopa += SZ_PAGE;
}
smmu_flush_all();
}
void smmu_map_huge(void *ptb, u32 iova, u64 iopa, u32 regions, u32 attr)
{
pde_t *pdir = (pde_t *)ptb;
// Map 4MB regions to page directory entries. VA/PA should be aligned to 4MB.
for (u32 i = 0; i < regions; i++)
{
u32 pdn = SMMU_ADDR_TO_PDN(iova);
pdir[pdn].huge.page = SMMU_ADDR_TO_PDN(iopa);
pdir[pdn].huge.next = SMMU_4MB_REGION;
pdir[pdn].huge.attr = attr;
iova += SZ_4M;
iopa += SZ_4M;
}
smmu_flush_all();
}

View File

@ -15,32 +15,57 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <assert.h>
#include <utils/types.h>
#define MC_SMMU_AVPC_ASID 0x23C
#define MC_SMMU_TSEC_ASID 0x294
#define SMMU_PDE_NEXT_SHIFT 28
#define MC_SMMU_PTB_DATA_0_ASID_NONSECURE_SHIFT 29
#define MC_SMMU_PTB_DATA_0_ASID_WRITABLE_SHIFT 30
#define MC_SMMU_PTB_DATA_0_ASID_READABLE_SHIFT 31
#define _READABLE (1 << MC_SMMU_PTB_DATA_0_ASID_READABLE_SHIFT)
#define _WRITABLE (1 << MC_SMMU_PTB_DATA_0_ASID_WRITABLE_SHIFT)
#define _NONSECURE (1 << MC_SMMU_PTB_DATA_0_ASID_NONSECURE_SHIFT)
#define _PDE_NEXT (1 << SMMU_PDE_NEXT_SHIFT)
#define _MASK_ATTR (_READABLE | _WRITABLE | _NONSECURE)
#define _PDIR_ATTR (_READABLE | _WRITABLE | _NONSECURE)
#define _PDE_ATTR (_READABLE | _WRITABLE | _NONSECURE)
#define _PDE_VACANT(pdn) (((pdn) << 10) | _PDE_ATTR)
#define _PTE_ATTR (_READABLE | _WRITABLE | _NONSECURE)
#define _PTE_VACANT(addr) (((addr) >> SMMU_PAGE_SHIFT) | _PTE_ATTR)
#define SMMU_NS BIT(0)
#define SMMU_WRITE BIT(1)
#define SMMU_READ BIT(2)
#define SMMU_ATTR_ALL (SMMU_READ | SMMU_WRITE | SMMU_NS)
typedef struct _pde_t {
union {
union {
struct {
u32 table:22;
u32 rsvd:6;
u32 next:1;
u32 attr:3;
} tbl;
struct {
u32 rsvd_:10;
u32 page:12;
u32 rsvd:6;
u32 next:1;
u32 attr:3;
} huge;
};
u32 pde;
};
} pde_t;
typedef struct _pte_t {
u32 page:22;
u32 rsvd:7;
u32 attr:3;
} pte_t;
static_assert(sizeof(pde_t) == sizeof(u32), "pde_t size is wrong!");
static_assert(sizeof(pte_t) == sizeof(u32), "pte_t size is wrong!");
void *smmu_page_zalloc(u32 num);
void smmu_flush_all();
void smmu_init();
void smmu_enable();
u32 *smmu_init_domain4(u32 dev_base, u32 asid);
u32 *smmu_get_pte(u32 *pdir, u32 iova);
void smmu_map(u32 *pdir, u32 addr, u32 page, int cnt, u32 attr);
u32 *smmu_init_domain(u32 asid);
void smmu_deinit_domain(u32 asid);
void smmu_reset_heap();
void *smmu_init_domain(u32 dev_base, u32 asid);
void smmu_deinit_domain(u32 dev_base, u32 asid);
void smmu_domain_bypass(u32 dev_base, bool bypass);
void smmu_map(void *ptb, u32 iova, u64 iopa, u32 pages, u32 attr);
void smmu_map_huge(void *ptb, u32 iova, u64 iopa, u32 regions, u32 attr);

View File

@ -70,8 +70,9 @@ int tsec_query(void *tsec_keys, tsec_ctxt_t *tsec_ctxt)
int res = 0;
u8 *fwbuf = NULL;
u32 type = tsec_ctxt->type;
u32 *pdir, *car, *fuse, *pmc, *flowctrl, *se, *mc, *iram, *evec;
u32 *car, *fuse, *pmc, *flowctrl, *se, *mc, *iram, *evec;
u32 *pkg11_magic_off;
void *ptb;
bpmp_mmu_disable();
bpmp_freq_t prev_fid = bpmp_clk_rate_set(BPMP_CLK_NORMAL);
@ -145,7 +146,7 @@ int tsec_query(void *tsec_keys, tsec_ctxt_t *tsec_ctxt)
if (type == TSEC_FW_TYPE_EMU)
{
// Init SMMU translation for TSEC.
pdir = smmu_init_domain(MC_SMMU_TSEC_ASID);
ptb = smmu_init_domain(MC_SMMU_TSEC_ASID, 1);
smmu_init();
// Enable SMMU.
@ -155,7 +156,7 @@ int tsec_query(void *tsec_keys, tsec_ctxt_t *tsec_ctxt)
car = smmu_page_zalloc(1);
memcpy(car, (void *)CLOCK_BASE, SZ_PAGE);
car[CLK_RST_CONTROLLER_CLK_SOURCE_TSEC / 4] = CLK_SRC_DIV(2);
smmu_map(pdir, CLOCK_BASE, (u32)car, 1, _WRITABLE | _READABLE | _NONSECURE);
smmu_map(ptb, CLOCK_BASE, (u32)car, 1, SMMU_WRITE | SMMU_READ | SMMU_NS);
// Fuse driver.
fuse = smmu_page_zalloc(1);
@ -163,38 +164,38 @@ int tsec_query(void *tsec_keys, tsec_ctxt_t *tsec_ctxt)
fuse[0x82C / 4] = 0;
fuse[0x9E0 / 4] = (1 << (TSEC_HOS_KB_620 + 2)) - 1;
fuse[0x9E4 / 4] = (1 << (TSEC_HOS_KB_620 + 2)) - 1;
smmu_map(pdir, (FUSE_BASE - 0x800), (u32)fuse, 1, _READABLE | _NONSECURE);
smmu_map(ptb, (FUSE_BASE - 0x800), (u32)fuse, 1, SMMU_READ | SMMU_NS);
// Power management controller.
pmc = smmu_page_zalloc(1);
smmu_map(pdir, RTC_BASE, (u32)pmc, 1, _READABLE | _NONSECURE);
smmu_map(ptb, RTC_BASE, (u32)pmc, 1, SMMU_READ | SMMU_NS);
// Flow control.
flowctrl = smmu_page_zalloc(1);
smmu_map(pdir, FLOW_CTLR_BASE, (u32)flowctrl, 1, _WRITABLE | _NONSECURE);
smmu_map(ptb, FLOW_CTLR_BASE, (u32)flowctrl, 1, SMMU_WRITE | SMMU_NS);
// Security engine.
se = smmu_page_zalloc(1);
memcpy(se, (void *)SE_BASE, SZ_PAGE);
smmu_map(pdir, SE_BASE, (u32)se, 1, _READABLE | _WRITABLE | _NONSECURE);
smmu_map(ptb, SE_BASE, (u32)se, 1, SMMU_READ | SMMU_WRITE | SMMU_NS);
// Memory controller.
mc = smmu_page_zalloc(1);
memcpy(mc, (void *)MC_BASE, SZ_PAGE);
mc[MC_IRAM_BOM / 4] = 0;
mc[MC_IRAM_TOM / 4] = DRAM_START;
smmu_map(pdir, MC_BASE, (u32)mc, 1, _READABLE | _NONSECURE);
smmu_map(ptb, MC_BASE, (u32)mc, 1, SMMU_READ | SMMU_NS);
// IRAM
iram = smmu_page_zalloc(0x30);
memcpy(iram, tsec_ctxt->pkg1, 0x30000);
// PKG1.1 magic offset.
pkg11_magic_off = (u32 *)(iram + ((tsec_ctxt->pkg11_off + 0x20) / 4));
smmu_map(pdir, 0x40010000, (u32)iram, 0x30, _READABLE | _WRITABLE | _NONSECURE);
pkg11_magic_off = (u32 *)(iram + ((tsec_ctxt->pkg11_off + 0x20) / sizeof(u32)));
smmu_map(ptb, 0x40010000, (u32)iram, 0x30, SMMU_READ | SMMU_WRITE | SMMU_NS);
// Exception vectors
evec = smmu_page_zalloc(1);
smmu_map(pdir, EXCP_VEC_BASE, (u32)evec, 1, _READABLE | _WRITABLE | _NONSECURE);
smmu_map(ptb, EXCP_VEC_BASE, (u32)evec, 1, SMMU_READ | SMMU_WRITE | SMMU_NS);
}
// Execute firmware.
@ -229,7 +230,7 @@ int tsec_query(void *tsec_keys, tsec_ctxt_t *tsec_ctxt)
if (kidx != 8)
{
res = -6;
smmu_deinit_domain(MC_SMMU_TSEC_ASID);
smmu_deinit_domain(MC_SMMU_TSEC_ASID, 1);
goto out_free;
}
@ -240,7 +241,7 @@ int tsec_query(void *tsec_keys, tsec_ctxt_t *tsec_ctxt)
memcpy(tsec_keys, &key, 0x20);
memcpy(tsec_ctxt->pkg1, iram, 0x30000);
smmu_deinit_domain(MC_SMMU_TSEC_ASID);
smmu_deinit_domain(MC_SMMU_TSEC_ASID, 1);
// for (int i = 0; i < kidx; i++)
// gfx_printf("key %08X\n", key[i]);