diff options
| -rw-r--r-- | Makefile | 4 | ||||
| -rw-r--r-- | arch/arm64/include/asm/mmu.h | 51 | ||||
| -rw-r--r-- | arch/arm64/kernel/start.S | 4 | ||||
| -rw-r--r-- | arch/arm64/kernel/tashaboot.lds | 10 | ||||
| -rw-r--r-- | arch/arm64/lib/mmu.c | 205 | ||||
| -rw-r--r-- | common/main.c | 17 | ||||
| -rw-r--r-- | common/mmutest.c | 77 |
7 files changed, 365 insertions, 3 deletions
@@ -13,7 +13,7 @@ OBJCOPY := $(CROSS)objcopy CFLAGS := -nostdlib -ffreestanding -mgeneral-regs-only \ -fno-builtin -fno-stack-protector -fno-pie -no-pie \ - -Wall -Werror -O2 \ + -Wall -Werror -O2 -DTB_ENABLE_MMU \ -Iinclude -Iarch/arm64/include LDFLAGS := -T arch/arm64/kernel/tashaboot.lds @@ -24,7 +24,9 @@ OBJS := arch/arm64/kernel/start.o \ arch/arm64/kernel/boot.o \ arch/arm64/lib/cache.o \ arch/arm64/lib/semihosting.o \ + arch/arm64/lib/mmu.o \ common/main.o common/console.o common/image.o common/load.o \ + common/mmutest.o \ lib/printf.o lib/itoa.o lib/semihosting.o \ $(patsubst %.c,%.o,$(wildcard lib/string/*.c)) diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h new file mode 100644 index 0000000..342a2ff --- /dev/null +++ b/arch/arm64/include/asm/mmu.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_MMU_H +#define __ASM_MMU_H + +/* + * VMSAv8-64 stage 1 translation at EL2. descriptor layouts and + * attribute fields per the ARM ARM (DDI 0487), block and table + * descriptors D5-2444, page descriptors D5-2447, stage 1 attribute + * fields D5-2451, MAIR region attributes D5-2476. + */ + +#include <stdint.h> + +/* descriptor bits[1:0]: 0b01 block (page at level 3), 0b11 table */ +#define TB_DESC_FAULT 0ULL +#define TB_DESC_BLOCK 1ULL +#define TB_DESC_TABLE 3ULL + +/* lower block/page attribute bits, D5-2451 */ +#define TB_DESC_AF (1ULL << 10) /* access flag, set by hand */ +#define TB_DESC_SH_IS (3ULL << 8) /* inner shareable */ +#define TB_DESC_XN (1ULL << 54) /* XN at EL2, no execute */ + +/* MAIR_ELx attribute indices used by the maps below */ +#define TB_ATTR_NORMAL 0 /* writeback, read allocate */ +#define TB_ATTR_DEVICE 1 /* device nGnRE */ + +/* + * TCR setup, 4KB granule. T0SZ 16 gives a 48-bit VA and the walk + * starts at level 0 (Address size configuration, D5-2399), which is + * what the three level table structure below assumes. a 39-bit VA + * (T0SZ 25) would start the walk at level 1 and misread the whole + * table. + */ +#define TB_TCR_T0SZ_48 16 +#define TB_TCR_SH0_IS (3ULL << 12) +#define TB_TCR_TG0_4K (0ULL << 14) +#define TB_TCR_IRGN0_WB (1ULL << 8) +#define TB_TCR_ORGN0_WB (1ULL << 10) +#define TB_TCR_IPS(x) ((uint64_t)(x) << 16) /* PA size from PARange */ + +/* the map itself, PA == VA everywhere, identity */ +#define TB_MAP_MMIO_BASE 0x00000000ULL +#define TB_MAP_MMIO_SIZE (1ULL << 30) /* low 1GB, devices live here */ +#define TB_MAP_RAM_BASE 0x40000000ULL +#define TB_MAP_RAM_SIZE (128ULL << 20) /* qemu virt default, 128MB */ + +int tb_mmu_enable(void); +void tb_mmu_disable(void); + +#endif /* __ASM_MMU_H */ diff --git a/arch/arm64/kernel/start.S b/arch/arm64/kernel/start.S index 705721f..2d4b5c0 100644 --- a/arch/arm64/kernel/start.S +++ b/arch/arm64/kernel/start.S @@ -143,8 +143,8 @@ c_entry: 3: isb - /* stack for the bootloader, grows down from the image end */ - ldr x0, =__image_end + /* stack for the bootloader, its own region above the bss */ + ldr x0, =__stack_top mov sp, x0 /* clear bss */ diff --git a/arch/arm64/kernel/tashaboot.lds b/arch/arm64/kernel/tashaboot.lds index 4f8dfb1..4d7adad 100644 --- a/arch/arm64/kernel/tashaboot.lds +++ b/arch/arm64/kernel/tashaboot.lds @@ -52,6 +52,16 @@ SECTIONS . = ALIGN(8); __bss_end = .; + /* + * the stack lives in its own region, clear of bss. page tables + * and buffers are bss objects, a stack sharing their address + * space grows down into them and the first deep call crushes + * whatever it meets. + */ + . = ALIGN(4096); + __stack_bottom = .; + . += 0x4000; + __stack_top = .; __image_copy_end = .; /DISCARD/ : { *(.dynsym) } diff --git a/arch/arm64/lib/mmu.c b/arch/arm64/lib/mmu.c new file mode 100644 index 0000000..03eb355 --- /dev/null +++ b/arch/arm64/lib/mmu.c @@ -0,0 +1,205 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * mmu.c - VMSAv8-64 stage 1 identity map for EL2. + * + * One level 0 table plus the subtables for the low 1GB of MMIO and + * the RAM region. everything is identity mapped, the bootloader + * never needs a different VA view, it just needs caching rules that + * let the payload start from an architecture-defined state. + * + * The descriptor layouts are from the manual (DDI 0487), level 0/1/2 + * and level 3 formats at D5-2444 and D5-2447, attribute fields at + * D5-2451, MAIR at D5-2476. feature bits come from the ID registers, + * never hardcoded, the PA size from ID_AA64MMFR0_EL1.PARange per + * "Address size configuration" D5-2399. + * + * Copyright (C) 2026 Bradley Morgan <brads@mainlining.org> + */ + +#include <asm/mmu.h> + +/* 4KB granule, 3 level tables below level 0 for 1GB blocks */ +#define L0_ENTRIES 512 +#define L1_ENTRIES 512 +#define L2_ENTRIES 512 + + + +/* + * MAIR: attr 0 normal writeback cacheable read allocate, attr 1 + * device nGnRE. encodings straight from D5-2476, B2-122 for the + * memory types. + */ +#define TB_MAIR_EL2_VAL 0x04ffULL + +static uint64_t l0_table[L0_ENTRIES] __attribute__((aligned(4096))); +static uint64_t ram_l1[L1_ENTRIES] __attribute__((aligned(4096))); +static uint64_t ram_l2[L2_ENTRIES] __attribute__((aligned(4096))); + +/* + * Device and normal descriptor templates, upper attributes from + * D5-2451, the AF is set by hand, hardware page table walks without + * hardware access flag update will fault otherwise. + */ +#define DEV_DESC(x) (TB_DESC_BLOCK | TB_DESC_AF | TB_DESC_XN | \ + TB_DESC_SH_IS | \ + ((uint64_t)TB_ATTR_DEVICE << 2) | (x)) +#define RAM_DESC(x) (TB_DESC_BLOCK | TB_DESC_AF | TB_DESC_SH_IS | \ + ((uint64_t)TB_ATTR_NORMAL << 2) | (x)) + +static void build_identity_map(void) +{ + int i; + + /* + * one level 1 table under l0[0], covering the low 512GB. the + * MMIO hole and RAM are both in it, device block at index 0 + * (0..1GB) and the RAM table at index 1 (1GB..2GB). + */ + l0_table[0] = TB_DESC_TABLE | + ((uint64_t)(uintptr_t)ram_l1 & ~0xfffULL); + + /* low 1GB, device nGnRE, non executable */ + ram_l1[0] = DEV_DESC(TB_MAP_MMIO_BASE); + + /* + * RAM, 0x40000000 for 128MB on qemu virt, normal writeback. + * the level 2 table splits the 1GB into 2MB blocks so the map + * can be carved later. + */ + for (i = 0; i < TB_MAP_RAM_SIZE / (2ULL << 20); i++) + ram_l2[i] = RAM_DESC(TB_MAP_RAM_BASE + (i * (2ULL << 20))); + + ram_l1[1] = TB_DESC_TABLE | + ((uint64_t)(uintptr_t)ram_l2 & ~0xfffULL); +} + +/* + * clean the table memory to the point of coherency. the tables were + * written with the dcache off, the page table walker reads them as + * memory the TCR walk attributes describe, and a dirty line sitting + * in the cache would never reach RAM. dc cvac is by cache line, walk + * every page of table memory. + */ +static void tb_clean_tables(void) +{ + uint64_t addr; + uint64_t tables[] = { (uint64_t)(uintptr_t)l0_table, + (uint64_t)(uintptr_t)ram_l1, + (uint64_t)(uintptr_t)ram_l2 }; + int i; + + for (i = 0; i < 3; i++) { + for (addr = tables[i]; addr < tables[i] + 4096; addr += 64) { + asm volatile("dc cvac, %0" :: "r" (addr) : "memory"); + } + } + + asm volatile("dsb sy" ::: "memory"); +} + +static uint64_t read_parange(void) +{ + uint64_t ips; + + asm volatile("mrs %0, id_aa64mmfr0_el1" : "=r" (ips)); + return (ips >> 0) & 0xf; +} + +/* + * EL aware enable. the EL1&0 regime registers at EL1, the EL2 regime + * registers at EL2, one code path per the manual, one translation + * regime per exception level (D1-2146). + */ +int tb_mmu_enable(void) +{ + uint64_t tcr, mair; + uint64_t el; + + build_identity_map(); + tb_clean_tables(); + + asm volatile("mrs %0, CurrentEL" : "=r" (el)); + el >>= 2; + + /* tcr value and PA size, D5-2399 address size configuration */ + tcr = TB_TCR_T0SZ_48 | TB_TCR_SH0_IS | TB_TCR_TG0_4K | + TB_TCR_IRGN0_WB | TB_TCR_ORGN0_WB | TB_TCR_IPS(read_parange()); + mair = TB_MAIR_EL2_VAL; + + if (el == 2) { + asm volatile( + "dsb sy\n" + "msr ttbr0_el2, %1\n" + "msr tcr_el2, %2\n" + "msr mair_el2, %3\n" + "isb\n" + "tlbi alle2\n" + "dsb sy\n" + "ic iallu\n" + "dsb sy\n" + "isb\n" + : "=r" (tcr) + : "r" (l0_table), "r" (tcr), "r" (mair) + : "memory"); + asm volatile( + "mrs x0, sctlr_el2\n" + "orr x0, x0, #1\n" + "msr sctlr_el2, x0\n" + "isb\n" + ::: "x0", "memory"); + } else { + asm volatile( + "dsb sy\n" + "msr ttbr0_el1, %1\n" + "msr tcr_el1, %2\n" + "msr mair_el1, %3\n" + "isb\n" + "tlbi vmalle1\n" + "dsb sy\n" + "ic iallu\n" + "dsb sy\n" + "isb\n" + : "=r" (tcr) + : "r" (l0_table), "r" (tcr), "r" (mair) + : "memory"); + asm volatile( + "mrs x0, sctlr_el1\n" + "orr x0, x0, #1\n" + "msr sctlr_el1, x0\n" + "isb\n" + ::: "x0", "memory"); + } + + return 0; +} + +void tb_mmu_disable(void) +{ + uint64_t el; + + asm volatile("mrs %0, CurrentEL" : "=r" (el)); + el >>= 2; + + if (el == 2) { + asm volatile( + "mrs x0, sctlr_el2\n" + "bic x0, x0, #1\n" + "msr sctlr_el2, x0\n" + "dsb sy\n" + "tlbi alle2\n" + "dsb sy\n" + "isb\n" + ::: "x0", "memory"); + } else { + asm volatile( + "mrs x0, sctlr_el1\n" + "bic x0, x0, #1\n" + "msr sctlr_el1, x0\n" + "dsb sy\n" + "tlbi vmalle1\n" + "dsb sy\n" + "isb\n" + ::: "x0", "memory"); + } +} diff --git a/common/main.c b/common/main.c index fb388cb..8d5ee63 100644 --- a/common/main.c +++ b/common/main.c @@ -61,6 +61,23 @@ void tashaboot_main(uintptr_t fw_arg) dprintf(ALWAYS, "tashaboot " TB_VERSION "\n"); +#ifdef TB_ENABLE_MMU + { + extern int tb_mmu_enable(void); + extern int tb_mmu_selftest(void); + extern void tb_mmu_disable(void); + + if (tb_mmu_enable() == 0) { + if (tb_mmu_selftest() == 0) + dprintf(ALWAYS, "mmu: identity map on\n"); + else + dprintf(ALWAYS, "mmu: self test failed, " + "running unmapped\n"); + tb_mmu_disable(); + } + } +#endif + ret = tb_load_semihosting(TB_BOOTFILE, TB_LOAD_ADDR, &img); if (ret) { dprintf(ALWAYS, "load failed (%d), halting\n", ret); diff --git a/common/mmutest.c b/common/mmutest.c new file mode 100644 index 0000000..1b60110 --- /dev/null +++ b/common/mmutest.c @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * mmutest.c - self test for the identity map, AT S1E2R translates a + * VA through the tables and PAR_EL1 returns the walk result. if the + * map is wrong the instruction faults to our vectors instead, so a + * clean return with a valid PA in PAR means the tables walk. + * + * Copyright (C) 2026 Bradley Morgan <brads@mainlining.org> + */ + +#include <asm/mmu.h> +#include <debug.h> + +#define PAR_F (1ULL << 0) /* fault, no translation */ +#define PAR_PA_MASK 0x000ffffffffff000ULL + +static uint64_t translate(uint64_t va) +{ + uint64_t par, el; + + asm volatile("mrs %0, CurrentEL" : "=r" (el)); + el >>= 2; + + if (el == 2) + asm volatile( + "at s1e2r, %1\n" + "isb\n" + "mrs %0, par_el1\n" + : "=r" (par) + : "r" (va) + : "memory"); + else + asm volatile( + "at s1e1r, %1\n" + "isb\n" + "mrs %0, par_el1\n" + : "=r" (par) + : "r" (va) + : "memory"); + return par; +} + +static int check(const char *name, uint64_t va) +{ + uint64_t par = translate(va); + + if (par & PAR_F) { + dprintf(ALWAYS, "mmu: %s faulted (par 0x%016llx)\n", + name, (unsigned long long)par); + return 1; + } + + if ((par & PAR_PA_MASK) != (va & PAR_PA_MASK)) { + dprintf(ALWAYS, "mmu: %s pa %llx != va %llx\n", + name, (unsigned long long)(par & PAR_PA_MASK), + (unsigned long long)va); + return 1; + } + + dprintf(ALWAYS, "mmu: %s ok, pa %llx\n", + name, (unsigned long long)(par & PAR_PA_MASK)); + return 0; +} + +int tb_mmu_selftest(void) +{ + int ret = 0; + + ret |= check("mmio 0x09000000 (uart)", 0x09000000); + ret |= check("mmio 0x00000000", 0x00000000); + ret |= check("ram 0x40200000 (load)", 0x40200000); + ret |= check("ram 0x41000000", 0x41000000); + ret |= check("self 0x40080000 (stack guard region, no map)", + 0x40080000); + + return ret; +} |