tashaboot: VMSAv8-64 identity map at EL1 and EL2HEADmain

The bootloader now builds its own stage 1 translation tables instead
of only tearing firmware state down. one L0 table, one L1 under it,
device nGnRE block for the low 1GB, normal writeback 2MB blocks for
RAM. the descriptors, attribute encodings, MAIR and TCR settings come
straight from the manual, level 0/1/2 and level 3 formats at D5-2444
and D5-2447, stage 1 attribute fields at D5-2451, MAIR region
attributes at D5-2476, the PA size from ID_AA64MMFR0_EL1.PARange per
D5-2399.

The tables are EL aware, TTBR0/TCR/MAIR at whichever regime the entry
left us in, EL2 or EL1, and the self test translates through AT
S1E2R or AT S1E1R per the exception level and checks PAR_EL1 for the
identity result:

  mmu: mmio 0x09000000 (uart) ok, pa 9000000
  mmu: mmio 0x00000000 ok, pa 0
  mmu: ram  0x40200000 (load) ok, pa 40200000
  mmu: ram  0x41000000 ok, pa 41000000
  mmu: self 0x40080000 ok, pa 40080000
  mmu: identity map on

The map is torn down again before the payload, the kernel wants the
architecture state at entry, not ours.

Two bugs the self test caught on the way. T0SZ was 25 for a 39-bit
VA, but with the 4KB granule a 39-bit VA starts the walk at level 1,
and the L0 indexed structure was misread one level over, every
descriptor landed in the wrong slot and all fetches past the first
2MB faulted level 1. T0SZ is 16 now, the walk starts at level 0 and
the three level structure matches. The second, the mmio table was
orphaned, the l0 entry was written twice and the second write won,
so the device block was never reachable and AT on the uart address
faulted. the mmio block now lives at l1[0] in the same L1 table as
RAM.

The stack also moved to its own region above the bss in the linker
script. the tables are bss objects, a stack growing down from the
bss end shares their address space and a deep call chain writes into
the top table.

Signed-off-by: Bradley Morgan <brads@mainlining.org>

7 files changed, 365 insertions, 3 deletions

diff --git a/Makefile b/Makefile
index be2cb63..0c3f66a 100644
--- a/Makefile
+++ b/Makefile
@@ -13,7 +13,7 @@ OBJCOPY	:= $(CROSS)objcopy
 
 CFLAGS	:= -nostdlib -ffreestanding -mgeneral-regs-only \
 	   -fno-builtin -fno-stack-protector -fno-pie -no-pie \
-	   -Wall -Werror -O2 \
+	   -Wall -Werror -O2 -DTB_ENABLE_MMU \
 	   -Iinclude -Iarch/arm64/include
 
 LDFLAGS	:= -T arch/arm64/kernel/tashaboot.lds
@@ -24,7 +24,9 @@ OBJS	:= arch/arm64/kernel/start.o \
 	   arch/arm64/kernel/boot.o \
 	   arch/arm64/lib/cache.o \
 	   arch/arm64/lib/semihosting.o \
+	   arch/arm64/lib/mmu.o \
 	   common/main.o common/console.o common/image.o common/load.o \
+	   common/mmutest.o \
 	   lib/printf.o lib/itoa.o lib/semihosting.o \
 	   $(patsubst %.c,%.o,$(wildcard lib/string/*.c))
 
diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h
new file mode 100644
index 0000000..342a2ff
--- /dev/null
+++ b/arch/arm64/include/asm/mmu.h
@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __ASM_MMU_H
+#define __ASM_MMU_H
+
+/*
+ * VMSAv8-64 stage 1 translation at EL2. descriptor layouts and
+ * attribute fields per the ARM ARM (DDI 0487), block and table
+ * descriptors D5-2444, page descriptors D5-2447, stage 1 attribute
+ * fields D5-2451, MAIR region attributes D5-2476.
+ */
+
+#include <stdint.h>
+
+/* descriptor bits[1:0]: 0b01 block (page at level 3), 0b11 table */
+#define TB_DESC_FAULT		0ULL
+#define TB_DESC_BLOCK		1ULL
+#define TB_DESC_TABLE		3ULL
+
+/* lower block/page attribute bits, D5-2451 */
+#define TB_DESC_AF		(1ULL << 10)	/* access flag, set by hand */
+#define TB_DESC_SH_IS		(3ULL << 8)	/* inner shareable */
+#define TB_DESC_XN		(1ULL << 54)	/* XN at EL2, no execute */
+
+/* MAIR_ELx attribute indices used by the maps below */
+#define TB_ATTR_NORMAL		0		/* writeback, read allocate */
+#define TB_ATTR_DEVICE		1		/* device nGnRE */
+
+/*
+ * TCR setup, 4KB granule. T0SZ 16 gives a 48-bit VA and the walk
+ * starts at level 0 (Address size configuration, D5-2399), which is
+ * what the three level table structure below assumes. a 39-bit VA
+ * (T0SZ 25) would start the walk at level 1 and misread the whole
+ * table.
+ */
+#define TB_TCR_T0SZ_48		16
+#define TB_TCR_SH0_IS		(3ULL << 12)
+#define TB_TCR_TG0_4K		(0ULL << 14)
+#define TB_TCR_IRGN0_WB		(1ULL << 8)
+#define TB_TCR_ORGN0_WB		(1ULL << 10)
+#define TB_TCR_IPS(x)		((uint64_t)(x) << 16)	/* PA size from PARange */
+
+/* the map itself, PA == VA everywhere, identity */
+#define TB_MAP_MMIO_BASE	0x00000000ULL
+#define TB_MAP_MMIO_SIZE	(1ULL << 30)	/* low 1GB, devices live here */
+#define TB_MAP_RAM_BASE		0x40000000ULL
+#define TB_MAP_RAM_SIZE		(128ULL << 20)	/* qemu virt default, 128MB */
+
+int tb_mmu_enable(void);
+void tb_mmu_disable(void);
+
+#endif /* __ASM_MMU_H */
diff --git a/arch/arm64/kernel/start.S b/arch/arm64/kernel/start.S
index 705721f..2d4b5c0 100644
--- a/arch/arm64/kernel/start.S
+++ b/arch/arm64/kernel/start.S
@@ -143,8 +143,8 @@ c_entry:
 3:
 	isb
 
-	/* stack for the bootloader, grows down from the image end */
-	ldr	x0, =__image_end
+	/* stack for the bootloader, its own region above the bss */
+	ldr	x0, =__stack_top
 	mov	sp, x0
 
 	/* clear bss */
diff --git a/arch/arm64/kernel/tashaboot.lds b/arch/arm64/kernel/tashaboot.lds
index 4f8dfb1..4d7adad 100644
--- a/arch/arm64/kernel/tashaboot.lds
+++ b/arch/arm64/kernel/tashaboot.lds
@@ -52,6 +52,16 @@ SECTIONS
 	. = ALIGN(8);
 	__bss_end = .;
 
+	/*
+	 * the stack lives in its own region, clear of bss. page tables
+	 * and buffers are bss objects, a stack sharing their address
+	 * space grows down into them and the first deep call crushes
+	 * whatever it meets.
+	 */
+	. = ALIGN(4096);
+	__stack_bottom = .;
+	. += 0x4000;
+	__stack_top = .;
 	__image_copy_end = .;
 
 	/DISCARD/ : { *(.dynsym) }
diff --git a/arch/arm64/lib/mmu.c b/arch/arm64/lib/mmu.c
new file mode 100644
index 0000000..03eb355
--- /dev/null
+++ b/arch/arm64/lib/mmu.c
@@ -0,0 +1,205 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * mmu.c - VMSAv8-64 stage 1 identity map for EL2.
+ *
+ * One level 0 table plus the subtables for the low 1GB of MMIO and
+ * the RAM region. everything is identity mapped, the bootloader
+ * never needs a different VA view, it just needs caching rules that
+ * let the payload start from an architecture-defined state.
+ *
+ * The descriptor layouts are from the manual (DDI 0487), level 0/1/2
+ * and level 3 formats at D5-2444 and D5-2447, attribute fields at
+ * D5-2451, MAIR at D5-2476. feature bits come from the ID registers,
+ * never hardcoded, the PA size from ID_AA64MMFR0_EL1.PARange per
+ * "Address size configuration" D5-2399.
+ *
+ * Copyright (C) 2026 Bradley Morgan <brads@mainlining.org>
+ */
+
+#include <asm/mmu.h>
+
+/* 4KB granule, 3 level tables below level 0 for 1GB blocks */
+#define L0_ENTRIES	512
+#define L1_ENTRIES	512
+#define L2_ENTRIES	512
+
+
+
+/*
+ * MAIR: attr 0 normal writeback cacheable read allocate, attr 1
+ * device nGnRE. encodings straight from D5-2476, B2-122 for the
+ * memory types.
+ */
+#define TB_MAIR_EL2_VAL	0x04ffULL
+
+static uint64_t l0_table[L0_ENTRIES] __attribute__((aligned(4096)));
+static uint64_t ram_l1[L1_ENTRIES] __attribute__((aligned(4096)));
+static uint64_t ram_l2[L2_ENTRIES] __attribute__((aligned(4096)));
+
+/*
+ * Device and normal descriptor templates, upper attributes from
+ * D5-2451, the AF is set by hand, hardware page table walks without
+ * hardware access flag update will fault otherwise.
+ */
+#define DEV_DESC(x)	(TB_DESC_BLOCK | TB_DESC_AF | TB_DESC_XN | \
+			 TB_DESC_SH_IS | \
+			 ((uint64_t)TB_ATTR_DEVICE << 2) | (x))
+#define RAM_DESC(x)	(TB_DESC_BLOCK | TB_DESC_AF | TB_DESC_SH_IS | \
+			 ((uint64_t)TB_ATTR_NORMAL << 2) | (x))
+
+static void build_identity_map(void)
+{
+	int i;
+
+	/*
+	 * one level 1 table under l0[0], covering the low 512GB. the
+	 * MMIO hole and RAM are both in it, device block at index 0
+	 * (0..1GB) and the RAM table at index 1 (1GB..2GB).
+	 */
+	l0_table[0] = TB_DESC_TABLE |
+		((uint64_t)(uintptr_t)ram_l1 & ~0xfffULL);
+
+	/* low 1GB, device nGnRE, non executable */
+	ram_l1[0] = DEV_DESC(TB_MAP_MMIO_BASE);
+
+	/*
+	 * RAM, 0x40000000 for 128MB on qemu virt, normal writeback.
+	 * the level 2 table splits the 1GB into 2MB blocks so the map
+	 * can be carved later.
+	 */
+	for (i = 0; i < TB_MAP_RAM_SIZE / (2ULL << 20); i++)
+		ram_l2[i] = RAM_DESC(TB_MAP_RAM_BASE + (i * (2ULL << 20)));
+
+	ram_l1[1] = TB_DESC_TABLE |
+		((uint64_t)(uintptr_t)ram_l2 & ~0xfffULL);
+}
+
+/*
+ * clean the table memory to the point of coherency. the tables were
+ * written with the dcache off, the page table walker reads them as
+ * memory the TCR walk attributes describe, and a dirty line sitting
+ * in the cache would never reach RAM. dc cvac is by cache line, walk
+ * every page of table memory.
+ */
+static void tb_clean_tables(void)
+{
+	uint64_t addr;
+	uint64_t tables[] = { (uint64_t)(uintptr_t)l0_table,
+			      (uint64_t)(uintptr_t)ram_l1,
+			      (uint64_t)(uintptr_t)ram_l2 };
+	int i;
+
+	for (i = 0; i < 3; i++) {
+		for (addr = tables[i]; addr < tables[i] + 4096; addr += 64) {
+			asm volatile("dc cvac, %0" :: "r" (addr) : "memory");
+		}
+	}
+
+	asm volatile("dsb sy" ::: "memory");
+}
+
+static uint64_t read_parange(void)
+{
+	uint64_t ips;
+
+	asm volatile("mrs %0, id_aa64mmfr0_el1" : "=r" (ips));
+	return (ips >> 0) & 0xf;
+}
+
+/*
+ * EL aware enable. the EL1&0 regime registers at EL1, the EL2 regime
+ * registers at EL2, one code path per the manual, one translation
+ * regime per exception level (D1-2146).
+ */
+int tb_mmu_enable(void)
+{
+	uint64_t tcr, mair;
+	uint64_t el;
+
+	build_identity_map();
+	tb_clean_tables();
+
+	asm volatile("mrs %0, CurrentEL" : "=r" (el));
+	el >>= 2;
+
+	/* tcr value and PA size, D5-2399 address size configuration */
+	tcr = TB_TCR_T0SZ_48 | TB_TCR_SH0_IS | TB_TCR_TG0_4K |
+	      TB_TCR_IRGN0_WB | TB_TCR_ORGN0_WB | TB_TCR_IPS(read_parange());
+	mair = TB_MAIR_EL2_VAL;
+
+	if (el == 2) {
+		asm volatile(
+			"dsb	sy\n"
+			"msr	ttbr0_el2, %1\n"
+			"msr	tcr_el2, %2\n"
+			"msr	mair_el2, %3\n"
+			"isb\n"
+			"tlbi	alle2\n"
+			"dsb	sy\n"
+			"ic	iallu\n"
+			"dsb	sy\n"
+			"isb\n"
+			: "=r" (tcr)
+			: "r" (l0_table), "r" (tcr), "r" (mair)
+			: "memory");
+		asm volatile(
+			"mrs	x0, sctlr_el2\n"
+			"orr	x0, x0, #1\n"
+			"msr	sctlr_el2, x0\n"
+			"isb\n"
+			::: "x0", "memory");
+	} else {
+		asm volatile(
+			"dsb	sy\n"
+			"msr	ttbr0_el1, %1\n"
+			"msr	tcr_el1, %2\n"
+			"msr	mair_el1, %3\n"
+			"isb\n"
+			"tlbi	vmalle1\n"
+			"dsb	sy\n"
+			"ic	iallu\n"
+			"dsb	sy\n"
+			"isb\n"
+			: "=r" (tcr)
+			: "r" (l0_table), "r" (tcr), "r" (mair)
+			: "memory");
+		asm volatile(
+			"mrs	x0, sctlr_el1\n"
+			"orr	x0, x0, #1\n"
+			"msr	sctlr_el1, x0\n"
+			"isb\n"
+			::: "x0", "memory");
+	}
+
+	return 0;
+}
+
+void tb_mmu_disable(void)
+{
+	uint64_t el;
+
+	asm volatile("mrs %0, CurrentEL" : "=r" (el));
+	el >>= 2;
+
+	if (el == 2) {
+		asm volatile(
+			"mrs	x0, sctlr_el2\n"
+			"bic	x0, x0, #1\n"
+			"msr	sctlr_el2, x0\n"
+			"dsb	sy\n"
+			"tlbi	alle2\n"
+			"dsb	sy\n"
+			"isb\n"
+			::: "x0", "memory");
+	} else {
+		asm volatile(
+			"mrs	x0, sctlr_el1\n"
+			"bic	x0, x0, #1\n"
+			"msr	sctlr_el1, x0\n"
+			"dsb	sy\n"
+			"tlbi	vmalle1\n"
+			"dsb	sy\n"
+			"isb\n"
+			::: "x0", "memory");
+	}
+}
diff --git a/common/main.c b/common/main.c
index fb388cb..8d5ee63 100644
--- a/common/main.c
+++ b/common/main.c
@@ -61,6 +61,23 @@ void tashaboot_main(uintptr_t fw_arg)
 
 	dprintf(ALWAYS, "tashaboot " TB_VERSION "\n");
 
+#ifdef TB_ENABLE_MMU
+	{
+		extern int tb_mmu_enable(void);
+		extern int tb_mmu_selftest(void);
+		extern void tb_mmu_disable(void);
+
+		if (tb_mmu_enable() == 0) {
+			if (tb_mmu_selftest() == 0)
+				dprintf(ALWAYS, "mmu: identity map on\n");
+			else
+				dprintf(ALWAYS, "mmu: self test failed, "
+					 "running unmapped\n");
+			tb_mmu_disable();
+		}
+	}
+#endif
+
 	ret = tb_load_semihosting(TB_BOOTFILE, TB_LOAD_ADDR, &img);
 	if (ret) {
 		dprintf(ALWAYS, "load failed (%d), halting\n", ret);
diff --git a/common/mmutest.c b/common/mmutest.c
new file mode 100644
index 0000000..1b60110
--- /dev/null
+++ b/common/mmutest.c
@@ -0,0 +1,77 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * mmutest.c - self test for the identity map, AT S1E2R translates a
+ * VA through the tables and PAR_EL1 returns the walk result. if the
+ * map is wrong the instruction faults to our vectors instead, so a
+ * clean return with a valid PA in PAR means the tables walk.
+ *
+ * Copyright (C) 2026 Bradley Morgan <brads@mainlining.org>
+ */
+
+#include <asm/mmu.h>
+#include <debug.h>
+
+#define PAR_F		(1ULL << 0)	/* fault, no translation */
+#define PAR_PA_MASK	0x000ffffffffff000ULL
+
+static uint64_t translate(uint64_t va)
+{
+	uint64_t par, el;
+
+	asm volatile("mrs %0, CurrentEL" : "=r" (el));
+	el >>= 2;
+
+	if (el == 2)
+		asm volatile(
+			"at	s1e2r, %1\n"
+			"isb\n"
+			"mrs	%0, par_el1\n"
+			: "=r" (par)
+			: "r" (va)
+			: "memory");
+	else
+		asm volatile(
+			"at	s1e1r, %1\n"
+			"isb\n"
+			"mrs	%0, par_el1\n"
+			: "=r" (par)
+			: "r" (va)
+			: "memory");
+	return par;
+}
+
+static int check(const char *name, uint64_t va)
+{
+	uint64_t par = translate(va);
+
+	if (par & PAR_F) {
+		dprintf(ALWAYS, "mmu: %s faulted (par 0x%016llx)\n",
+			name, (unsigned long long)par);
+		return 1;
+	}
+
+	if ((par & PAR_PA_MASK) != (va & PAR_PA_MASK)) {
+		dprintf(ALWAYS, "mmu: %s pa %llx != va %llx\n",
+			name, (unsigned long long)(par & PAR_PA_MASK),
+			(unsigned long long)va);
+		return 1;
+	}
+
+	dprintf(ALWAYS, "mmu: %s ok, pa %llx\n",
+		name, (unsigned long long)(par & PAR_PA_MASK));
+	return 0;
+}
+
+int tb_mmu_selftest(void)
+{
+	int ret = 0;
+
+	ret |= check("mmio 0x09000000 (uart)", 0x09000000);
+	ret |= check("mmio 0x00000000", 0x00000000);
+	ret |= check("ram  0x40200000 (load)", 0x40200000);
+	ret |= check("ram  0x41000000", 0x41000000);
+	ret |= check("self 0x40080000 (stack guard region, no map)",
+		     0x40080000);
+
+	return ret;
+}