path: root/arch/arm64/kernel/start.S

/* SPDX-License-Identifier: GPL-2.0+ */
/*
 * tashaboot arm64 entry. handles whatever EL the firmware left us in,
 * EL3, EL2 or EL1, with the MMU either on or off, and arrives at a
 * clean EL1 with the MMU off before calling C.
 *
 * the secondary cores park, spin table bringup is a later problem.
 *
 * Copyright (C) 2026 Bradley Morgan <brads@mainlining.org>
 */

#include <asm/macro.h>

.section .text.boot
.globl _start
_start:
	/* code0: branch over the 64 byte Image header to reset */
	b	reset

	.balign 8
/*
 * the arm64 Image header fields, per Documentation/arch/arm64/
 * booting.rst: text_offset 0x08, image_size 0x10, flags 0x18,
 * magic 0x38. code0 above branches over all of it. text_offset 0
 * and image_size filled after link by tools/fillsize.py, the
 * magic pins it as a proper Image so qemu -kernel enters at
 * RAMBASE instead of guessing +0x80000.
 */
	.quad	0x0			/* text_offset, 0x08, filled below */
	.quad	0x0			/* image_size, 0x10, filled below */
	.quad	0x0			/* flags, 0x18: LE, 4k pages, unset */
	.quad	0x0			/* reserved 0x20 */
	.quad	0x0			/* reserved 0x28 */
	.quad	0x0			/* reserved 0x30 */
	.quad	0x644d5241		/* magic, 0x38: ARM\x64 */

reset:
	/* keep the dtb pointer before anything clobbers x0 */
	mov	x19, x0

	/* park secondary cores, they have nothing to do yet */
	mrs	x0, mpidr_el1
	and	x0, x0, #0xff
	cbnz	x0, park

	/* which EL are we in, 0x8 per level shifted into bits 3:2 */
	mrs	x0, CurrentEL
	lsr	x0, x0, #2
	cmp	x0, #3
	b.eq	from_el3
	cmp	x0, #2
	b.eq	from_el2
	cmp	x0, #1
	b.eq	mmu_check
	b	park

from_el3:
	/*
	 * EL3 holds the security state. the kernel runs non-secure, so
	 * set SCR_EL3.NS before dropping to EL2, which the kernel
	 * prefers (booting.rst, EL2 RECOMMENDED).
	 */
	mrs	x0, scr_el3
	orr	x0, x0, #1		/* SCR_EL3.NS = 1, non-secure */
	msr	scr_el3, x0
	isb

	mov	x0, #0x3c9		/* EL2h, DAIF masked */
	msr	spsr_el3, x0
	adr	x0, from_el2
	msr	elr_el3, x0
	eret

from_el2:
	/*
	 * scrub the EL2 state and drop to EL1 for the C runtime. the
	 * semihosting hlt trap is an EL1 service on qemu, calling it
	 * from EL2 corrupts the return state. the kernel handoff goes
	 * back to EL2, booting.rst prefers it there, through the
	 * trampoline in boot.S.
	 */

	/* EL1 will be aarch64 */
	mov	x0, #(1 << 31)		/* HCR_EL2.RW = 1 */
	msr	hcr_el2, x0

	/* let EL1 reach the counter, booting.rst demands it */
	mrs	x0, cnthctl_el2
	orr	x0, x0, #(3 << 0)	/* EL1PCTEN | EL1PCEN */
	msr	cnthctl_el2, x0

	/* no traps to EL2 behind EL1's back */
	msr	cptr_el2, xzr
	msr	hstr_el2, xzr
	msr	vpidr_el2, xzr

	/* drop to EL1, SPSR EL1h with DAIF masked */
	mov	x0, #0x3c5
	msr	spsr_el2, x0
	adr	x0, mmu_check
	msr	elr_el2, x0
	eret

mmu_check:
	/*
	 * whether the firmware left an MMU on: M bit, bit 0, of sctlr at
	 * the current EL. writing the register off would not fault, but
	 * the page tables it built are in its own memory, better to kill
	 * it here than trip over a stale mapping.
	 */
	mrs	x0, CurrentEL
	lsr	x0, x0, #2
	cmp	x0, #2
	b.lt	mmu_el1
	mrs	x0, sctlr_el2
	tbz	x0, #0, c_entry

	mov	x0, xzr
	msr	sctlr_el2, x0
	isb
	tlbi	alle2
	dsb	sy
	isb
	b	c_entry

mmu_el1:
	mrs	x0, sctlr_el1
	tbz	x0, #0, c_entry

	mov	x0, xzr
	msr	sctlr_el1, x0
	isb
	ic	iallu
	dsb	sy
	tlbi	vmalle1
	dsb	sy
	isb

c_entry:
	/*
	 * program the counter frequency, the kernel reads CNTFRQ right
	 * away (booting.rst). qemu virt runs the system counter at
	 * 62.5 MHz. the register is RW only at the highest implemented EL.
	 */
	mrs	x0, CurrentEL
	lsr	x0, x0, #2
	cmp	x0, #2
	b.lt	1f
	ldr	x0, =62500000
	msr	cntfrq_el0, x0
	isb
1:
	/* our own vectors, so aborts print instead of vanishing */
	adr	x0, vectors
	mrs	x1, CurrentEL
	lsr	x1, x1, #2
	cmp	x1, #2
	b.lt	2f
	msr	vbar_el2, x0
	b	3f
2:
	msr	vbar_el1, x0
3:
	isb

	/* stack for the bootloader, its own region above the bss */
	ldr	x0, =__stack_top
	mov	sp, x0

	/* export the spin gate array address for the dtb patcher */
	adr	x0, tb_spin_gates
	adrp	x1, tb_spin_gates_ptr
	str	x0, [x1, #:lo12:tb_spin_gates_ptr]

	/* clear bss */
	ldr	x0, =__bss_start
	ldr	x1, =__bss_end
1:	cmp	x0, x1
	b.hs	2f
	str	xzr, [x0], #8
	b	1b
2:

	/* FP/SIMD access, some kernels assume it is on */
	mov	x0, #(3 << 20)
	msr	cpacr_el1, x0
	isb

	/* dtb pointer into C arg 0 */
	mov	x0, x19
	bl	tashaboot_main

	/* if main returns there is nothing sensible to do */
/*
 * the spin table pen, the Wait For Event mechanism from the manual
 * (B2-144, D1-2255). each secondary watches its own gate, the
 * cpu-release-addr the dtb names. WFE clears the event register and
 * sleeps, the kernel writes the secondary entry to the gate, makes
 * it visible, then SEV sets the event register on every PE. the load
 * recheck after each wake covers a release that lands between the
 * load and the WFE. entered with MMU and caches off, left the same.
 */
.globl park_ret
park_ret:
park:
	adr	x0, tb_spin_gates
	mrs	x1, mpidr_el1
	and	x1, x1, #0xff		/* affinity 0, the core number */
	add	x0, x0, x1, lsl #3	/* gate = gates + core * 8 */

	/* diagnostic: stamp arrival, primary prints it later */
	adr	x3, tb_pen_stamps
	strb	w1, [x3, x1]
	sevl
	wfe
	sevl
	wfe

1:
	ldr	x2, [x0]
	cbnz	x2, 2f
	wfe
	b	1b
2:
	/* interrupts masked at release, the manual's boot state */
	msr	daifset, #0xf
	/*
	 * every PE must read the same virtual counter. whatever
	 * ran before this loader could have left a per cpu offset
	 * in the virtual counter view, the kernel has no way to
	 * repair that itself. CNTVOFF_EL2 is writable at EL2 and
	 * the write holds for the EL1 virtual timer the kernel
	 * runs on. below EL2 it is out of reach, the reset value
	 * is the best a lower EL can do.
	 */
	mrs	x4, CurrentEL
	lsr	x4, x4, #2
	cmp	x4, #2
	b.lt	3f
	msr	cntvoff_el2, xzr
	isb
3:
	mov	x0, xzr		/* secondaries enter with x0-x3 zero */
	mov	x1, xzr
	mov	x2, xzr
	mov	x3, xzr
	dsb	sy
	isb
	br	x2

/*
 * exception vectors, the armv8 layout: 16 slots, 128 bytes each, in
 * the order the manual fixes. taken from EL1h the interesting slots
 * are 0x200 sync and 0x380 SError, irq and fiq just park, the
 * bootloader never enables interrupts on purpose.
 */
	.balign	2048
vectors:
	/* 0x000: current EL, SP_EL0 */
	.align	7
	b	exc_sync
	.align	7
	b	exc_park_irq
	.align	7
	b	exc_park_irq
	.align	7
	b	exc_serr

	/* 0x200: current EL, SP_ELx */
	.align	7
	b	exc_sync
	.align	7
	b	exc_park_irq
	.align	7
	b	exc_park_irq
	.align	7
	b	exc_serr

	/* 0x400: lower EL, AArch64 */
	.align	7
	b	exc_sync
	.align	7
	b	exc_park_irq
	.align	7
	b	exc_park_irq
	.align	7
	b	exc_serr

	/* 0x600: lower EL, AArch32 */
	.align	7
	b	exc_sync
	.align	7
	b	exc_park_irq
	.align	7
	b	exc_park_irq
	.align	7
	b	exc_serr

.pushsection .data.tb_spin, "aw"
.align 3
.globl tb_spin_gates
tb_spin_gates:
	.quad	0, 0, 0, 0, 0, 0, 0, 0
.globl tb_spin_gates_ptr
tb_spin_gates_ptr:
	.quad	0
.globl tb_pen_stamps
tb_pen_stamps:
	.byte	0, 0, 0, 0, 0, 0, 0, 0
.popsection

exc_sync:
	stp	x29, x30, [sp, #-16]!
	mov	x29, sp
	mrs	x3, CurrentEL
	lsr	x3, x3, #2
	cmp	x3, #2
	b.lt	1f
	mrs	x0, esr_el2
	mrs	x2, elr_el2
	lsr	x1, x0, #26
	cmp	x1, #0x16		/* HVC from lower EL */
	b.eq	hvc_from_el1
	mrs	x1, far_el2
	b	2f
1:
	mrs	x0, esr_el1
	mrs	x1, far_el1
2:
	/* x2 = the faulting PC when it is the sync path */
	mrs	x4, CurrentEL
	lsr	x4, x4, #2
	cmp	x4, #2
	b.lt	3f
	mrs	x2, elr_el2
	b	4f
3:
	mrs	x2, elr_el1
4:
	bl	exc_report
	ldp	x29, x30, [sp], #16
	b	park

/*
 * HVC from EL1, the PSCI conduit. x0-x3 are the PSCI args in the
 * caller registers, dispatch and return in x0. ELR_EL2 is already
 * the resume point, eret takes it back.
 */
hvc_from_el1:
	/*
	 * the lower EL sync slot. three arrivals share it: PSCI hvc
	 * from the kernel (EC 0x16, PSCI id in x0), our own boot
	 * handoff (hvc with the payload entry in x8), and semihosting
	 * hlt #0xf000 from the EL1 C runtime (EC 0x14). qemu only
	 * answers the hlt when it executes at EL2, so the handler
	 * replays the trap at EL2 and erets home with the result.
	 */
	mrs	x1, esr_el2
	lsr	x1, x1, #26		/* EC */
	cmp	x1, #0x14		/* HLT from lower EL, semihosting */
	b.eq	smh_replay

	/*
	 * the hvc arrives with either a PSCI function id in x0 (the
	 * kernel calling) or the boot handoff staging the payload
	 * entry in x8 and the dtb in x0. PSCI ids have the 0x84/0xc4
	 * prefix, a dtb pointer never does.
	 */
	lsr	x1, x0, #24
	cmp	x1, #0x84
	b.eq	psci_call
	cmp	x1, #0xc4
	b.eq	psci_call

	/* the boot handoff: ELR_EL2 = entry, eret to the payload */
	msr	elr_el2, x8
	eret

smh_replay:
	/*
	 * x0 holds the semihosting syscall number, x1 the parameter
	 * block, both live in the caller's registers. replay the hlt
	 * here at EL2 where qemu answers it, then eret back.
	 */
	hlt	#0xf000
	eret

psci_call:
	stp	x4, x5, [sp, #-16]!
	stp	x6, x7, [sp, #-16]!
	stp	x29, x30, [sp, #-16]!
	mov	x29, sp

	bl	tb_psci_dispatch

	ldp	x29, x30, [sp], #16
	ldp	x6, x7, [sp], #16
	ldp	x4, x5, [sp], #16
	ldp	x29, x30, [sp], #16
	eret

exc_serr:
	stp	x29, x30, [sp, #-16]!
	mov	x29, sp
	mrs	x3, CurrentEL
	lsr	x3, x3, #2
	cmp	x3, #2
	b.lt	1f
	mrs	x0, esr_el2
	b	2f
1:
	mrs	x0, esr_el1
2:
	mov	x1, #0
	mov	x2, lr
	bl	exc_report
	/*
	 * an SError while this loader runs means the machine is
	 * broken. handing the kernel a cpu that already lost is
	 * worse than stopping: report, then drive the reset domain
	 * the same way PSCI SYSTEM_RESET does. the reset call does
	 * not return, the park below is the fallback if a reset
	 * domain ignores the request.
	 */
	bl	tb_system_reset
	ldp	x29, x30, [sp], #16
	b	park

exc_park_irq:
	b	park