arch/arm64/lib/cache_va.c


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73

/* SPDX-License-Identifier: GPL-2.0+ */
/*
 * cache_va.c - cache maintenance by virtual address, the operations
 * the manual prescribes for boot handoff: clean to point of
 * coherency (dc cvac), invalidate (dc ivac), and clean and
 * invalidate (dc civac), plus icache invalidate by VA to the point
 * of unification (ic ivau). by VA beats by set and way when the
 * address range is known, the manual's own guidance, set and way
 * only for the full flush cases in cache.S.
 *
 * Copyright (C) 2026 Bradley Morgan <brads@mainlining.org>
 */

#include <stdint.h>
#include <sys/types.h>

#define CACHE_LINE_SHIFT	6	/* 64 byte lines on cortex-a class */
#define CACHE_LINE_SIZE		(1 << CACHE_LINE_SHIFT)

void tb_clean_dcache_range(uintptr_t start, size_t len)
{
	uintptr_t line = start & ~(uintptr_t)(CACHE_LINE_SIZE - 1);
	uintptr_t end = start + len;

	while (line < end) {
		asm volatile("dc cvac, %0" :: "r" (line) : "memory");
		line += CACHE_LINE_SIZE;
	}

	asm volatile("dsb sy" ::: "memory");
}

void tb_inval_dcache_range(uintptr_t start, size_t len)
{
	uintptr_t line = start & ~(uintptr_t)(CACHE_LINE_SIZE - 1);
	uintptr_t end = start + len;

	while (line < end) {
		asm volatile("dc ivac, %0" :: "r" (line) : "memory");
		line += CACHE_LINE_SIZE;
	}

	asm volatile("dsb sy" ::: "memory");
}

void tb_clean_inval_dcache_range(uintptr_t start, size_t len)
{
	uintptr_t line = start & ~(uintptr_t)(CACHE_LINE_SIZE - 1);
	uintptr_t end = start + len;

	while (line < end) {
		asm volatile("dc civac, %0" :: "r" (line) : "memory");
		line += CACHE_LINE_SIZE;
	}

	asm volatile("dsb sy" ::: "memory");
}

void tb_inval_icache_range(uintptr_t start, size_t len)
{
	uintptr_t line = start & ~(uintptr_t)(CACHE_LINE_SIZE - 1);
	uintptr_t end = start + len;

	while (line < end) {
		asm volatile("ic ivau, %0" :: "r" (line) : "memory");
		line += CACHE_LINE_SIZE;
	}

	asm volatile(
		"dsb ish\n"
		"isb\n"
		::: "memory");
}