Implement support for the Zicbom and Svpbmt extensions. Rework the cache

flushing code to operate on virtual addresses instead of physical
addresses.  Seems the Zicbom implementation on the SpacemiT X60 cores
doesn't flush the caches if the mapping is non-cachable.  So adjust
_pmap_kenter_pa() to use a temporary cachable mapping to clean a page we
want to map non-cachable.

ok jca@

sys/arch/riscv64/dev/sfcc.c

@@ -1,4 +1,4 @@
-/*	$OpenBSD: sfcc.c,v 1.4 2022/12/27 21:13:25 kettenis Exp $	*/
+/*	$OpenBSD: sfcc.c,v 1.5 2026/04/05 11:48:17 kettenis Exp $	*/
 /*
  * Copyright (c) 2021 Mark Kettenis <kettenis@openbsd.org>
  *
@@ -19,6 +19,8 @@
 #include <sys/systm.h>
 #include <sys/device.h>
 
+#include <uvm/uvm_extern.h>
+
 #include <machine/bus.h>
 #include <machine/cpufunc.h>
 #include <machine/fdt.h>
@@ -50,7 +52,7 @@ struct cfdriver sfcc_cd = {
 	NULL, "sfcc", DV_DULL
 };
 
-void	sfcc_cache_wbinv_range(paddr_t, psize_t);
+void	sfcc_cache_wbinv_range(vaddr_t, vsize_t);
 
 int
 sfcc_match(struct device *parent, void *match, void *aux)
@@ -94,19 +96,23 @@ sfcc_attach(struct device *parent, struct device *self, void *aux)
 }
 
 void
-sfcc_cache_wbinv_range(paddr_t pa, psize_t len)
+sfcc_cache_wbinv_range(vaddr_t va, vsize_t len)
 {
 	struct sfcc_softc *sc = sfcc_sc;
-	paddr_t end, mask;
+	vaddr_t end, mask;
+	paddr_t pa;
 
 	mask = sc->sc_line_size - 1;
-	end = (pa + len + mask) & ~mask;
-	pa &= ~mask;
+	end = (va + len + mask) & ~mask;
+	va &= ~mask;
 
 	__asm volatile ("fence iorw,iorw" ::: "memory");
-	while (pa != end) {
-		bus_space_write_8(sc->sc_iot, sc->sc_ioh, SFCC_FLUSH64, pa);
-		__asm volatile ("fence iorw,iorw" ::: "memory");
-		pa += sc->sc_line_size;
+	while (va != end) {
+		if (pmap_extract(pmap_kernel(), va, &pa)) {
+			bus_space_write_8(sc->sc_iot, sc->sc_ioh,
+			    SFCC_FLUSH64, pa);
+			__asm volatile ("fence iorw,iorw" ::: "memory");
+		}
+		va += sc->sc_line_size;
 	}
 }

sys/arch/riscv64/include/cpufunc.h

@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpufunc.h,v 1.6 2023/08/21 20:17:30 miod Exp $	*/
+/*	$OpenBSD: cpufunc.h,v 1.7 2026/04/05 11:48:17 kettenis Exp $	*/
 
 /*-
  * Copyright (c) 2014 Andrew Turner
@@ -89,9 +89,9 @@ sfence_vma_page_asid(uintptr_t addr, uint64_t asid)
 extern int64_t dcache_line_size;
 extern int64_t icache_line_size;
 
-extern void (*cpu_dcache_wbinv_range)(paddr_t, psize_t);
-extern void (*cpu_dcache_inv_range)(paddr_t, psize_t);
-extern void (*cpu_dcache_wb_range)(paddr_t, psize_t);
+extern void (*cpu_dcache_wbinv_range)(vaddr_t, vsize_t);
+extern void (*cpu_dcache_inv_range)(vaddr_t, vsize_t);
+extern void (*cpu_dcache_wb_range)(vaddr_t, vsize_t);
 
 static __inline void
 load_satp(uint64_t val)

sys/arch/riscv64/include/pmap.h

@@ -1,4 +1,4 @@
-/*	$OpenBSD: pmap.h,v 1.14 2025/02/03 17:59:40 jca Exp $	*/
+/*	$OpenBSD: pmap.h,v 1.15 2026/04/05 11:48:17 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2019-2020 Brian Bamsch <bbamsch@google.com>
@@ -75,6 +75,10 @@ struct pmap {
 #define PMAP_NOCACHE	0x1 /* non-cacheable memory */
 #define PMAP_DEVICE	0x2 /* device memory */
 
+extern uint64_t pmap_pma;
+extern uint64_t pmap_nc;
+extern uint64_t pmap_io;
+
 #define PG_PMAP_MOD		PG_PMAP0
 #define PG_PMAP_REF		PG_PMAP1
 #define PG_PMAP_EXE		PG_PMAP2

sys/arch/riscv64/include/pte.h

@@ -1,4 +1,4 @@
-/*	$OpenBSD: pte.h,v 1.4 2024/10/14 12:02:16 jsg Exp $	*/
+/*	$OpenBSD: pte.h,v 1.5 2026/04/05 11:48:17 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2019 Brian Bamsch <bbamsch@google.com>
@@ -38,6 +38,14 @@
 #define	PTE_RX		(PTE_R | PTE_X)
 #define	PTE_KERN	(PTE_V | PTE_R | PTE_W | PTE_A | PTE_D)
 
+/* Svpmbt extension */
+#define	PTE_PMA		(0ULL << 61)
+#define	PTE_NC		(1ULL << 61)
+#define	PTE_IO		(2ULL << 61)
+
+/* Svnapot extension */
+#define	PTE_N		(1ULL << 63)
+
 /* T-Head extended page attributes */
 #define	PTE_THEAD_SO	(1ULL << 63)
 #define	PTE_THEAD_C	(1ULL << 62)

sys/arch/riscv64/riscv64/bus_dma.c

@@ -1,4 +1,4 @@
-/*	$OpenBSD: bus_dma.c,v 1.7 2022/12/29 11:35:01 kettenis Exp $	*/
+/*	$OpenBSD: bus_dma.c,v 1.8 2026/04/05 11:48:17 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2003-2004 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -366,16 +366,16 @@ _dmamap_unload(bus_dma_tag_t t, bus_dmamap_t map)
 }
 
 static void
-_dmamap_sync_segment(paddr_t pa, psize_t len, int ops)
+_dmamap_sync_segment(vaddr_t va, vsize_t len, int ops)
 {
 	switch (ops) {
 	case BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE:
 	case BUS_DMASYNC_PREREAD:
-		cpu_dcache_wbinv_range(pa, len);
+		cpu_dcache_wbinv_range(va, len);
 		break;
 
 	case BUS_DMASYNC_PREWRITE:
-		cpu_dcache_wb_range(pa, len);
+		cpu_dcache_wb_range(va, len);
 		break;
 
 	/*
@@ -387,7 +387,7 @@ _dmamap_sync_segment(paddr_t pa, psize_t len, int ops)
 	case BUS_DMASYNC_POSTREAD:
 	case BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE:
 		membar_sync();
-		cpu_dcache_inv_range(pa, len);
+		cpu_dcache_inv_range(va, len);
 		break;
 	}
 }
@@ -417,18 +417,18 @@ _dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t addr,
 	curseg = 0;
 
 	while (size && nsegs) {
-		paddr_t paddr;
+		vaddr_t vaddr;
 		bus_size_t ssize;
 
 		ssize = map->dm_segs[curseg].ds_len;
-		paddr = map->dm_segs[curseg]._ds_paddr;
+		vaddr = map->dm_segs[curseg]._ds_vaddr;
 
 		if (addr != 0) {
 			if (addr >= ssize) {
 				addr -= ssize;
 				ssize = 0;
 			} else {
-				paddr += addr;
+				vaddr += addr;
 				ssize -= addr;
 				addr = 0;
 			}
@@ -437,7 +437,7 @@ _dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t addr,
 			ssize = size;
 
 		if (ssize != 0) {
-			_dmamap_sync_segment(paddr, ssize, op);
+			_dmamap_sync_segment(vaddr, ssize, op);
 			size -= ssize;
 		}
 		curseg++;

sys/arch/riscv64/riscv64/cpu.c

@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.c,v 1.24 2026/04/03 22:01:46 sf Exp $	*/
+/*	$OpenBSD: cpu.c,v 1.25 2026/04/05 11:48:17 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2016 Dale Rahn <drahn@dalerahn.com>
@@ -30,6 +30,7 @@
 #include <machine/cpufunc.h>
 #include <machine/elf.h>
 #include <machine/fdt.h>
+#include <machine/pmap.h>
 #include <machine/sbi.h>
 
 #include <dev/ofw/openfirm.h>
@@ -89,6 +90,9 @@ const struct vendor {
 	{ 0, NULL }
 };
 
+int riscv_has_svpbmt;
+int riscv_has_zicbom;
+
 char cpu_model[64];
 int cpu_node;
 
@@ -109,11 +113,15 @@ int cpu_errata_sifive_cip_1200;
 
 void	cpu_opp_init(struct cpu_info *, uint32_t);
 
-void	thead_dcache_wbinv_range(paddr_t, psize_t);
-void	thead_dcache_inv_range(paddr_t, psize_t);
-void	thead_dcache_wb_range(paddr_t, psize_t);
+size_t	zicbom_dcache_line_size;
+void	zicbom_dcache_wbinv_range(vaddr_t, vsize_t);
+void	zicbom_dcache_inv_range(vaddr_t, vsize_t);
+void	zicbom_dcache_wb_range(vaddr_t, vsize_t);
 
 size_t	thead_dcache_line_size;
+void	thead_dcache_wbinv_range(vaddr_t, vsize_t);
+void	thead_dcache_inv_range(vaddr_t, vsize_t);
+void	thead_dcache_wb_range(vaddr_t, vsize_t);
 
 void
 cpu_identify(struct cpu_info *ci)
@@ -123,7 +131,10 @@ cpu_identify(struct cpu_info *ci)
 	const char *vendor_name = NULL;
 	const char *arch_name = NULL;
 	struct arch *archlist = cpu_arch_none;
-	int i;
+	char *names;
+	char *name;
+	char *end;
+	int i, len;
 
 	mvendorid = sbi_get_mvendorid();
 	marchid = sbi_get_marchid();
@@ -166,6 +177,36 @@ cpu_identify(struct cpu_info *ci)
 			snprintf(cpu_model, sizeof(cpu_model), "Unknown");
 	}
 
+	len = OF_getproplen(ci->ci_node, "riscv,isa-extensions");
+	if (len > 0) {
+		names = malloc(len, M_TEMP, M_WAITOK);
+		OF_getprop(ci->ci_node, "riscv,isa-extensions", names, len);
+		end = names + len;
+		name = names;
+		while (name < end) {
+			if (strcmp(name, "svpbmt") == 0)
+				riscv_has_svpbmt = 1;
+			if (strcmp(name, "zicbom") == 0)
+				riscv_has_zicbom = 1;
+			name += strlen(name) + 1;
+		}
+		free(names, M_TEMP, len);
+	}
+
+	if (riscv_has_svpbmt) {
+		pmap_pma = PTE_PMA;
+		pmap_nc = PTE_NC;
+		pmap_io = PTE_IO;
+	}
+
+	if (riscv_has_zicbom) {
+		cpu_dcache_wbinv_range = zicbom_dcache_wbinv_range;
+		cpu_dcache_inv_range = zicbom_dcache_inv_range;
+		cpu_dcache_wb_range = zicbom_dcache_wb_range;
+		zicbom_dcache_line_size =
+		    OF_getpropint(ci->ci_node, "riscv,cbom-block-size", 64);
+	}
+
 	/* Handle errata. */
 	if (mvendorid == CPU_VENDOR_SIFIVE && marchid == CPU_ARCH_U7)
 		cpu_errata_sifive_cip_1200 = 1;
@@ -346,70 +387,122 @@ cpu_clockspeed(int *freq)
 }
 
 void
-cpu_cache_nop_range(paddr_t pa, psize_t len)
+cpu_cache_nop_range(vaddr_t va, vsize_t len)
 {
 }
 
+__attribute__((target("arch=+zicbom")))
 void
-thead_dcache_wbinv_range(paddr_t pa, psize_t len)
+zicbom_dcache_wbinv_range(vaddr_t va, vsize_t len)
 {
-	paddr_t end, mask;
+	vaddr_t end, mask;
 
-	mask = thead_dcache_line_size - 1;
-	end = (pa + len + mask) & ~mask;
-	pa &= ~mask;
+	mask = zicbom_dcache_line_size - 1;
+	end = (va + len + mask) & ~mask;
+	va &= ~mask;
 
-	while (pa != end) {
-		/* th.dcache.cipa a0 */
-		__asm volatile ("mv a0, %0; .long 0x02b5000b" :: "r"(pa)
-		    : "a0", "memory");
-		pa += thead_dcache_line_size;
+	while (va != end) {
+		__asm volatile ("cbo.flush (%0)" :: "r"(va) : "memory");
+		va += zicbom_dcache_line_size;
 	}
-	/* th.sync.s */
-	__asm volatile (".long 0x0190000b" ::: "memory");
+
+	__asm volatile ("fence iorw,iorw" ::: "memory");
 }
 
+__attribute__((target("arch=+zicbom")))
 void
-thead_dcache_inv_range(paddr_t pa, psize_t len)
+zicbom_dcache_inv_range(vaddr_t va, vsize_t len)
 {
-	paddr_t end, mask;
+	vaddr_t end, mask;
+
+	mask = zicbom_dcache_line_size - 1;
+	end = (va + len + mask) & ~mask;
+	va &= ~mask;
+
+	while (va != end) {
+		__asm volatile ("cbo.inval (%0)" :: "r"(va) : "memory");
+		va += zicbom_dcache_line_size;
+	}
+
+	__asm volatile ("fence iorw,iorw" ::: "memory");
+}
+
+__attribute__((target("arch=+zicbom")))
+void
+zicbom_dcache_wb_range(vaddr_t va, vsize_t len)
+{
+	vaddr_t end, mask;
+
+	mask = zicbom_dcache_line_size - 1;
+	end = (va + len + mask) & ~mask;
+	va &= ~mask;
+
+	while (va != end) {
+		__asm volatile ("cbo.clean (%0)" :: "r"(va) : "memory");
+		va += zicbom_dcache_line_size;
+	}
+
+	__asm volatile ("fence iorw,iorw" ::: "memory");
+}
+
+__attribute__((target("arch=+xtheadcmo,+xtheadsync")))
+void
+thead_dcache_wbinv_range(vaddr_t va, vsize_t len)
+{
+	vaddr_t end, mask;
 
 	mask = thead_dcache_line_size - 1;
-	end = (pa + len + mask) & ~mask;
-	pa &= ~mask;
+	end = (va + len + mask) & ~mask;
+	va &= ~mask;
 
-	while (pa != end) {
+	while (va != end) {
+		__asm volatile ("th.dcache.civa %0" :: "r"(va) : "memory");
+		va += thead_dcache_line_size;
+	}
+
+	__asm volatile ("th.sync.s" ::: "memory");
+}
+
+__attribute__((target("arch=+xtheadcmo,+xtheadsync")))
+void
+thead_dcache_inv_range(vaddr_t va, vsize_t len)
+{
+	vaddr_t end, mask;
+
+	mask = thead_dcache_line_size - 1;
+	end = (va + len + mask) & ~mask;
+	va &= ~mask;
+
+	while (va != end) {
 		/* th.dcache.ipa a0 */
-		__asm volatile ("mv a0, %0; .long 0x02a5000b" :: "r"(pa)
-		    : "a0", "memory");
-		pa += thead_dcache_line_size;
+		__asm volatile ("th.dcache.iva %0" :: "r"(va) : "memory");
+		va += thead_dcache_line_size;
 	}
-	/* th.sync.s */
-	__asm volatile (".long 0x0190000b" ::: "memory");
+
+	__asm volatile ("th.sync.s" ::: "memory");
 }
 
+__attribute__((target("arch=+xtheadcmo,+xtheadsync")))
 void
-thead_dcache_wb_range(paddr_t pa, psize_t len)
+thead_dcache_wb_range(vaddr_t va, vsize_t len)
 {
-	paddr_t end, mask;
+	vaddr_t end, mask;
 
 	mask = thead_dcache_line_size - 1;
-	end = (pa + len + mask) & ~mask;
-	pa &= ~mask;
+	end = (va + len + mask) & ~mask;
+	va &= ~mask;
 
-	while (pa != end) {
-		/* th.dcache.cpa a0 */
-		__asm volatile ("mv a0, %0; .long 0x0295000b" :: "r"(pa)
-		    : "a0", "memory");
-		pa += thead_dcache_line_size;
+	while (va != end) {
+		__asm volatile ("th.dcache.cva %0" :: "r"(va) : "memory");
+		va += thead_dcache_line_size;
 	}
-	/* th.sync.s */
-	__asm volatile (".long 0x0190000b" ::: "memory");
+
+	__asm volatile ("th.sync.s" ::: "memory");
 }
 
-void (*cpu_dcache_wbinv_range)(paddr_t, psize_t) = cpu_cache_nop_range;
-void (*cpu_dcache_inv_range)(paddr_t, psize_t) = cpu_cache_nop_range;
-void (*cpu_dcache_wb_range)(paddr_t, psize_t) = cpu_cache_nop_range;
+void (*cpu_dcache_wbinv_range)(vaddr_t, vsize_t) = cpu_cache_nop_range;
+void (*cpu_dcache_inv_range)(vaddr_t, vsize_t) = cpu_cache_nop_range;
+void (*cpu_dcache_wb_range)(vaddr_t, vsize_t) = cpu_cache_nop_range;
 
 #ifdef MULTIPROCESSOR
 

sys/arch/riscv64/riscv64/pmap.c

@@ -1,4 +1,4 @@
-/*	$OpenBSD: pmap.c,v 1.47 2025/07/14 12:23:28 jca Exp $	*/
+/*	$OpenBSD: pmap.c,v 1.48 2026/04/05 11:48:17 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2019-2020 Brian Bamsch <bbamsch@google.com>
@@ -301,7 +301,7 @@ const pt_entry_t ap_bits_kern[8] = {
 	[PROT_EXEC|PROT_WRITE|PROT_READ]	= PTE_A|PTE_X|PTE_R|PTE_D|PTE_W,
 };
 
-/* PBMT encodings for the Svpmbt modes. */
+/* PBMT encodings for the Svpbmt modes. */
 uint64_t pmap_pma;
 uint64_t pmap_nc;
 uint64_t pmap_io;
@@ -693,6 +693,12 @@ _pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, int flags, int cache)
 	struct pte_desc *pted;
 	struct vm_page *pg;
 
+	pg = PHYS_TO_VM_PAGE(pa);
+	if (pg && cache == PMAP_CACHE_CI) {
+		_pmap_kenter_pa(va, pa, prot, flags, PMAP_CACHE_WB);
+		cpu_dcache_wbinv_range(va & ~PAGE_MASK, PAGE_SIZE);
+	}
+
 	pted = pmap_vp_lookup(pm, va, NULL);
 
 	/* Do not have pted for this, get one and put it in VP */
@@ -717,10 +723,6 @@ _pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, int flags, int cache)
 	 */
 	pmap_pte_insert(pted);
 	tlb_flush_page(pm, va & ~PAGE_MASK);
-
-	pg = PHYS_TO_VM_PAGE(pa);
-	if (pg && cache == PMAP_CACHE_CI)
-		cpu_dcache_wbinv_range(pa & ~PAGE_MASK, PAGE_SIZE);
 }
 
 void