diff --git a/sys/arch/riscv64/dev/sfcc.c b/sys/arch/riscv64/dev/sfcc.c index 15f01967261..70c395680a9 100644 --- a/sys/arch/riscv64/dev/sfcc.c +++ b/sys/arch/riscv64/dev/sfcc.c @@ -1,4 +1,4 @@ -/* $OpenBSD: sfcc.c,v 1.4 2022/12/27 21:13:25 kettenis Exp $ */ +/* $OpenBSD: sfcc.c,v 1.5 2026/04/05 11:48:17 kettenis Exp $ */ /* * Copyright (c) 2021 Mark Kettenis * @@ -19,6 +19,8 @@ #include #include +#include + #include #include #include @@ -50,7 +52,7 @@ struct cfdriver sfcc_cd = { NULL, "sfcc", DV_DULL }; -void sfcc_cache_wbinv_range(paddr_t, psize_t); +void sfcc_cache_wbinv_range(vaddr_t, vsize_t); int sfcc_match(struct device *parent, void *match, void *aux) @@ -94,19 +96,23 @@ sfcc_attach(struct device *parent, struct device *self, void *aux) } void -sfcc_cache_wbinv_range(paddr_t pa, psize_t len) +sfcc_cache_wbinv_range(vaddr_t va, vsize_t len) { struct sfcc_softc *sc = sfcc_sc; - paddr_t end, mask; + vaddr_t end, mask; + paddr_t pa; mask = sc->sc_line_size - 1; - end = (pa + len + mask) & ~mask; - pa &= ~mask; + end = (va + len + mask) & ~mask; + va &= ~mask; __asm volatile ("fence iorw,iorw" ::: "memory"); - while (pa != end) { - bus_space_write_8(sc->sc_iot, sc->sc_ioh, SFCC_FLUSH64, pa); - __asm volatile ("fence iorw,iorw" ::: "memory"); - pa += sc->sc_line_size; + while (va != end) { + if (pmap_extract(pmap_kernel(), va, &pa)) { + bus_space_write_8(sc->sc_iot, sc->sc_ioh, + SFCC_FLUSH64, pa); + __asm volatile ("fence iorw,iorw" ::: "memory"); + } + va += sc->sc_line_size; } } diff --git a/sys/arch/riscv64/include/cpufunc.h b/sys/arch/riscv64/include/cpufunc.h index a55a06fcd3a..43efd38dab0 100644 --- a/sys/arch/riscv64/include/cpufunc.h +++ b/sys/arch/riscv64/include/cpufunc.h @@ -1,4 +1,4 @@ -/* $OpenBSD: cpufunc.h,v 1.6 2023/08/21 20:17:30 miod Exp $ */ +/* $OpenBSD: cpufunc.h,v 1.7 2026/04/05 11:48:17 kettenis Exp $ */ /*- * Copyright (c) 2014 Andrew Turner @@ -89,9 +89,9 @@ sfence_vma_page_asid(uintptr_t addr, uint64_t asid) extern int64_t dcache_line_size; extern int64_t icache_line_size; -extern void (*cpu_dcache_wbinv_range)(paddr_t, psize_t); -extern void (*cpu_dcache_inv_range)(paddr_t, psize_t); -extern void (*cpu_dcache_wb_range)(paddr_t, psize_t); +extern void (*cpu_dcache_wbinv_range)(vaddr_t, vsize_t); +extern void (*cpu_dcache_inv_range)(vaddr_t, vsize_t); +extern void (*cpu_dcache_wb_range)(vaddr_t, vsize_t); static __inline void load_satp(uint64_t val) diff --git a/sys/arch/riscv64/include/pmap.h b/sys/arch/riscv64/include/pmap.h index e59fdf581c5..cdc3ff7c0e6 100644 --- a/sys/arch/riscv64/include/pmap.h +++ b/sys/arch/riscv64/include/pmap.h @@ -1,4 +1,4 @@ -/* $OpenBSD: pmap.h,v 1.14 2025/02/03 17:59:40 jca Exp $ */ +/* $OpenBSD: pmap.h,v 1.15 2026/04/05 11:48:17 kettenis Exp $ */ /* * Copyright (c) 2019-2020 Brian Bamsch @@ -75,6 +75,10 @@ struct pmap { #define PMAP_NOCACHE 0x1 /* non-cacheable memory */ #define PMAP_DEVICE 0x2 /* device memory */ +extern uint64_t pmap_pma; +extern uint64_t pmap_nc; +extern uint64_t pmap_io; + #define PG_PMAP_MOD PG_PMAP0 #define PG_PMAP_REF PG_PMAP1 #define PG_PMAP_EXE PG_PMAP2 diff --git a/sys/arch/riscv64/include/pte.h b/sys/arch/riscv64/include/pte.h index 6d9bd29fb89..faca511e7af 100644 --- a/sys/arch/riscv64/include/pte.h +++ b/sys/arch/riscv64/include/pte.h @@ -1,4 +1,4 @@ -/* $OpenBSD: pte.h,v 1.4 2024/10/14 12:02:16 jsg Exp $ */ +/* $OpenBSD: pte.h,v 1.5 2026/04/05 11:48:17 kettenis Exp $ */ /* * Copyright (c) 2019 Brian Bamsch @@ -38,6 +38,14 @@ #define PTE_RX (PTE_R | PTE_X) #define PTE_KERN (PTE_V | PTE_R | PTE_W | PTE_A | PTE_D) +/* Svpmbt extension */ +#define PTE_PMA (0ULL << 61) +#define PTE_NC (1ULL << 61) +#define PTE_IO (2ULL << 61) + +/* Svnapot extension */ +#define PTE_N (1ULL << 63) + /* T-Head extended page attributes */ #define PTE_THEAD_SO (1ULL << 63) #define PTE_THEAD_C (1ULL << 62) diff --git a/sys/arch/riscv64/riscv64/bus_dma.c b/sys/arch/riscv64/riscv64/bus_dma.c index 3cd6db63ec6..2600fccf085 100644 --- a/sys/arch/riscv64/riscv64/bus_dma.c +++ b/sys/arch/riscv64/riscv64/bus_dma.c @@ -1,4 +1,4 @@ -/* $OpenBSD: bus_dma.c,v 1.7 2022/12/29 11:35:01 kettenis Exp $ */ +/* $OpenBSD: bus_dma.c,v 1.8 2026/04/05 11:48:17 kettenis Exp $ */ /* * Copyright (c) 2003-2004 Opsycon AB (www.opsycon.se / www.opsycon.com) @@ -366,16 +366,16 @@ _dmamap_unload(bus_dma_tag_t t, bus_dmamap_t map) } static void -_dmamap_sync_segment(paddr_t pa, psize_t len, int ops) +_dmamap_sync_segment(vaddr_t va, vsize_t len, int ops) { switch (ops) { case BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE: case BUS_DMASYNC_PREREAD: - cpu_dcache_wbinv_range(pa, len); + cpu_dcache_wbinv_range(va, len); break; case BUS_DMASYNC_PREWRITE: - cpu_dcache_wb_range(pa, len); + cpu_dcache_wb_range(va, len); break; /* @@ -387,7 +387,7 @@ _dmamap_sync_segment(paddr_t pa, psize_t len, int ops) case BUS_DMASYNC_POSTREAD: case BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE: membar_sync(); - cpu_dcache_inv_range(pa, len); + cpu_dcache_inv_range(va, len); break; } } @@ -417,18 +417,18 @@ _dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t addr, curseg = 0; while (size && nsegs) { - paddr_t paddr; + vaddr_t vaddr; bus_size_t ssize; ssize = map->dm_segs[curseg].ds_len; - paddr = map->dm_segs[curseg]._ds_paddr; + vaddr = map->dm_segs[curseg]._ds_vaddr; if (addr != 0) { if (addr >= ssize) { addr -= ssize; ssize = 0; } else { - paddr += addr; + vaddr += addr; ssize -= addr; addr = 0; } @@ -437,7 +437,7 @@ _dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t addr, ssize = size; if (ssize != 0) { - _dmamap_sync_segment(paddr, ssize, op); + _dmamap_sync_segment(vaddr, ssize, op); size -= ssize; } curseg++; diff --git a/sys/arch/riscv64/riscv64/cpu.c b/sys/arch/riscv64/riscv64/cpu.c index 95ed0de1621..f5955bcedfc 100644 --- a/sys/arch/riscv64/riscv64/cpu.c +++ b/sys/arch/riscv64/riscv64/cpu.c @@ -1,4 +1,4 @@ -/* $OpenBSD: cpu.c,v 1.24 2026/04/03 22:01:46 sf Exp $ */ +/* $OpenBSD: cpu.c,v 1.25 2026/04/05 11:48:17 kettenis Exp $ */ /* * Copyright (c) 2016 Dale Rahn @@ -30,6 +30,7 @@ #include #include #include +#include #include #include @@ -89,6 +90,9 @@ const struct vendor { { 0, NULL } }; +int riscv_has_svpbmt; +int riscv_has_zicbom; + char cpu_model[64]; int cpu_node; @@ -109,11 +113,15 @@ int cpu_errata_sifive_cip_1200; void cpu_opp_init(struct cpu_info *, uint32_t); -void thead_dcache_wbinv_range(paddr_t, psize_t); -void thead_dcache_inv_range(paddr_t, psize_t); -void thead_dcache_wb_range(paddr_t, psize_t); +size_t zicbom_dcache_line_size; +void zicbom_dcache_wbinv_range(vaddr_t, vsize_t); +void zicbom_dcache_inv_range(vaddr_t, vsize_t); +void zicbom_dcache_wb_range(vaddr_t, vsize_t); size_t thead_dcache_line_size; +void thead_dcache_wbinv_range(vaddr_t, vsize_t); +void thead_dcache_inv_range(vaddr_t, vsize_t); +void thead_dcache_wb_range(vaddr_t, vsize_t); void cpu_identify(struct cpu_info *ci) @@ -123,7 +131,10 @@ cpu_identify(struct cpu_info *ci) const char *vendor_name = NULL; const char *arch_name = NULL; struct arch *archlist = cpu_arch_none; - int i; + char *names; + char *name; + char *end; + int i, len; mvendorid = sbi_get_mvendorid(); marchid = sbi_get_marchid(); @@ -166,6 +177,36 @@ cpu_identify(struct cpu_info *ci) snprintf(cpu_model, sizeof(cpu_model), "Unknown"); } + len = OF_getproplen(ci->ci_node, "riscv,isa-extensions"); + if (len > 0) { + names = malloc(len, M_TEMP, M_WAITOK); + OF_getprop(ci->ci_node, "riscv,isa-extensions", names, len); + end = names + len; + name = names; + while (name < end) { + if (strcmp(name, "svpbmt") == 0) + riscv_has_svpbmt = 1; + if (strcmp(name, "zicbom") == 0) + riscv_has_zicbom = 1; + name += strlen(name) + 1; + } + free(names, M_TEMP, len); + } + + if (riscv_has_svpbmt) { + pmap_pma = PTE_PMA; + pmap_nc = PTE_NC; + pmap_io = PTE_IO; + } + + if (riscv_has_zicbom) { + cpu_dcache_wbinv_range = zicbom_dcache_wbinv_range; + cpu_dcache_inv_range = zicbom_dcache_inv_range; + cpu_dcache_wb_range = zicbom_dcache_wb_range; + zicbom_dcache_line_size = + OF_getpropint(ci->ci_node, "riscv,cbom-block-size", 64); + } + /* Handle errata. */ if (mvendorid == CPU_VENDOR_SIFIVE && marchid == CPU_ARCH_U7) cpu_errata_sifive_cip_1200 = 1; @@ -346,70 +387,122 @@ cpu_clockspeed(int *freq) } void -cpu_cache_nop_range(paddr_t pa, psize_t len) +cpu_cache_nop_range(vaddr_t va, vsize_t len) { } +__attribute__((target("arch=+zicbom"))) void -thead_dcache_wbinv_range(paddr_t pa, psize_t len) +zicbom_dcache_wbinv_range(vaddr_t va, vsize_t len) { - paddr_t end, mask; + vaddr_t end, mask; - mask = thead_dcache_line_size - 1; - end = (pa + len + mask) & ~mask; - pa &= ~mask; + mask = zicbom_dcache_line_size - 1; + end = (va + len + mask) & ~mask; + va &= ~mask; - while (pa != end) { - /* th.dcache.cipa a0 */ - __asm volatile ("mv a0, %0; .long 0x02b5000b" :: "r"(pa) - : "a0", "memory"); - pa += thead_dcache_line_size; + while (va != end) { + __asm volatile ("cbo.flush (%0)" :: "r"(va) : "memory"); + va += zicbom_dcache_line_size; } - /* th.sync.s */ - __asm volatile (".long 0x0190000b" ::: "memory"); + + __asm volatile ("fence iorw,iorw" ::: "memory"); } +__attribute__((target("arch=+zicbom"))) void -thead_dcache_inv_range(paddr_t pa, psize_t len) +zicbom_dcache_inv_range(vaddr_t va, vsize_t len) { - paddr_t end, mask; + vaddr_t end, mask; + + mask = zicbom_dcache_line_size - 1; + end = (va + len + mask) & ~mask; + va &= ~mask; + + while (va != end) { + __asm volatile ("cbo.inval (%0)" :: "r"(va) : "memory"); + va += zicbom_dcache_line_size; + } + + __asm volatile ("fence iorw,iorw" ::: "memory"); +} + +__attribute__((target("arch=+zicbom"))) +void +zicbom_dcache_wb_range(vaddr_t va, vsize_t len) +{ + vaddr_t end, mask; + + mask = zicbom_dcache_line_size - 1; + end = (va + len + mask) & ~mask; + va &= ~mask; + + while (va != end) { + __asm volatile ("cbo.clean (%0)" :: "r"(va) : "memory"); + va += zicbom_dcache_line_size; + } + + __asm volatile ("fence iorw,iorw" ::: "memory"); +} + +__attribute__((target("arch=+xtheadcmo,+xtheadsync"))) +void +thead_dcache_wbinv_range(vaddr_t va, vsize_t len) +{ + vaddr_t end, mask; mask = thead_dcache_line_size - 1; - end = (pa + len + mask) & ~mask; - pa &= ~mask; + end = (va + len + mask) & ~mask; + va &= ~mask; - while (pa != end) { + while (va != end) { + __asm volatile ("th.dcache.civa %0" :: "r"(va) : "memory"); + va += thead_dcache_line_size; + } + + __asm volatile ("th.sync.s" ::: "memory"); +} + +__attribute__((target("arch=+xtheadcmo,+xtheadsync"))) +void +thead_dcache_inv_range(vaddr_t va, vsize_t len) +{ + vaddr_t end, mask; + + mask = thead_dcache_line_size - 1; + end = (va + len + mask) & ~mask; + va &= ~mask; + + while (va != end) { /* th.dcache.ipa a0 */ - __asm volatile ("mv a0, %0; .long 0x02a5000b" :: "r"(pa) - : "a0", "memory"); - pa += thead_dcache_line_size; + __asm volatile ("th.dcache.iva %0" :: "r"(va) : "memory"); + va += thead_dcache_line_size; } - /* th.sync.s */ - __asm volatile (".long 0x0190000b" ::: "memory"); + + __asm volatile ("th.sync.s" ::: "memory"); } +__attribute__((target("arch=+xtheadcmo,+xtheadsync"))) void -thead_dcache_wb_range(paddr_t pa, psize_t len) +thead_dcache_wb_range(vaddr_t va, vsize_t len) { - paddr_t end, mask; + vaddr_t end, mask; mask = thead_dcache_line_size - 1; - end = (pa + len + mask) & ~mask; - pa &= ~mask; + end = (va + len + mask) & ~mask; + va &= ~mask; - while (pa != end) { - /* th.dcache.cpa a0 */ - __asm volatile ("mv a0, %0; .long 0x0295000b" :: "r"(pa) - : "a0", "memory"); - pa += thead_dcache_line_size; + while (va != end) { + __asm volatile ("th.dcache.cva %0" :: "r"(va) : "memory"); + va += thead_dcache_line_size; } - /* th.sync.s */ - __asm volatile (".long 0x0190000b" ::: "memory"); + + __asm volatile ("th.sync.s" ::: "memory"); } -void (*cpu_dcache_wbinv_range)(paddr_t, psize_t) = cpu_cache_nop_range; -void (*cpu_dcache_inv_range)(paddr_t, psize_t) = cpu_cache_nop_range; -void (*cpu_dcache_wb_range)(paddr_t, psize_t) = cpu_cache_nop_range; +void (*cpu_dcache_wbinv_range)(vaddr_t, vsize_t) = cpu_cache_nop_range; +void (*cpu_dcache_inv_range)(vaddr_t, vsize_t) = cpu_cache_nop_range; +void (*cpu_dcache_wb_range)(vaddr_t, vsize_t) = cpu_cache_nop_range; #ifdef MULTIPROCESSOR diff --git a/sys/arch/riscv64/riscv64/pmap.c b/sys/arch/riscv64/riscv64/pmap.c index 4b1b23424dc..36cddf23109 100644 --- a/sys/arch/riscv64/riscv64/pmap.c +++ b/sys/arch/riscv64/riscv64/pmap.c @@ -1,4 +1,4 @@ -/* $OpenBSD: pmap.c,v 1.47 2025/07/14 12:23:28 jca Exp $ */ +/* $OpenBSD: pmap.c,v 1.48 2026/04/05 11:48:17 kettenis Exp $ */ /* * Copyright (c) 2019-2020 Brian Bamsch @@ -301,7 +301,7 @@ const pt_entry_t ap_bits_kern[8] = { [PROT_EXEC|PROT_WRITE|PROT_READ] = PTE_A|PTE_X|PTE_R|PTE_D|PTE_W, }; -/* PBMT encodings for the Svpmbt modes. */ +/* PBMT encodings for the Svpbmt modes. */ uint64_t pmap_pma; uint64_t pmap_nc; uint64_t pmap_io; @@ -693,6 +693,12 @@ _pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, int flags, int cache) struct pte_desc *pted; struct vm_page *pg; + pg = PHYS_TO_VM_PAGE(pa); + if (pg && cache == PMAP_CACHE_CI) { + _pmap_kenter_pa(va, pa, prot, flags, PMAP_CACHE_WB); + cpu_dcache_wbinv_range(va & ~PAGE_MASK, PAGE_SIZE); + } + pted = pmap_vp_lookup(pm, va, NULL); /* Do not have pted for this, get one and put it in VP */ @@ -717,10 +723,6 @@ _pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, int flags, int cache) */ pmap_pte_insert(pted); tlb_flush_page(pm, va & ~PAGE_MASK); - - pg = PHYS_TO_VM_PAGE(pa); - if (pg && cache == PMAP_CACHE_CI) - cpu_dcache_wbinv_range(pa & ~PAGE_MASK, PAGE_SIZE); } void