SEV-ES: GHCB protocol for IO and MMIO

Implement the parts of the GHCB protocol for IO and MMIO;  to be
used to paravirtualize bus_space(9) for SEV-ES:

 - extend the ghcb_sync_out() and ghcb_sync_in() to pass a data
   buffer to or from the hypervisor

 - implement for both IO and MMIO generic functions _ghcb_io_rw()
   and _ghcb_mem_rw() that build, send and receive the respective
   GHCB messages for various data sizes

 - add wrapper functions that call _ghcb_io_rw() and _ghcb_mem_rw()
   with the actual data sizes; these will be used for bus_space(9)

From Sebastian Sturm with tweaks by hshoexer@ and myself

ok mlarkin@

sys/arch/amd64/amd64/ghcb.c

@@ -1,4 +1,4 @@
-/*	$OpenBSD: ghcb.c,v 1.5 2025/08/27 06:11:12 sf Exp $	*/
+/*	$OpenBSD: ghcb.c,v 1.6 2025/09/17 18:37:44 sf Exp $	*/
 
 /*
  * Copyright (c) 2024, 2025 Hans-Joerg Hoexer <hshoexer@genua.de>
@@ -16,11 +16,15 @@
  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  */
 
+#include <sys/param.h>
 #include <sys/types.h>
 #include <sys/systm.h>
 
+#include <uvm/uvm_extern.h>
+
 #include <machine/frame.h>
 #include <machine/ghcb.h>
+#include <machine/vmmvar.h>
 
 /* Masks for adjusting GPR sizes. */
 const uint64_t ghcb_sz_masks[] = {
@@ -167,6 +171,7 @@ ghcb_sync_val(int type, int size, struct ghcb_sync *gs)
 	case GHCB_SW_EXITCODE:
 	case GHCB_SW_EXITINFO1:
 	case GHCB_SW_EXITINFO2:
+	case GHCB_SW_SCRATCH:
 		break;
 
 	default:
@@ -188,6 +193,8 @@ void
 ghcb_sync_out(struct trapframe *frame, const struct ghcb_extra_regs *regs,
     struct ghcb_sa *ghcb, struct ghcb_sync *gsout)
 {
+	size_t data_sz;
+
 	ghcb_clear(ghcb);
 
 	memcpy(ghcb->valid_bitmap, gsout->valid_bitmap,
@@ -208,6 +215,14 @@ ghcb_sync_out(struct trapframe *frame, const struct ghcb_extra_regs *regs,
 		ghcb->v_sw_exitinfo1 = regs->exitinfo1;
 	if (ghcb_valbm_isset(gsout->valid_bitmap, GHCB_SW_EXITINFO2))
 		ghcb->v_sw_exitinfo2 = regs->exitinfo2;
+	if (ghcb_valbm_isset(gsout->valid_bitmap, GHCB_SW_SCRATCH))
+		ghcb->v_sw_scratch = regs->scratch;
+
+	if (regs && regs->data) {
+		data_sz = regs->data_sz;
+		KASSERT(data_sz <= sizeof(ghcb->v_sharedbuf));
+		memcpy(ghcb->v_sharedbuf, regs->data, data_sz);
+	}
 }
 
 /*
@@ -217,9 +232,11 @@ ghcb_sync_out(struct trapframe *frame, const struct ghcb_extra_regs *regs,
  * Used by guest only.
  */
 void
-ghcb_sync_in(struct trapframe *frame, struct ghcb_sa *ghcb,
-    struct ghcb_sync *gsin)
+ghcb_sync_in(struct trapframe *frame, struct ghcb_extra_regs *regs,
+    struct ghcb_sa *ghcb, struct ghcb_sync *gsin)
 {
+	size_t data_sz;
+
 	if (ghcb_valbm_isset(gsin->valid_bitmap, GHCB_RAX)) {
 		frame->tf_rax &= ~ghcb_sz_clear_masks[gsin->sz_a];
 		frame->tf_rax |= (ghcb->v_rax & ghcb_sz_masks[gsin->sz_a]);
@@ -237,5 +254,167 @@ ghcb_sync_in(struct trapframe *frame, struct ghcb_sa *ghcb,
 		frame->tf_rdx |= (ghcb->v_rdx & ghcb_sz_masks[gsin->sz_d]);
 	}
 
+	if (regs && regs->data) {
+		data_sz = regs->data_sz;
+		KASSERT(data_sz <= sizeof(ghcb->v_sharedbuf));
+		memcpy(regs->data, ghcb->v_sharedbuf, data_sz);
+	}
+
 	ghcb_clear(ghcb);
 }
+
+void
+_ghcb_mem_rw(vaddr_t addr, int valsz, void *val, bool read)
+{
+	size_t			 size;
+	paddr_t			 paddr;
+	struct ghcb_sync	 syncout, syncin;
+	struct ghcb_sa		*ghcb;
+	unsigned long		 s;
+	struct ghcb_extra_regs	 ghcb_regs;
+
+	KASSERT(val != NULL);
+
+	switch (valsz) {
+	case GHCB_SZ8:
+		size = sizeof(uint8_t);
+		break;
+	case GHCB_SZ16:
+		size = sizeof(uint16_t);
+		break;
+	case GHCB_SZ32:
+		size = sizeof(uint32_t);
+		break;
+	case GHCB_SZ64:
+		size = sizeof(uint64_t);
+		break;
+	default:
+		panic("%s: invalid size", __func__);
+	}
+
+	if (!pmap_extract(pmap_kernel(), addr, &paddr))
+		panic("%s: pmap_extract %#lx failed", __func__, addr);
+
+	memset(&syncout, 0, sizeof(syncout));
+	memset(&syncin, 0, sizeof(syncin));
+	memset(&ghcb_regs, 0, sizeof(ghcb_regs));
+
+	if (read) {
+		ghcb_regs.exitcode = SEV_VMGEXIT_MMIO_READ;
+		ghcb_regs.exitinfo1 = paddr;
+		ghcb_regs.exitinfo2 = size;
+		ghcb_regs.scratch = ghcb_paddr + offsetof(struct ghcb_sa,
+		    v_sharedbuf);
+	} else {
+		ghcb_regs.exitcode = SEV_VMGEXIT_MMIO_WRITE;
+		ghcb_regs.exitinfo1 = paddr;
+		ghcb_regs.exitinfo2 = size;
+		ghcb_regs.scratch = ghcb_paddr + offsetof(struct ghcb_sa,
+		    v_sharedbuf);
+		ghcb_regs.data = val;
+		ghcb_regs.data_sz = size;
+	}
+
+	ghcb_sync_val(GHCB_SW_EXITCODE, GHCB_SZ64, &syncout);
+	ghcb_sync_val(GHCB_SW_EXITINFO1, GHCB_SZ64, &syncout);
+	ghcb_sync_val(GHCB_SW_EXITINFO2, GHCB_SZ64, &syncout);
+	ghcb_sync_val(GHCB_SW_SCRATCH, GHCB_SZ64, &syncout);
+
+	s = intr_disable();
+
+	ghcb = (struct ghcb_sa *)ghcb_vaddr;
+	ghcb_sync_out(NULL, &ghcb_regs, ghcb, &syncout);
+
+	wrmsr(MSR_SEV_GHCB, ghcb_paddr);
+
+	vmgexit();
+
+	if (ghcb_verify_bm(ghcb->valid_bitmap, syncin.valid_bitmap)) {
+		ghcb_clear(ghcb);
+		panic("invalid hypervisor response");
+	}
+
+	memset(&ghcb_regs, 0, sizeof(ghcb_regs));
+
+	if (read) {
+		ghcb_regs.data = val;
+		ghcb_regs.data_sz = size;
+
+		ghcb_sync_in(NULL, &ghcb_regs, ghcb, &syncin);
+	}
+
+	intr_restore(s);
+}
+
+#define SVM_IOIO_INTERCEPT_READ		1
+#define SVM_IOIO_INTERCEPT_STRING	(1 << 2)
+#define SVM_IOIO_INTERCEPT_REP		(1 << 3)
+#define SVM_IOIO_INTERCEPT_SZ8		(1 << 4)
+#define SVM_IOIO_INTERCEPT_SZ16		(1 << 5)
+#define SVM_IOIO_INTERCEPT_SZ32		(1 << 6)
+
+void
+_ghcb_io_rw(uint16_t port, int valsz, uint32_t *val, bool read)
+{
+	struct ghcb_sync syncout, syncin;
+	struct ghcb_extra_regs ghcb_regs;
+	struct ghcb_sa *ghcb;
+	struct trapframe frame;
+	unsigned long s;
+
+	KASSERT(val != NULL);
+
+	memset(&syncout, 0, sizeof(syncout));
+	memset(&syncin, 0, sizeof(syncin));
+	memset(&ghcb_regs, 0, sizeof(ghcb_regs));
+
+	ghcb_regs.exitcode = SVM_VMEXIT_IOIO;
+	ghcb_regs.exitinfo1 = ((uint64_t)port) << 16;
+
+	switch (valsz) {
+	case GHCB_SZ8:
+		ghcb_regs.exitinfo1 |= SVM_IOIO_INTERCEPT_SZ8;
+		break;
+	case GHCB_SZ16:
+		ghcb_regs.exitinfo1 |= SVM_IOIO_INTERCEPT_SZ16;
+		break;
+	case GHCB_SZ32:
+		ghcb_regs.exitinfo1 |= SVM_IOIO_INTERCEPT_SZ32;
+		break;
+	default:
+		panic("%s: invalid size", __func__);
+	}
+
+	if (!read) {
+		frame.tf_rax = *val;
+		ghcb_sync_val(GHCB_RAX, valsz, &syncout);
+	} else {
+		ghcb_regs.exitinfo1 |= SVM_IOIO_INTERCEPT_READ;
+		ghcb_sync_val(GHCB_RAX, valsz, &syncin);
+	}
+
+	ghcb_sync_val(GHCB_SW_EXITCODE, GHCB_SZ64, &syncout);
+	ghcb_sync_val(GHCB_SW_EXITINFO1, GHCB_SZ64, &syncout);
+	ghcb_sync_val(GHCB_SW_EXITINFO2, GHCB_SZ64, &syncout);
+
+	s = intr_disable();
+
+	ghcb = (struct ghcb_sa *)ghcb_vaddr;
+	ghcb_sync_out(&frame, &ghcb_regs, ghcb, &syncout);
+
+	wrmsr(MSR_SEV_GHCB, ghcb_paddr);
+
+	vmgexit();
+
+	if (ghcb_verify_bm(ghcb->valid_bitmap, syncin.valid_bitmap)) {
+		ghcb_clear(ghcb);
+		panic("invalid hypervisor response");
+	}
+
+	ghcb_sync_in(&frame, NULL, ghcb, &syncin);
+
+	intr_restore(s);
+
+	if (read)
+		*val = frame.tf_rax;
+}

sys/arch/amd64/amd64/trap.c

@@ -1,4 +1,4 @@
-/*	$OpenBSD: trap.c,v 1.113 2025/08/27 06:11:12 sf Exp $	*/
+/*	$OpenBSD: trap.c,v 1.114 2025/09/17 18:37:44 sf Exp $	*/
 /*	$NetBSD: trap.c,v 1.2 2003/05/04 23:51:56 fvdl Exp $	*/
 
 /*-
@@ -439,6 +439,8 @@ vctrap(struct trapframe *frame, int user)
 	ghcb = (struct ghcb_sa *)ghcb_vaddr;
 	ghcb_sync_out(frame, &ghcb_regs, ghcb, &syncout);
 
+	wrmsr(MSR_SEV_GHCB, ghcb_paddr);
+
 	/* Call hypervisor. */
 	vmgexit();
 
@@ -449,7 +451,7 @@ vctrap(struct trapframe *frame, int user)
 	}
 
 	/* Sync in from GHCB */
-	ghcb_sync_in(frame, ghcb, &syncin);
+	ghcb_sync_in(frame, NULL, ghcb, &syncin);
 
 	return 1;
 }

sys/arch/amd64/include/ghcb.h

@@ -1,4 +1,4 @@
-/*	$OpenBSD: ghcb.h,v 1.5 2025/08/27 06:11:12 sf Exp $	*/
+/*	$OpenBSD: ghcb.h,v 1.6 2025/09/17 18:37:44 sf Exp $	*/
 
 /*
  * Copyright (c) 2024, 2025 Hans-Joerg Hoexer <hshoexer@genua.de>
@@ -21,6 +21,8 @@
 
 #ifndef _LOCORE
 
+#include <sys/systm.h>
+
 #include <machine/frame.h>
 
 #define GHCB_OFFSET(m)			((m) / 8)
@@ -113,9 +115,12 @@ extern vaddr_t ghcb_vaddr;
 extern paddr_t ghcb_paddr;
 
 struct ghcb_extra_regs {
-	uint64_t	exitcode;
-	uint64_t	exitinfo1;
-	uint64_t	exitinfo2;
+	uint64_t	 exitcode;
+	uint64_t	 exitinfo1;
+	uint64_t	 exitinfo2;
+	uint64_t	 scratch;
+	void		*data;
+	size_t		 data_sz;
 };
 
 void	ghcb_clear(struct ghcb_sa *);
@@ -127,7 +132,119 @@ int	ghcb_valid(struct ghcb_sa *);
 void	ghcb_sync_val(int, int, struct ghcb_sync *);
 void	ghcb_sync_out(struct trapframe *, const struct ghcb_extra_regs *,
 	    struct ghcb_sa *, struct ghcb_sync *);
-void	ghcb_sync_in(struct trapframe *, struct ghcb_sa *, struct ghcb_sync *);
+void	ghcb_sync_in(struct trapframe *, struct ghcb_extra_regs *,
+	    struct ghcb_sa *, struct ghcb_sync *);
+void	_ghcb_mem_rw(vaddr_t, int, void *, bool);
+void	_ghcb_io_rw(uint16_t, int, uint32_t *, bool);
+
+static inline uint8_t
+ghcb_mem_read_1(vaddr_t addr)
+{
+	uint8_t val;
+
+	_ghcb_mem_rw(addr, GHCB_SZ8, &val, true);
+	return val;
+}
+
+static inline uint16_t
+ghcb_mem_read_2(vaddr_t addr)
+{
+	uint16_t val;
+
+	_ghcb_mem_rw(addr, GHCB_SZ16, &val, true);
+	return val;
+}
+
+static inline uint32_t
+ghcb_mem_read_4(vaddr_t addr)
+{
+	uint32_t val;
+
+	_ghcb_mem_rw(addr, GHCB_SZ32, &val, true);
+	return val;
+}
+
+static inline uint64_t
+ghcb_mem_read_8(vaddr_t addr)
+{
+	uint64_t val;
+
+	_ghcb_mem_rw(addr, GHCB_SZ64, &val, true);
+	return val;
+}
+
+static inline void
+ghcb_mem_write_1(vaddr_t addr, uint8_t v)
+{
+	_ghcb_mem_rw(addr, GHCB_SZ8, &v, false);
+}
+
+static inline void
+ghcb_mem_write_2(vaddr_t addr, uint16_t v)
+{
+	_ghcb_mem_rw(addr, GHCB_SZ16, &v, false);
+}
+
+static inline void
+ghcb_mem_write_4(vaddr_t addr, uint32_t v)
+{
+	_ghcb_mem_rw(addr, GHCB_SZ32, &v, false);
+}
+
+static inline void
+ghcb_mem_write_8(vaddr_t addr, uint64_t v)
+{
+	_ghcb_mem_rw(addr, GHCB_SZ64, &v, false);
+}
+
+static inline uint8_t
+ghcb_io_read_1(uint16_t port)
+{
+	uint32_t val;
+
+	_ghcb_io_rw(port, GHCB_SZ8, &val, true);
+	return val;
+}
+
+static inline uint16_t
+ghcb_io_read_2(uint16_t port)
+{
+	uint32_t val;
+
+	_ghcb_io_rw(port, GHCB_SZ16, &val, true);
+	return val;
+}
+
+static inline uint32_t
+ghcb_io_read_4(uint16_t port)
+{
+	uint32_t val;
+
+	_ghcb_io_rw(port, GHCB_SZ32, &val, true);
+	return val;
+}
+
+static inline void
+ghcb_io_write_1(uint16_t port, uint8_t v)
+{
+	uint32_t val = v;
+
+	_ghcb_io_rw(port, GHCB_SZ8, &val, false);
+}
+
+static inline void
+ghcb_io_write_2(uint16_t port, uint16_t v)
+{
+	uint32_t val = v;
+
+	_ghcb_io_rw(port, GHCB_SZ16, &val, false);
+}
+
+static inline void
+ghcb_io_write_4(uint16_t port, uint32_t v)
+{
+	_ghcb_io_rw(port, GHCB_SZ32, &v, false);
+}
 
 #endif /* !_LOCORE */
 

sys/arch/amd64/include/vmmvar.h

@@ -1,4 +1,4 @@
-/*	$OpenBSD: vmmvar.h,v 1.116 2025/09/14 15:52:28 mlarkin Exp $	*/
+/*	$OpenBSD: vmmvar.h,v 1.117 2025/09/17 18:37:44 sf Exp $	*/
 /*
  * Copyright (c) 2014 Mike Larkin <mlarkin@openbsd.org>
  *
@@ -266,6 +266,12 @@
 #define SVM_VMEXIT_VMGEXIT			0x403
 #define SVM_VMEXIT_INVALID			-1
 
+/*
+ *  Additional VMEXIT codes used in SEV-ES/SNP in the GHCB
+ */
+#define SEV_VMGEXIT_MMIO_READ			0x80000001
+#define SEV_VMGEXIT_MMIO_WRITE			0x80000002
+
 #ifndef _LOCORE
 
 /*