From: Dave Voutila Subject: Re: SEV-ES: vmm(4): Allocate VMSA To: tech@openbsd.org Cc: Hans-Joerg_Hoexer@genua.de Date: Thu, 17 Apr 2025 09:42:35 -0400 Hans-Jörg Höxer writes: > Hi, > > as a first step towards SEV-ES let vmm(4) allocate the VM Save Area. > It will be used by the CPU to store and load encrypted vCPU state. > > Non SEV-ES VMs are not affected by this change. > > A seconde diff will provide the psp(4) related pieces to prepare the VMSA. > > Comments? This looks good to me. We can always iterate on it. ok dv@ > > Take care, > HJ. > --------------------------------------------------------------------------- > commit ad3a54885df560a678b3dfb5112b19ed603b6bbc > Author: Hans-Joerg Hoexer > Date: Mon Apr 14 16:41:14 2025 +0200 > > vmm: provide VMSA > > For SEV-ES the hypervisor has to provide a VM Save Area (VMSA). It > will be used by the CPU to load and store encrypted guest CPU state. > > Also supply a helpfer function to retrieve the physical address of > the VMSA. > > diff --git a/sys/arch/amd64/amd64/vmm_machdep.c b/sys/arch/amd64/amd64/vmm_machdep.c > index e3205f48eed..5a8c9a14612 100644 > --- a/sys/arch/amd64/amd64/vmm_machdep.c > +++ b/sys/arch/amd64/amd64/vmm_machdep.c > @@ -138,6 +138,7 @@ void vmx_setmsrbw(struct vcpu *, uint32_t); > void vmx_setmsrbrw(struct vcpu *, uint32_t); > void svm_set_clean(struct vcpu *, uint32_t); > void svm_set_dirty(struct vcpu *, uint32_t); > +int svm_get_vmsa_pa(uint32_t, uint32_t, uint64_t *); > > int vmm_gpa_is_valid(struct vcpu *vcpu, paddr_t gpa, size_t obj_size); > void vmm_init_pvclock(struct vcpu *, paddr_t); > @@ -2849,8 +2850,31 @@ vcpu_init_svm(struct vcpu *vcpu, struct vm_create_params *vcp) > (uint64_t)vcpu->vc_svm_ioio_va, > (uint64_t)vcpu->vc_svm_ioio_pa); > > - /* Shall we enable SEV? */ > + /* Shall we enable SEV/SEV-ES? */ > vcpu->vc_sev = vcp->vcp_sev; > + vcpu->vc_seves = vcp->vcp_seves; > + > + if (vcpu->vc_seves) { > + /* Allocate VM save area VA */ > + vcpu->vc_svm_vmsa_va = (vaddr_t)km_alloc(PAGE_SIZE, &kv_page, > + &kp_zero, &kd_waitok); > + > + if (!vcpu->vc_svm_vmsa_va) { > + ret = ENOMEM; > + goto exit; > + } > + > + /* Compute VM save area PA */ > + if (!pmap_extract(pmap_kernel(), vcpu->vc_svm_vmsa_va, > + &vcpu->vc_svm_vmsa_pa)) { > + ret = ENOMEM; > + goto exit; > + } > + > + DPRINTF("%s: VMSA va @ 0x%llx, pa @ 0x%llx\n", __func__, > + (uint64_t)vcpu->vc_svm_vmsa_va, > + (uint64_t)vcpu->vc_svm_vmsa_pa); > + } > > /* Inform vmd(8) about ASID and C bit position. */ > vcp->vcp_poscbit = amd64_pos_cbit; > @@ -2957,6 +2981,11 @@ vcpu_deinit_svm(struct vcpu *vcpu) > &kp_zero); > vcpu->vc_svm_hsa_va = 0; > } > + if (vcpu->vc_svm_vmsa_va) { > + km_free((void *)vcpu->vc_svm_vmsa_va, PAGE_SIZE, &kv_page, > + &kp_zero); > + vcpu->vc_svm_vmsa_va = 0; > + } > if (vcpu->vc_svm_ioio_va) { > km_free((void *)vcpu->vc_svm_ioio_va, 3 * PAGE_SIZE, &kv_any, > &vmm_kp_contig); > @@ -6779,6 +6808,36 @@ vcpu_state_decode(u_int state) > } > } > > +/* > + * svm_get_vmsa_pa > + * > + * Return physical address of VMSA for specified VCPU. > + */ > +int > +svm_get_vmsa_pa(uint32_t vmid, uint32_t vcpuid, uint64_t *vmsapa) > +{ > + struct vm *vm; > + struct vcpu *vcpu; > + int error, ret = 0; > + > + error = vm_find(vmid, &vm); > + if (error) > + return (error); > + > + vcpu = vm_find_vcpu(vm, vcpuid); > + if (vcpu == NULL || !vcpu->vc_seves) { > + ret = ENOENT; > + goto out; > + } > + > + if (vmsapa) > + *vmsapa = vcpu->vc_svm_vmsa_pa; > + > +out: > + refcnt_rele_wake(&vm->vm_refcnt); > + return (ret); > +} > + > #ifdef VMM_DEBUG > /* > * dump_vcpu > diff --git a/sys/arch/amd64/include/vmmvar.h b/sys/arch/amd64/include/vmmvar.h > index cd04fe4f07b..b1a8451792f 100644 > --- a/sys/arch/amd64/include/vmmvar.h > +++ b/sys/arch/amd64/include/vmmvar.h > @@ -668,6 +668,8 @@ struct vmcb { > uint64_t v_pad5; /* 0E8h-0EFh */ > uint64_t v_avic_logical_table; /* 0F0h */ > uint64_t v_avic_phys; /* 0F8h */ > + uint64_t v_pad12; /* 100h */ > + uint64_t v_vmsa_pa; /* 108h */ > > }; > uint8_t vmcb_control[0x400]; > @@ -723,6 +725,128 @@ struct vmcb { > }; > }; > > +struct vmsa { > + struct vmcb_segment v_es; /* 000h */ > + struct vmcb_segment v_cs; /* 010h */ > + struct vmcb_segment v_ss; /* 020h */ > + struct vmcb_segment v_ds; /* 030h */ > + struct vmcb_segment v_fs; /* 040h */ > + struct vmcb_segment v_gs; /* 050h */ > + struct vmcb_segment v_gdtr; /* 060h */ > + struct vmcb_segment v_ldtr; /* 070h */ > + struct vmcb_segment v_idtr; /* 080h */ > + struct vmcb_segment v_tr; /* 090h */ > + uint64_t v_pl0_ssp; /* 0A0h */ > + uint64_t v_pl1_ssp; /* 0A8h */ > + uint64_t v_pl2_ssp; /* 0B0h */ > + uint64_t v_pl3_ssp; /* 0B8h */ > + uint64_t v_u_cet; /* 0C0h */ > + uint8_t v_pad1[0x2]; /* 0C8h-0C9h */ > + uint8_t v_vmpl; /* 0CAh */ > + uint8_t v_cpl; /* 0CBh */ > + uint8_t v_pad2[0x4]; /* 0CCh-0CFh */ > + uint64_t v_efer; /* 0D0h */ > + uint8_t v_pad3[0x68]; /* 0D8h-13Fh */ > + uint64_t v_xss; /* 140h */ > + uint64_t v_cr4; /* 148h */ > + uint64_t v_cr3; /* 150h */ > + uint64_t v_cr0; /* 158h */ > + uint64_t v_dr7; /* 160h */ > + uint64_t v_dr6; /* 168h */ > + uint64_t v_rflags; /* 170h */ > + uint64_t v_rip; /* 178h */ > + uint64_t v_dr0; /* 180h */ > + uint64_t v_dr1; /* 188h */ > + uint64_t v_dr2; /* 190h */ > + uint64_t v_dr3; /* 198h */ > + uint64_t v_dr0_addr_msk; /* 1A0h */ > + uint64_t v_dr1_addr_msk; /* 1A8h */ > + uint64_t v_dr2_addr_msk; /* 1B0h */ > + uint64_t v_dr3_addr_msk; /* 1B8h */ > + uint8_t v_pad4[0x18]; /* 1C0h-1D7h */ > + uint64_t v_rsp; /* 1D8h */ > + uint64_t v_s_cet; /* 1E0h */ > + uint64_t v_ssp; /* 1E8h */ > + uint64_t v_isst_addr; /* 1F0h */ > + uint64_t v_rax; /* 1F8h */ > + uint64_t v_star; /* 200h */ > + uint64_t v_lstar; /* 208h */ > + uint64_t v_cstar; /* 210h */ > + uint64_t v_sfmask; /* 218h */ > + uint64_t v_kgsbase; /* 220h */ > + uint64_t v_sysenter_cs; /* 228h */ > + uint64_t v_sysenter_esp; /* 230h */ > + uint64_t v_sysenter_eip; /* 238h */ > + uint64_t v_cr2; /* 240h */ > + uint8_t v_pad5[0x20]; /* 248h-267h */ > + uint64_t v_g_pat; /* 268h */ > + uint64_t v_dbgctl; /* 270h */ > + uint64_t v_br_from; /* 278h */ > + uint64_t v_br_to; /* 280h */ > + uint64_t v_lastexcpfrom; /* 288h */ > + uint64_t v_lastexcpto; /* 290h */ > + uint8_t v_pad6[0x48]; /* 298h-2DFh */ > + uint8_t v_pad7[0x8]; /* 2E0h-2E7h */ > + uint32_t v_pkru; /* 2E8h */ > + uint32_t v_tsc_aux; /* 2ECh */ > + uint64_t v_gst_tsc_scale;/* 2F0h */ > + uint64_t v_gst_tsc_off; /* 2F8h */ > + uint64_t v_reg_prot_nce; /* 300h */ > + uint64_t v_rcx; /* 308h */ > + uint64_t v_rdx; /* 310h */ > + uint64_t v_rbx; /* 318h */ > + uint64_t v_pad8; /* 320h */ > + uint64_t v_rbp; /* 328h */ > + uint64_t v_rsi; /* 330h */ > + uint64_t v_rdi; /* 338h */ > + uint64_t v_r8; /* 340h */ > + uint64_t v_r9; /* 348h */ > + uint64_t v_r10; /* 350h */ > + uint64_t v_r11; /* 358h */ > + uint64_t v_r12; /* 360h */ > + uint64_t v_r13; /* 368h */ > + uint64_t v_r14; /* 370h */ > + uint64_t v_r15; /* 378h */ > + uint8_t v_pad9[0x10]; /* 380h-38Fh */ > + uint64_t v_gst_exitinfo1;/* 390h */ > + uint64_t v_gst_exitinfo2;/* 398h */ > + uint64_t v_gst_exitiinfo;/* 3A0h */ > + uint64_t v_gst_nrip; /* 3A8h */ > + uint64_t v_sev_features; /* 3B0h */ > + uint64_t v_intr_ctrl; /* 3B8h */ > + uint64_t v_gst_exitcode; /* 3C0h */ > + uint64_t v_virtual_tom; /* 3C8h */ > + uint64_t v_tlb_id; /* 3D0h */ > + uint64_t v_pcup_id; /* 3D8h */ > + uint64_t v_eventinj; /* 3E0h */ > + uint64_t v_xcr0; /* 3E8h */ > + uint8_t v_pad10[0x10]; /* 3F0h-3FFh */ > + uint64_t v_x87_dp; /* 400h */ > + uint32_t v_mxcsr; /* 408h */ > + uint16_t v_x87_ftw; /* 40Ch */ > + uint16_t v_x87_fsw; /* 40Eh */ > + uint16_t v_x87_fcw; /* 410h */ > + uint16_t v_x87_fop; /* 412h */ > + uint16_t v_x87_ds; /* 414h */ > + uint16_t v_x87_cs; /* 416h */ > + uint64_t v_x87_rip; /* 418h */ > + uint8_t v_fp_x87[0x50]; /* 420h-46Fh */ > + uint8_t v_fp_xmm[0x100];/* 470h-56Fh */ > + uint8_t v_fp_ymm[0x100];/* 570h-66fh */ > + uint8_t v_lbr_st[0x100];/* 670h-76Fh */ > + uint64_t v_lbr_select; /* 770h */ > + uint64_t v_ibs_fetch_ctl;/* 778h */ > + uint64_t v_ibs_fetch_la; /* 780h */ > + uint64_t v_ibs_op_ctl; /* 788h */ > + uint64_t v_ibs_op_rip; /* 790h */ > + uint64_t v_ibs_op_data; /* 798h */ > + uint64_t v_ibs_op_data2; /* 7A0h */ > + uint64_t v_ibs_op_data3; /* 7A8h */ > + uint64_t v_ibs_dc_la; /* 7B0h */ > + uint64_t v_ibstgt_rip; /* 7B8h */ > + uint64_t v_ic_ibs_xtd_ct;/* 7C0h */ > +}; > + > struct vmcs { > uint32_t vmcs_revision; > }; > @@ -876,9 +1000,12 @@ struct vcpu { > /* SVM only (all requiring [v]) */ > vaddr_t vc_svm_hsa_va; > paddr_t vc_svm_hsa_pa; > + vaddr_t vc_svm_vmsa_va; > + paddr_t vc_svm_vmsa_pa; > vaddr_t vc_svm_ioio_va; > paddr_t vc_svm_ioio_pa; > int vc_sev; /* [I] */ > + int vc_seves; /* [I] */ > }; > > SLIST_HEAD(vcpu_head, vcpu); > @@ -911,6 +1038,7 @@ int vcpu_init(struct vcpu *, struct vm_create_params *); > void vcpu_deinit(struct vcpu *); > int vm_rwregs(struct vm_rwregs_params *, int); > int vcpu_reset_regs(struct vcpu *, struct vcpu_reg_state *); > +int svm_get_vmsa_pa(uint32_t, uint32_t, uint64_t *); > > #endif /* _KERNEL */ > > diff --git a/sys/dev/vmm/vmm.h b/sys/dev/vmm/vmm.h > index ca5a152f550..886ff51760a 100644 > --- a/sys/dev/vmm/vmm.h > +++ b/sys/dev/vmm/vmm.h > @@ -50,6 +50,7 @@ struct vm_create_params { > struct vm_mem_range vcp_memranges[VMM_MAX_MEM_RANGES]; > char vcp_name[VMM_MAX_NAME_LEN]; > int vcp_sev; > + int vcp_seves; > > /* Output parameter from VMM_IOC_CREATE */ > uint32_t vcp_id;