On Tue, Apr 15, 2025 at 03:54:31PM +0200, Hans-J?rg H?xer wrote:
> Hi,
> 
> With SEV-ES we have to respect a "minimum ASID" when running both SEV-ES
> and non-SEV-ES guests.  The minimum ASID value is configured in the BIOS.
> Non-SEV-ES guests will have to use ASIDs starting at that value; SEV-ES
> enabled VMs use ASIDs starting at 1.  Setting the minimum ASID in the
> BIOS to 1, disables SEV-ES.
> 
> At runtime the minimum ASID is determined by CPUID.
> 
> This change determins the minimum ASID value in identifycpu() and uses it
> when allocating ASIDs for SVM based VMs.  For Intel VMX there's no change.
> 
> Comments?

I figured out what the problem with my SEV setup is.  (actually
hshoexer figured it out)

To test SEV-ES I have changed the BIOS setting.  Now the host kernel
no longer handles SEV without ES correctly.  Diff below fixes it.

> ---------------------------------------------------------------------------
> commit 76cad15af8a10628e446aa31f08cdab99a28a0bc
> Author: Hans-Joerg Hoexer <hshoexer@genua.de>
> Date:   Wed Jul 10 13:25:04 2024 +0200
> 
>     vmm: minimum ASID for non-SEVES guests
>     
>     Introduce vmm_alloc_asid() to be used by SVM base VMs.  No change
>     vor VMX based VMs.
> 
> diff --git a/sys/arch/amd64/amd64/identcpu.c b/sys/arch/amd64/amd64/identcpu.c
> index 18ecbc8f4c8..1a9a99e9bbb 100644
> --- a/sys/arch/amd64/amd64/identcpu.c
> +++ b/sys/arch/amd64/amd64/identcpu.c
> @@ -67,6 +67,7 @@ int cpuspeed;
>  
>  int amd64_has_xcrypt;
>  int amd64_pos_cbit;	/* C bit position for SEV */
> +int amd64_min_noes_asid;
>  int has_rdrand;
>  int has_rdseed;
>  
> @@ -710,6 +711,7 @@ identifycpu(struct cpu_info *ci)
>  		    'd', CPUID_MEMBER(ci_feature_amdsev_edx),
>  		    CPUID_AMDSEV_EDX_BITS);
>  		amd64_pos_cbit = (ci->ci_feature_amdsev_ebx & 0x3f);
> +		amd64_min_noes_asid = (ci->ci_feature_amdsev_edx);
>  	}
>  
>  	printf("\n");
> diff --git a/sys/arch/amd64/amd64/vmm_machdep.c b/sys/arch/amd64/amd64/vmm_machdep.c
> index 5a8c9a14612..f75a5feb815 100644
> --- a/sys/arch/amd64/amd64/vmm_machdep.c
> +++ b/sys/arch/amd64/amd64/vmm_machdep.c
> @@ -125,6 +125,7 @@ int vmx_fault_page(struct vcpu *, paddr_t);
>  int vmx_handle_np_fault(struct vcpu *);
>  int svm_handle_np_fault(struct vcpu *);
>  int vmm_alloc_vpid(uint16_t *);
> +int vmm_alloc_asid(uint16_t *, struct vcpu *);
>  void vmm_free_vpid(uint16_t);
>  const char *vcpu_state_decode(u_int);
>  const char *vmx_exit_reason_decode(uint32_t);
> @@ -232,6 +233,9 @@ extern struct vmm_softc *vmm_softc;
>  extern vaddr_t idt_vaddr;
>  extern struct gate_descriptor *idt;
>  
> +/* Minimum ASID value for an SEV enabled, SEV-ES disabled guest. */
> +extern int amd64_min_noes_asid;
> +
>  /* Constants used in "CR access exit" */
>  #define CR_WRITE	0
>  #define CR_READ		1
> @@ -2766,7 +2770,7 @@ vcpu_init_svm(struct vcpu *vcpu, struct vm_create_params *vcp)
>  	int ret = 0;
>  
>  	/* Allocate an ASID early to avoid km_alloc if we're out of ASIDs. */
> -	if (vmm_alloc_vpid(&vcpu->vc_vpid))
> +	if (vmm_alloc_asid(&vcpu->vc_vpid, vcpu))
>  		return (ENOMEM);
>  
>  	/* Allocate VMCB VA */
> @@ -6354,27 +6358,32 @@ vcpu_run_svm(struct vcpu *vcpu, struct vm_run_params *vrp)
>  }
>  
>  /*
> - * vmm_alloc_vpid
> + * _vmm_alloc_vpid
>   *
>   * Sets the memory location pointed to by "vpid" to the next available VPID
> - * or ASID.
> + * or ASID. For SEV-ES consider minimum ASID value for non-ES enabled guests.
>   *
>   * Parameters:
>   *  vpid: Pointer to location to receive the next VPID/ASID
> + *  vcpu: Pointer to VCPU data structure
>   *
>   * Return Values:
>   *  0: The operation completed successfully
>   *  ENOMEM: No VPIDs/ASIDs were available. Content of 'vpid' is unchanged.
>   */
>  int
> -vmm_alloc_vpid(uint16_t *vpid)
> +_vmm_alloc_vpid(uint16_t *vpid, struct vcpu *vcpu)
>  {
> -	uint16_t i;
> +	uint16_t i, minasid;
>  	uint8_t idx, bit;
>  	struct vmm_softc *sc = vmm_softc;
>  
>  	rw_enter_write(&vmm_softc->vpid_lock);
> -	for (i = 1; i <= sc->max_vpid; i++) {
> +	if (vcpu == NULL || vcpu->vc_seves)
> +		minasid = 1;
> +	else
> +		minasid = amd64_min_noes_asid;
> +	for (i = minasid; i <= sc->max_vpid; i++) {
>  		idx = i / 8;
>  		bit = i - (idx * 8);
>  
> @@ -6396,6 +6405,18 @@ vmm_alloc_vpid(uint16_t *vpid)
>  	return ENOMEM;
>  }
>  
> +int
> +vmm_alloc_vpid(uint16_t *vpid)
> +{
> +	return _vmm_alloc_vpid(vpid, NULL);
> +}
> +
> +int
> +vmm_alloc_asid(uint16_t *asid, struct vcpu *vcpu)
> +{
> +	return _vmm_alloc_vpid(asid, vcpu);
> +}
> +
>  /*
>   * vmm_free_vpid
>   *