From: Jan Klemkow <jan@openbsd.org>
Subject: Re: ix: preparing vf support
To: YASUOKA Masahiko <yasuoka@openbsd.org>
Cc: tech@openbsd.org, naito.yuichiro@gmail.com, jmatthew@openbsd.org
Date: Fri, 25 Oct 2024 10:50:12 +0200

On Thu, Oct 24, 2024 at 03:06:54PM GMT, YASUOKA Masahiko wrote:
> On Mon, 02 Sep 2024 13:27:04 +0900 (JST)
> YASUOKA Masahiko <yasuoka@openbsd.org> wrote:
> > I'm commiting ixv(4) from NAITO Yuichiro.  Almost of the changes can
> > be done separately from ix(4), but the diff bellow is the part which
> > actually affect ix(4).
> 
> This is the second step.  This diff will not affect existing ix(4)
> because it only changes "mailbox" behavior.  "mailbox" is used only
> for primary function or virtual function and our ix(4) doesn't support
> either yet.
> 
> I got ok from jmatthew already.  Any additional ok is welcome.

ok jan@

> Index: sys/dev/pci/ixgbe.c
> ===================================================================
> RCS file: /disk/cvs/openbsd/src/sys/dev/pci/ixgbe.c,v
> diff -u -p -u -p -r1.27 ixgbe.c
> --- sys/dev/pci/ixgbe.c	27 Jan 2022 18:28:45 -0000	1.27
> +++ sys/dev/pci/ixgbe.c	24 Oct 2024 05:55:26 -0000
> @@ -70,7 +70,6 @@ int32_t prot_autoc_write_generic(struct 
>  /* MBX */
>  int32_t ixgbe_poll_for_msg(struct ixgbe_hw *hw, uint16_t mbx_id);
>  int32_t ixgbe_poll_for_ack(struct ixgbe_hw *hw, uint16_t mbx_id);
> -uint32_t ixgbe_read_v2p_mailbox(struct ixgbe_hw *hw);
>  int32_t ixgbe_check_for_bit_pf(struct ixgbe_hw *hw, uint32_t mask,
>  			       int32_t index);
>  int32_t ixgbe_check_for_msg_pf(struct ixgbe_hw *hw, uint16_t vf_number);
> @@ -4473,7 +4472,6 @@ void ixgbe_enable_rx(struct ixgbe_hw *hw
>  int32_t ixgbe_read_mbx(struct ixgbe_hw *hw, uint32_t *msg, uint16_t size, uint16_t mbx_id)
>  {
>  	struct ixgbe_mbx_info *mbx = &hw->mbx;
> -	int32_t ret_val = IXGBE_ERR_MBX;
>  
>  	DEBUGFUNC("ixgbe_read_mbx");
>  
> @@ -4482,7 +4480,40 @@ int32_t ixgbe_read_mbx(struct ixgbe_hw *
>  		size = mbx->size;
>  
>  	if (mbx->ops.read)
> -		ret_val = mbx->ops.read(hw, msg, size, mbx_id);
> +		return mbx->ops.read(hw, msg, size, mbx_id);
> +
> +	return IXGBE_ERR_CONFIG;
> +}
> +
> +/**
> + * ixgbe_poll_mbx - Wait for message and read it from the mailbox
> + * @hw: pointer to the HW structure
> + * @msg: The message buffer
> + * @size: Length of buffer
> + * @mbx_id: id of mailbox to read
> + *
> + * returns SUCCESS if it successfully read message from buffer
> + **/
> +int32_t ixgbe_poll_mbx(struct ixgbe_hw *hw, uint32_t *msg, uint16_t size,
> +		       uint16_t mbx_id)
> +{
> +	struct ixgbe_mbx_info *mbx = &hw->mbx;
> +	int32_t ret_val;
> +
> +	DEBUGFUNC("ixgbe_poll_mbx");
> +
> +	if (!mbx->ops.read || !mbx->ops.check_for_msg ||
> +	    !mbx->timeout)
> +		return IXGBE_ERR_CONFIG;
> +
> +	/* limit read to size of mailbox */
> +	if (size > mbx->size)
> +		size = mbx->size;
> +
> +	ret_val = ixgbe_poll_for_msg(hw, mbx_id);
> +	/* if ack received read message, otherwise we timed out */
> +	if (!ret_val)
> +		return mbx->ops.read(hw, msg, size, mbx_id);
>  
>  	return ret_val;
>  }
> @@ -4499,15 +4530,25 @@ int32_t ixgbe_read_mbx(struct ixgbe_hw *
>  int32_t ixgbe_write_mbx(struct ixgbe_hw *hw, uint32_t *msg, uint16_t size, uint16_t mbx_id)
>  {
>  	struct ixgbe_mbx_info *mbx = &hw->mbx;
> -	int32_t ret_val = IXGBE_SUCCESS;
> +	int32_t ret_val = IXGBE_ERR_MBX;
>  
>  	DEBUGFUNC("ixgbe_write_mbx");
>  
> -	if (size > mbx->size)
> -		ret_val = IXGBE_ERR_MBX;
> +	/*
> +	 * exit if either we can't write, release
> +	 * or there is no timeout defined
> +	 */
> +	if (!mbx->ops.write || !mbx->ops.check_for_ack ||
> +	    !mbx->ops.release || !mbx->timeout)
> +		return IXGBE_ERR_CONFIG;
>  
> -	else if (mbx->ops.write)
> +	if (size > mbx->size) {
> +		ret_val = IXGBE_ERR_PARAM;
> +		ERROR_REPORT2(IXGBE_ERROR_ARGUMENT,
> +			     "Invalid mailbox message size %u", size);
> +	} else {
>  		ret_val = mbx->ops.write(hw, msg, size, mbx_id);
> +	}
>  
>  	return ret_val;
>  }
> @@ -4587,7 +4628,7 @@ int32_t ixgbe_poll_for_msg(struct ixgbe_
>  	DEBUGFUNC("ixgbe_poll_for_msg");
>  
>  	if (!countdown || !mbx->ops.check_for_msg)
> -		goto out;
> +		return IXGBE_ERR_CONFIG;
>  
>  	while (countdown && mbx->ops.check_for_msg(hw, mbx_id)) {
>  		countdown--;
> @@ -4596,12 +4637,13 @@ int32_t ixgbe_poll_for_msg(struct ixgbe_
>  		usec_delay(mbx->usec_delay);
>  	}
>  
> -	if (countdown == 0)
> +	if (countdown == 0) {
>  		ERROR_REPORT2(IXGBE_ERROR_POLLING,
> -			   "Polling for VF%d mailbox message timedout", mbx_id);
> +			   "Polling for VF%u mailbox message timedout", mbx_id);
> +		return IXGBE_ERR_TIMEOUT;
> +	}
>  
> -out:
> -	return countdown ? IXGBE_SUCCESS : IXGBE_ERR_MBX;
> +	return IXGBE_SUCCESS;
>  }
>  
>  /**
> @@ -4619,7 +4661,7 @@ int32_t ixgbe_poll_for_ack(struct ixgbe_
>  	DEBUGFUNC("ixgbe_poll_for_ack");
>  
>  	if (!countdown || !mbx->ops.check_for_ack)
> -		goto out;
> +		return IXGBE_ERR_CONFIG;
>  
>  	while (countdown && mbx->ops.check_for_ack(hw, mbx_id)) {
>  		countdown--;
> @@ -4628,12 +4670,180 @@ int32_t ixgbe_poll_for_ack(struct ixgbe_
>  		usec_delay(mbx->usec_delay);
>  	}
>  
> -	if (countdown == 0)
> +	if (countdown == 0) {
>  		ERROR_REPORT2(IXGBE_ERROR_POLLING,
> -			     "Polling for VF%d mailbox ack timedout", mbx_id);
> +			     "Polling for VF%u mailbox ack timedout", mbx_id);
> +		return IXGBE_ERR_TIMEOUT;
> +	}
>  
> -out:
> -	return countdown ? IXGBE_SUCCESS : IXGBE_ERR_MBX;
> +	return IXGBE_SUCCESS;
> +}
> +
> +/**
> + * ixgbe_read_mailbox_vf - read VF's mailbox register
> + * @hw: pointer to the HW structure
> + *
> + * This function is used to read the mailbox register dedicated for VF without
> + * losing the read to clear status bits.
> + **/
> +static uint32_t ixgbe_read_mailbox_vf(struct ixgbe_hw *hw)
> +{
> +	uint32_t vf_mailbox = IXGBE_READ_REG(hw, IXGBE_VFMAILBOX);
> +
> +	vf_mailbox |= hw->mbx.vf_mailbox;
> +	hw->mbx.vf_mailbox |= vf_mailbox & IXGBE_VFMAILBOX_R2C_BITS;
> +
> +	return vf_mailbox;
> +}
> +
> +static void ixgbe_clear_msg_vf(struct ixgbe_hw *hw)
> +{
> +	uint32_t vf_mailbox = ixgbe_read_mailbox_vf(hw);
> +
> +	if (vf_mailbox & IXGBE_VFMAILBOX_PFSTS) {
> +		hw->mbx.stats.reqs++;
> +		hw->mbx.vf_mailbox &= ~IXGBE_VFMAILBOX_PFSTS;
> +	}
> +}
> +
> +static void ixgbe_clear_ack_vf(struct ixgbe_hw *hw)
> +{
> +	uint32_t vf_mailbox = ixgbe_read_mailbox_vf(hw);
> +
> +	if (vf_mailbox & IXGBE_VFMAILBOX_PFACK) {
> +		hw->mbx.stats.acks++;
> +		hw->mbx.vf_mailbox &= ~IXGBE_VFMAILBOX_PFACK;
> +	}
> +}
> +
> +static void ixgbe_clear_rst_vf(struct ixgbe_hw *hw)
> +{
> +	uint32_t vf_mailbox = ixgbe_read_mailbox_vf(hw);
> +
> +	if (vf_mailbox & (IXGBE_VFMAILBOX_RSTI | IXGBE_VFMAILBOX_RSTD)) {
> +		hw->mbx.stats.rsts++;
> +		hw->mbx.vf_mailbox &= ~(IXGBE_VFMAILBOX_RSTI |
> +					IXGBE_VFMAILBOX_RSTD);
> +	}
> +}
> +
> +/**
> + * ixgbe_check_for_bit_vf - Determine if a status bit was set
> + * @hw: pointer to the HW structure
> + * @mask: bitmask for bits to be tested and cleared
> + *
> + * This function is used to check for the read to clear bits within
> + * the V2P mailbox.
> + **/
> +static int32_t ixgbe_check_for_bit_vf(struct ixgbe_hw *hw, uint32_t mask)
> +{
> +	uint32_t vf_mailbox = ixgbe_read_mailbox_vf(hw);
> +
> +	if (vf_mailbox & mask)
> +		return IXGBE_SUCCESS;
> +
> +	return IXGBE_ERR_MBX;
> +}
> +
> +/**
> + * ixgbe_check_for_msg_vf - checks to see if the PF has sent mail
> + * @hw: pointer to the HW structure
> + * @mbx_id: id of mailbox to check
> + *
> + * returns SUCCESS if the PF has set the Status bit or else ERR_MBX
> + **/
> +static int32_t ixgbe_check_for_msg_vf(struct ixgbe_hw *hw, uint16_t mbx_id)
> +{
> +	DEBUGFUNC("ixgbe_check_for_msg_vf");
> +
> +	if (!ixgbe_check_for_bit_vf(hw, IXGBE_VFMAILBOX_PFSTS))
> +		return IXGBE_SUCCESS;
> +
> +	return IXGBE_ERR_MBX;
> +}
> +
> +/**
> + * ixgbe_check_for_ack_vf - checks to see if the PF has ACK'd
> + * @hw: pointer to the HW structure
> + * @mbx_id: id of mailbox to check
> + *
> + * returns SUCCESS if the PF has set the ACK bit or else ERR_MBX
> + **/
> +static int32_t ixgbe_check_for_ack_vf(struct ixgbe_hw *hw, uint16_t mbx_id)
> +{
> +	DEBUGFUNC("ixgbe_check_for_ack_vf");
> +
> +	if (!ixgbe_check_for_bit_vf(hw, IXGBE_VFMAILBOX_PFACK)) {
> +		/* TODO: should this be autocleared? */
> +		ixgbe_clear_ack_vf(hw);
> +		return IXGBE_SUCCESS;
> +	}
> +
> +	return IXGBE_ERR_MBX;
> +}
> +
> +/**
> + * ixgbe_check_for_rst_vf - checks to see if the PF has reset
> + * @hw: pointer to the HW structure
> + * @mbx_id: id of mailbox to check
> + *
> + * returns TRUE if the PF has set the reset done bit or else FALSE
> + **/
> +static int32_t ixgbe_check_for_rst_vf(struct ixgbe_hw *hw, uint16_t mbx_id)
> +{
> +	DEBUGFUNC("ixgbe_check_for_rst_vf");
> +
> +	if (!ixgbe_check_for_bit_vf(hw, IXGBE_VFMAILBOX_RSTI |
> +					  IXGBE_VFMAILBOX_RSTD)) {
> +		/* TODO: should this be autocleared? */
> +		ixgbe_clear_rst_vf(hw);
> +		return IXGBE_SUCCESS;
> +	}
> +
> +	return IXGBE_ERR_MBX;
> +}
> +
> +/**
> + * ixgbe_obtain_mbx_lock_vf - obtain mailbox lock
> + * @hw: pointer to the HW structure
> + *
> + * return SUCCESS if we obtained the mailbox lock
> + **/
> +static int32_t ixgbe_obtain_mbx_lock_vf(struct ixgbe_hw *hw)
> +{
> +	struct ixgbe_mbx_info *mbx = &hw->mbx;
> +	int countdown = mbx->timeout;
> +	int32_t ret_val = IXGBE_ERR_MBX;
> +	uint32_t vf_mailbox;
> +
> +	DEBUGFUNC("ixgbe_obtain_mbx_lock_vf");
> +
> +	if (!mbx->timeout)
> +		return IXGBE_ERR_CONFIG;
> +
> +	while (countdown--) {
> +		/* Reserve mailbox for VF use */
> +		vf_mailbox = ixgbe_read_mailbox_vf(hw);
> +		vf_mailbox |= IXGBE_VFMAILBOX_VFU;
> +		IXGBE_WRITE_REG(hw, IXGBE_VFMAILBOX, vf_mailbox);
> +
> +		/* Verify that VF is the owner of the lock */
> +		if (ixgbe_read_mailbox_vf(hw) & IXGBE_VFMAILBOX_VFU) {
> +			ret_val = IXGBE_SUCCESS;
> +			break;
> +		}
> +
> +		/* Wait a bit before trying again */
> +		usec_delay(mbx->usec_delay);
> +	}
> +
> +	if (ret_val != IXGBE_SUCCESS) {
> +		ERROR_REPORT1(IXGBE_ERROR_INVALID_STATE,
> +				"Failed to obtain mailbox lock");
> +		ret_val = IXGBE_ERR_TIMEOUT;
> +	}
> +
> +	return ret_val;
>  }
>  
>  /**
> @@ -4711,34 +4921,14 @@ void ixgbe_init_mbx_ops_generic(struct i
>  	mbx->ops.write_posted = ixgbe_write_posted_mbx;
>  }
>  
> -/**
> - *  ixgbe_read_v2p_mailbox - read v2p mailbox
> - *  @hw: pointer to the HW structure
> - *
> - *  This function is used to read the v2p mailbox without losing the read to
> - *  clear status bits.
> - **/
> -uint32_t ixgbe_read_v2p_mailbox(struct ixgbe_hw *hw)
> -{
> -	uint32_t v2p_mailbox = IXGBE_READ_REG(hw, IXGBE_VFMAILBOX);
> -
> -	v2p_mailbox |= hw->mbx.v2p_mailbox;
> -	hw->mbx.v2p_mailbox |= v2p_mailbox & IXGBE_VFMAILBOX_R2C_BITS;
> -
> -	return v2p_mailbox;
> -}
> -
>  int32_t ixgbe_check_for_bit_pf(struct ixgbe_hw *hw, uint32_t mask, int32_t index)
>  {
> -	uint32_t mbvficr = IXGBE_READ_REG(hw, IXGBE_MBVFICR(index));
> -	int32_t ret_val = IXGBE_ERR_MBX;
> +	uint32_t pfmbicr = IXGBE_READ_REG(hw, IXGBE_PFMBICR(index));
>  
> -	if (mbvficr & mask) {
> -		ret_val = IXGBE_SUCCESS;
> -		IXGBE_WRITE_REG(hw, IXGBE_MBVFICR(index), mask);
> -	}
> +	if (pfmbicr & mask)
> +		return IXGBE_SUCCESS;
>  
> -	return ret_val;
> +	return IXGBE_ERR_MBX;
>  }
>  
>  /**
> @@ -4748,21 +4938,47 @@ int32_t ixgbe_check_for_bit_pf(struct ix
>   *
>   *  returns SUCCESS if the VF has set the Status bit or else ERR_MBX
>   **/
> -int32_t ixgbe_check_for_msg_pf(struct ixgbe_hw *hw, uint16_t vf_number)
> +int32_t ixgbe_check_for_msg_pf(struct ixgbe_hw *hw, uint16_t vf_id)
>  {
> -	int32_t ret_val = IXGBE_ERR_MBX;
> -	int32_t index = IXGBE_MBVFICR_INDEX(vf_number);
> -	uint32_t vf_bit = vf_number % 16;
> -
> +	uint32_t vf_shift = IXGBE_PFMBICR_SHIFT(vf_id);
> +	int32_t index = IXGBE_PFMBICR_INDEX(vf_id);
>  	DEBUGFUNC("ixgbe_check_for_msg_pf");
>  
> -	if (!ixgbe_check_for_bit_pf(hw, IXGBE_MBVFICR_VFREQ_VF1 << vf_bit,
> -				    index)) {
> -		ret_val = IXGBE_SUCCESS;
> +	if (!ixgbe_check_for_bit_pf(hw, IXGBE_PFMBICR_VFREQ_VF1 << vf_shift,
> +				    index))
> +		return IXGBE_SUCCESS;
> +
> +	return IXGBE_ERR_MBX;
> +}
> +
> +static void ixgbe_clear_msg_pf(struct ixgbe_hw *hw, uint16_t vf_id)
> +{
> +	uint32_t vf_shift = IXGBE_PFMBICR_SHIFT(vf_id);
> +	int32_t index = IXGBE_PFMBICR_INDEX(vf_id);
> +	uint32_t pfmbicr;
> +
> +	pfmbicr = IXGBE_READ_REG(hw, IXGBE_PFMBICR(index));
> +
> +	if (pfmbicr & (IXGBE_PFMBICR_VFREQ_VF1 << vf_shift))
>  		hw->mbx.stats.reqs++;
> -	}
>  
> -	return ret_val;
> +	IXGBE_WRITE_REG(hw, IXGBE_PFMBICR(index),
> +			IXGBE_PFMBICR_VFREQ_VF1 << vf_shift);
> +}
> +
> +static void ixgbe_clear_ack_pf(struct ixgbe_hw *hw, uint16_t vf_id)
> +{
> +	uint32_t vf_shift = IXGBE_PFMBICR_SHIFT(vf_id);
> +	int32_t index = IXGBE_PFMBICR_INDEX(vf_id);
> +	uint32_t pfmbicr;
> +
> +	pfmbicr = IXGBE_READ_REG(hw, IXGBE_PFMBICR(index));
> +
> +	if (pfmbicr & (IXGBE_PFMBICR_VFACK_VF1 << vf_shift))
> +		hw->mbx.stats.acks++;
> +
> +	IXGBE_WRITE_REG(hw, IXGBE_PFMBICR(index),
> +			IXGBE_PFMBICR_VFACK_VF1 << vf_shift);
>  }
>  
>  /**
> @@ -4772,18 +4988,19 @@ int32_t ixgbe_check_for_msg_pf(struct ix
>   *
>   *  returns SUCCESS if the VF has set the Status bit or else ERR_MBX
>   **/
> -int32_t ixgbe_check_for_ack_pf(struct ixgbe_hw *hw, uint16_t vf_number)
> +int32_t ixgbe_check_for_ack_pf(struct ixgbe_hw *hw, uint16_t vf_id)
>  {
> +	uint32_t vf_shift = IXGBE_PFMBICR_SHIFT(vf_id);
> +	int32_t index = IXGBE_PFMBICR_INDEX(vf_id);
>  	int32_t ret_val = IXGBE_ERR_MBX;
> -	int32_t index = IXGBE_MBVFICR_INDEX(vf_number);
> -	uint32_t vf_bit = vf_number % 16;
>  
>  	DEBUGFUNC("ixgbe_check_for_ack_pf");
>  
> -	if (!ixgbe_check_for_bit_pf(hw, IXGBE_MBVFICR_VFACK_VF1 << vf_bit,
> +	if (!ixgbe_check_for_bit_pf(hw, IXGBE_PFMBICR_VFACK_VF1 << vf_shift,
>  				    index)) {
>  		ret_val = IXGBE_SUCCESS;
> -		hw->mbx.stats.acks++;
> +		/* TODO: should this be autocleared? */
> +		ixgbe_clear_ack_pf(hw, vf_id);
>  	}
>  
>  	return ret_val;
> @@ -4796,24 +5013,24 @@ int32_t ixgbe_check_for_ack_pf(struct ix
>   *
>   *  returns SUCCESS if the VF has set the Status bit or else ERR_MBX
>   **/
> -int32_t ixgbe_check_for_rst_pf(struct ixgbe_hw *hw, uint16_t vf_number)
> +int32_t ixgbe_check_for_rst_pf(struct ixgbe_hw *hw, uint16_t vf_id)
>  {
> -	uint32_t reg_offset = (vf_number < 32) ? 0 : 1;
> -	uint32_t vf_shift = vf_number % 32;
> -	uint32_t vflre = 0;
> +	uint32_t vf_shift = IXGBE_PFVFLRE_SHIFT(vf_id);
> +	uint32_t index = IXGBE_PFVFLRE_INDEX(vf_id);
>  	int32_t ret_val = IXGBE_ERR_MBX;
> +	uint32_t vflre = 0;
>  
>  	DEBUGFUNC("ixgbe_check_for_rst_pf");
>  
>  	switch (hw->mac.type) {
>  	case ixgbe_mac_82599EB:
> -		vflre = IXGBE_READ_REG(hw, IXGBE_VFLRE(reg_offset));
> +		vflre = IXGBE_READ_REG(hw, IXGBE_PFVFLRE(index));
>  		break;
>  	case ixgbe_mac_X550:
>  	case ixgbe_mac_X550EM_x:
>  	case ixgbe_mac_X550EM_a:
>  	case ixgbe_mac_X540:
> -		vflre = IXGBE_READ_REG(hw, IXGBE_VFLREC(reg_offset));
> +		vflre = IXGBE_READ_REG(hw, IXGBE_PFVFLREC(index));
>  		break;
>  	default:
>  		break;
> @@ -4821,7 +5038,7 @@ int32_t ixgbe_check_for_rst_pf(struct ix
>  
>  	if (vflre & (1 << vf_shift)) {
>  		ret_val = IXGBE_SUCCESS;
> -		IXGBE_WRITE_REG(hw, IXGBE_VFLREC(reg_offset), (1 << vf_shift));
> +		IXGBE_WRITE_REG(hw, IXGBE_PFVFLREC(index), (1 << vf_shift));
>  		hw->mbx.stats.rsts++;
>  	}
>  
> @@ -4835,24 +5052,40 @@ int32_t ixgbe_check_for_rst_pf(struct ix
>   *
>   *  return SUCCESS if we obtained the mailbox lock
>   **/
> -int32_t ixgbe_obtain_mbx_lock_pf(struct ixgbe_hw *hw, uint16_t vf_number)
> +int32_t ixgbe_obtain_mbx_lock_pf(struct ixgbe_hw *hw, uint16_t vf_id)
>  {
> +	struct ixgbe_mbx_info *mbx = &hw->mbx;
> +	int countdown = mbx->timeout;
>  	int32_t ret_val = IXGBE_ERR_MBX;
> -	uint32_t p2v_mailbox;
> +	uint32_t pf_mailbox;
>  
>  	DEBUGFUNC("ixgbe_obtain_mbx_lock_pf");
>  
> -	/* Take ownership of the buffer */
> -	IXGBE_WRITE_REG(hw, IXGBE_PFMAILBOX(vf_number), IXGBE_PFMAILBOX_PFU);
> +	if (!mbx->timeout)
> +		return IXGBE_ERR_CONFIG;
>  
> -	/* reserve mailbox for vf use */
> -	p2v_mailbox = IXGBE_READ_REG(hw, IXGBE_PFMAILBOX(vf_number));
> -	if (p2v_mailbox & IXGBE_PFMAILBOX_PFU)
> -		ret_val = IXGBE_SUCCESS;
> -	else
> -		ERROR_REPORT2(IXGBE_ERROR_POLLING,
> -			   "Failed to obtain mailbox lock for VF%d", vf_number);
> +	while (countdown--) {
> +		/* Reserve mailbox for PF use */
> +		pf_mailbox = IXGBE_READ_REG(hw, IXGBE_PFMAILBOX(vf_id));
> +		pf_mailbox |= IXGBE_PFMAILBOX_PFU;
> +		IXGBE_WRITE_REG(hw, IXGBE_PFMAILBOX(vf_id), pf_mailbox);
> +
> +		/* Verify that PF is the owner of the lock */
> +		pf_mailbox = IXGBE_READ_REG(hw, IXGBE_PFMAILBOX(vf_id));
> +		if (pf_mailbox & IXGBE_PFMAILBOX_PFU) {
> +			ret_val = IXGBE_SUCCESS;
> +			break;
> +		}
> +
> +		/* Wait a bit before trying again */
> +		usec_delay(mbx->usec_delay);
> +	}
>  
> +	if (ret_val != IXGBE_SUCCESS) {
> +		ERROR_REPORT1(IXGBE_ERROR_INVALID_STATE,
> +			      "Failed to obtain mailbox lock");
> +		ret_val = IXGBE_ERR_TIMEOUT;
> +	}
>  
>  	return ret_val;
>  }
> @@ -4867,35 +5100,79 @@ int32_t ixgbe_obtain_mbx_lock_pf(struct 
>   *  returns SUCCESS if it successfully copied message into the buffer
>   **/
>  int32_t ixgbe_write_mbx_pf(struct ixgbe_hw *hw, uint32_t *msg, uint16_t size,
> -			   uint16_t vf_number)
> +			   uint16_t vf_id)
>  {
> +	uint32_t pf_mailbox;
>  	int32_t ret_val;
>  	uint16_t i;
>  
>  	DEBUGFUNC("ixgbe_write_mbx_pf");
>  
>  	/* lock the mailbox to prevent pf/vf race condition */
> -	ret_val = ixgbe_obtain_mbx_lock_pf(hw, vf_number);
> +	ret_val = ixgbe_obtain_mbx_lock_pf(hw, vf_id);
>  	if (ret_val)
> -		goto out_no_write;
> +		goto out;
>  
>  	/* flush msg and acks as we are overwriting the message buffer */
> -	ixgbe_check_for_msg_pf(hw, vf_number);
> -	ixgbe_check_for_ack_pf(hw, vf_number);
> +	ixgbe_clear_msg_pf(hw, vf_id);
> +	ixgbe_clear_ack_pf(hw, vf_id);
>  
>  	/* copy the caller specified message to the mailbox memory buffer */
>  	for (i = 0; i < size; i++)
> -		IXGBE_WRITE_REG_ARRAY(hw, IXGBE_PFMBMEM(vf_number), i, msg[i]);
> +		IXGBE_WRITE_REG_ARRAY(hw, IXGBE_PFMBMEM(vf_id), i, msg[i]);
>  
> -	/* Interrupt VF to tell it a message has been sent and release buffer*/
> -	IXGBE_WRITE_REG(hw, IXGBE_PFMAILBOX(vf_number), IXGBE_PFMAILBOX_STS);
> +	/* Interrupt VF to tell it a message has been sent */
> +	pf_mailbox = IXGBE_READ_REG(hw, IXGBE_PFMAILBOX(vf_id));
> +	pf_mailbox |= IXGBE_PFMAILBOX_STS;
> +	IXGBE_WRITE_REG(hw, IXGBE_PFMAILBOX(vf_id), pf_mailbox);
> +
> +	/* if msg sent wait until we receive an ack */
> +	ixgbe_poll_for_ack(hw, vf_id);
>  
>  	/* update stats */
>  	hw->mbx.stats.msgs_tx++;
>  
> -out_no_write:
> +out:
> +	hw->mbx.ops.release(hw, vf_id);
> +
>  	return ret_val;
> +}
> +
> +/**
> + * ixgbe_read_mbx_pf_legacy - Read a message from the mailbox
> + * @hw: pointer to the HW structure
> + * @msg: The message buffer
> + * @size: Length of buffer
> + * @vf_id: the VF index
> + *
> + * This function copies a message from the mailbox buffer to the caller's
> + * memory buffer.  The presumption is that the caller knows that there was
> + * a message due to a VF request so no polling for message is needed.
> + **/
> +static int32_t ixgbe_read_mbx_pf_legacy(struct ixgbe_hw *hw, uint32_t *msg,
> +					uint16_t size, uint16_t vf_id)
> +{
> +	int32_t ret_val;
> +	uint16_t i;
> +
> +	DEBUGFUNC("ixgbe_read_mbx_pf_legacy");
> +
> +	/* lock the mailbox to prevent pf/vf race condition */
> +	ret_val = ixgbe_obtain_mbx_lock_pf(hw, vf_id);
> +	if (ret_val != IXGBE_SUCCESS)
> +		return ret_val;
>  
> +	/* copy the message to the mailbox memory buffer */
> +	for (i = 0; i < size; i++)
> +		msg[i] = IXGBE_READ_REG_ARRAY(hw, IXGBE_PFMBMEM(vf_id), i);
> +
> +	/* Acknowledge the message and release buffer */
> +	IXGBE_WRITE_REG(hw, IXGBE_PFMAILBOX(vf_id), IXGBE_PFMAILBOX_ACK);
> +
> +	/* update stats */
> +	hw->mbx.stats.msgs_rx++;
> +
> +	return IXGBE_SUCCESS;
>  }
>  
>  /**
> @@ -4910,30 +5187,213 @@ out_no_write:
>   *  a message due to a VF request so no polling for message is needed.
>   **/
>  int32_t ixgbe_read_mbx_pf(struct ixgbe_hw *hw, uint32_t *msg, uint16_t size,
> -			  uint16_t vf_number)
> +			  uint16_t vf_id)
>  {
> +	uint32_t pf_mailbox;
>  	int32_t ret_val;
>  	uint16_t i;
>  
>  	DEBUGFUNC("ixgbe_read_mbx_pf");
>  
> -	/* lock the mailbox to prevent pf/vf race condition */
> -	ret_val = ixgbe_obtain_mbx_lock_pf(hw, vf_number);
> -	if (ret_val)
> -		goto out_no_read;
> +	/* check if there is a message from VF */
> +	ret_val = ixgbe_check_for_msg_pf(hw, vf_id);
> +	if (ret_val != IXGBE_SUCCESS)
> +		return IXGBE_ERR_MBX_NOMSG;
> +
> +	ixgbe_clear_msg_pf(hw, vf_id);
>  
>  	/* copy the message to the mailbox memory buffer */
>  	for (i = 0; i < size; i++)
> -		msg[i] = IXGBE_READ_REG_ARRAY(hw, IXGBE_PFMBMEM(vf_number), i);
> +		msg[i] = IXGBE_READ_REG_ARRAY(hw, IXGBE_PFMBMEM(vf_id), i);
>  
>  	/* Acknowledge the message and release buffer */
> -	IXGBE_WRITE_REG(hw, IXGBE_PFMAILBOX(vf_number), IXGBE_PFMAILBOX_ACK);
> +	pf_mailbox = IXGBE_READ_REG(hw, IXGBE_PFMAILBOX(vf_id));
> +	pf_mailbox |= IXGBE_PFMAILBOX_ACK;
> +	IXGBE_WRITE_REG(hw, IXGBE_PFMAILBOX(vf_id), pf_mailbox);
>  
>  	/* update stats */
>  	hw->mbx.stats.msgs_rx++;
>  
> -out_no_read:
> -	return ret_val;
> +	return IXGBE_SUCCESS;
> +}
> +
> +/**
> + * ixgbe_release_mbx_lock_dummy - release mailbox lock
> + * @hw: pointer to the HW structure
> + * @mbx_id: id of mailbox to read
> + **/
> +static void ixgbe_release_mbx_lock_dummy(struct ixgbe_hw *hw, uint16_t mbx_id)
> +{
> +	DEBUGFUNC("ixgbe_release_mbx_lock_dummy");
> +}
> +
> +/**
> + * ixgbe_write_mbx_vf_legacy - Write a message to the mailbox
> + * @hw: pointer to the HW structure
> + * @msg: The message buffer
> + * @size: Length of buffer
> + * @mbx_id: id of mailbox to write
> + *
> + * returns SUCCESS if it successfully copied message into the buffer
> + **/
> +static int32_t ixgbe_write_mbx_vf_legacy(struct ixgbe_hw *hw, uint32_t *msg,
> +					 uint16_t size, uint16_t mbx_id)
> +{
> +	int32_t ret_val;
> +	uint16_t i;
> +
> +	DEBUGFUNC("ixgbe_write_mbx_vf_legacy");
> +
> +	/* lock the mailbox to prevent pf/vf race condition */
> +	ret_val = ixgbe_obtain_mbx_lock_vf(hw);
> +	if (ret_val)
> +		return ret_val;
> +
> +	/* flush msg and acks as we are overwriting the message buffer */
> +	ixgbe_check_for_msg_vf(hw, 0);
> +	ixgbe_clear_msg_vf(hw);
> +	ixgbe_check_for_ack_vf(hw, 0);
> +	ixgbe_clear_ack_vf(hw);
> +
> +	/* copy the caller specified message to the mailbox memory buffer */
> +	for (i = 0; i < size; i++)
> +		IXGBE_WRITE_REG_ARRAY(hw, IXGBE_VFMBMEM, i, msg[i]);
> +
> +	/* update stats */
> +	hw->mbx.stats.msgs_tx++;
> +
> +	/* interrupt the PF to tell it a message has been sent */
> +	IXGBE_WRITE_REG(hw, IXGBE_VFMAILBOX, IXGBE_VFMAILBOX_REQ);
> +
> +	return IXGBE_SUCCESS;
> +}
> +
> +/**
> + * ixgbe_read_mbx_vf_legacy - Reads a message from the inbox intended for vf
> + * @hw: pointer to the HW structure
> + * @msg: The message buffer
> + * @size: Length of buffer
> + * @mbx_id: id of mailbox to read
> + *
> + * returns SUCCESS if it successfully read message from buffer
> + **/
> +static int32_t ixgbe_read_mbx_vf_legacy(struct ixgbe_hw *hw, uint32_t *msg,
> +					uint16_t size, uint16_t mbx_id)
> +{
> +	int32_t ret_val;
> +	uint16_t i;
> +
> +	DEBUGFUNC("ixgbe_read_mbx_vf_legacy");
> +
> +	/* lock the mailbox to prevent pf/vf race condition */
> +	ret_val = ixgbe_obtain_mbx_lock_vf(hw);
> +	if (ret_val)
> +		return ret_val;
> +
> +	/* copy the message from the mailbox memory buffer */
> +	for (i = 0; i < size; i++)
> +		msg[i] = IXGBE_READ_REG_ARRAY(hw, IXGBE_VFMBMEM, i);
> +
> +	/* Acknowledge receipt and release mailbox, then we're done */
> +	IXGBE_WRITE_REG(hw, IXGBE_VFMAILBOX, IXGBE_VFMAILBOX_ACK);
> +
> +	/* update stats */
> +	hw->mbx.stats.msgs_rx++;
> +
> +	return IXGBE_SUCCESS;
> +}
> +
> +/**
> + * ixgbe_init_mbx_params_vf - set initial values for vf mailbox
> + * @hw: pointer to the HW structure
> + *
> + * Initializes single set the hw->mbx struct to correct values for vf mailbox
> + * Set of legacy functions is being used here
> + */
> +void ixgbe_init_mbx_params_vf(struct ixgbe_hw *hw)
> +{
> +	struct ixgbe_mbx_info *mbx = &hw->mbx;
> +
> +	mbx->timeout = IXGBE_VF_MBX_INIT_TIMEOUT;
> +	mbx->usec_delay = IXGBE_VF_MBX_INIT_DELAY;
> +
> +	mbx->size = IXGBE_VFMAILBOX_SIZE;
> +
> +	mbx->ops.release = ixgbe_release_mbx_lock_dummy;
> +	mbx->ops.read = ixgbe_read_mbx_vf_legacy;
> +	mbx->ops.write = ixgbe_write_mbx_vf_legacy;
> +	mbx->ops.check_for_msg = ixgbe_check_for_msg_vf;
> +	mbx->ops.check_for_ack = ixgbe_check_for_ack_vf;
> +	mbx->ops.check_for_rst = ixgbe_check_for_rst_vf;
> +	mbx->ops.clear = NULL;
> +
> +	mbx->stats.msgs_tx = 0;
> +	mbx->stats.msgs_rx = 0;
> +	mbx->stats.reqs = 0;
> +	mbx->stats.acks = 0;
> +	mbx->stats.rsts = 0;
> +}
> +
> +/**
> + * ixgbe_write_mbx_pf_legacy - Places a message in the mailbox
> + * @hw: pointer to the HW structure
> + * @msg: The message buffer
> + * @size: Length of buffer
> + * @vf_id: the VF index
> + *
> + * returns SUCCESS if it successfully copied message into the buffer
> + **/
> +static int32_t ixgbe_write_mbx_pf_legacy(struct ixgbe_hw *hw, uint32_t *msg,
> +					 uint16_t size, uint16_t vf_id)
> +{
> +	int32_t ret_val;
> +	uint16_t i;
> +
> +	DEBUGFUNC("ixgbe_write_mbx_pf_legacy");
> +
> +	/* lock the mailbox to prevent pf/vf race condition */
> +	ret_val = ixgbe_obtain_mbx_lock_pf(hw, vf_id);
> +	if (ret_val)
> +		return ret_val;
> +
> +	/* flush msg and acks as we are overwriting the message buffer */
> +	ixgbe_check_for_msg_pf(hw, vf_id);
> +	ixgbe_clear_msg_pf(hw, vf_id);
> +	ixgbe_check_for_ack_pf(hw, vf_id);
> +	ixgbe_clear_ack_pf(hw, vf_id);
> +
> +	/* copy the caller specified message to the mailbox memory buffer */
> +	for (i = 0; i < size; i++)
> +		IXGBE_WRITE_REG_ARRAY(hw, IXGBE_PFMBMEM(vf_id), i, msg[i]);
> +
> +	/* Interrupt VF to tell it a message has been sent and release buffer*/
> +	IXGBE_WRITE_REG(hw, IXGBE_PFMAILBOX(vf_id), IXGBE_PFMAILBOX_STS);
> +
> +	/* update stats */
> +	hw->mbx.stats.msgs_tx++;
> +
> +	return IXGBE_SUCCESS;
> +}
> +
> +/**
> + * ixgbe_clear_mbx_pf - Clear Mailbox Memory
> + * @hw: pointer to the HW structure
> + * @vf_id: the VF index
> + *
> + * Set VFMBMEM of given VF to 0x0.
> + **/
> +static int32_t ixgbe_clear_mbx_pf(struct ixgbe_hw *hw, uint16_t vf_id)
> +{
> +	uint16_t mbx_size = hw->mbx.size;
> +	uint16_t i;
> +
> +	if (vf_id > 63)
> +		return IXGBE_ERR_PARAM;
> +
> +	for (i = 0; i < mbx_size; ++i)
> +		IXGBE_WRITE_REG_ARRAY(hw, IXGBE_PFMBMEM(vf_id), i, 0x0);
> +
> +	return IXGBE_SUCCESS;
>  }
>  
>  /**
> @@ -4953,22 +5413,24 @@ void ixgbe_init_mbx_params_pf(struct ixg
>  	    hw->mac.type != ixgbe_mac_X540)
>  		return;
>  
> -	mbx->timeout = 0;
> -	mbx->usec_delay = 0;
> -
> +	/* Initialize common mailbox settings */
> +	mbx->timeout = IXGBE_VF_MBX_INIT_TIMEOUT;
> +	mbx->usec_delay = IXGBE_VF_MBX_INIT_DELAY;
>  	mbx->size = IXGBE_VFMAILBOX_SIZE;
>  
> -	mbx->ops.read = ixgbe_read_mbx_pf;
> -	mbx->ops.write = ixgbe_write_mbx_pf;
> -	mbx->ops.read_posted = ixgbe_read_posted_mbx;
> -	mbx->ops.write_posted = ixgbe_write_posted_mbx;
> -	mbx->ops.check_for_msg = ixgbe_check_for_msg_pf;
> -	mbx->ops.check_for_ack = ixgbe_check_for_ack_pf;
> -	mbx->ops.check_for_rst = ixgbe_check_for_rst_pf;
> -
> +	/* Initialize counters with zeroes */
>  	mbx->stats.msgs_tx = 0;
>  	mbx->stats.msgs_rx = 0;
>  	mbx->stats.reqs = 0;
>  	mbx->stats.acks = 0;
>  	mbx->stats.rsts = 0;
> +
> +	/* Initialize mailbox operations */
> +	mbx->ops.release = ixgbe_release_mbx_lock_dummy;
> +	mbx->ops.read = ixgbe_read_mbx_pf_legacy;
> +	mbx->ops.write = ixgbe_write_mbx_pf_legacy;
> +	mbx->ops.check_for_msg = ixgbe_check_for_msg_pf;
> +	mbx->ops.check_for_ack = ixgbe_check_for_ack_pf;
> +	mbx->ops.check_for_rst = ixgbe_check_for_rst_pf;
> +	mbx->ops.clear = ixgbe_clear_mbx_pf;
>  }
> Index: sys/dev/pci/ixgbe.h
> ===================================================================
> RCS file: /disk/cvs/openbsd/src/sys/dev/pci/ixgbe.h,v
> diff -u -p -u -p -r1.36 ixgbe.h
> --- sys/dev/pci/ixgbe.h	9 Jun 2024 05:18:12 -0000	1.36
> +++ sys/dev/pci/ixgbe.h	24 Oct 2024 05:55:26 -0000
> @@ -278,6 +278,7 @@ int32_t ixgbe_clear_vmdq(struct ixgbe_hw
>  int32_t ixgbe_init_uta_tables(struct ixgbe_hw *hw);
>  
>  void ixgbe_add_uc_addr(struct ixgbe_hw *hw, uint8_t *addr, uint32_t vmdq);
> +int32_t ixgbe_mta_vector(struct ixgbe_hw *hw, uint8_t *mc_addr);
>  void ixgbe_set_mta(struct ixgbe_hw *hw, uint8_t *mc_addr);
>  
>  void ixgbe_disable_rx(struct ixgbe_hw *hw);
> @@ -354,8 +355,37 @@ int32_t ixgbe_write_i2c_combined_generic
>  int32_t ixgbe_write_i2c_combined_generic_unlocked(struct ixgbe_hw *, uint8_t addr,
>  						  uint16_t reg, uint16_t val);
>  
> +/* Virtual Functions */
> +int32_t ixgbe_init_ops_vf(struct ixgbe_hw *hw);
> +int32_t ixgbe_init_hw_vf(struct ixgbe_hw *hw);
> +int32_t ixgbe_start_hw_vf(struct ixgbe_hw *hw);
> +int32_t ixgbe_reset_hw_vf(struct ixgbe_hw *hw);
> +int32_t ixgbe_stop_adapter_vf(struct ixgbe_hw *hw);
> +uint32_t ixgbe_get_num_of_tx_queues_vf(struct ixgbe_hw *hw);
> +uint32_t ixgbe_get_num_of_rx_queues_vf(struct ixgbe_hw *hw);
> +int32_t ixgbe_get_mac_addr_vf(struct ixgbe_hw *hw, uint8_t *mac_addr);
> +int32_t ixgbe_setup_mac_link_vf(struct ixgbe_hw *hw, ixgbe_link_speed speed,
> +			    bool autoneg_wait_to_complete);
> +int32_t ixgbe_check_mac_link_vf(struct ixgbe_hw *hw, ixgbe_link_speed *speed,
> +			    bool *link_up, bool autoneg_wait_to_complete);
> +int32_t ixgbe_set_rar_vf(struct ixgbe_hw *hw, uint32_t index, uint8_t *addr, uint32_t vmdq,
> +		     uint32_t enable_addr);
> +int32_t ixgbevf_set_uc_addr_vf(struct ixgbe_hw *hw, uint32_t index, uint8_t *addr);
> +int32_t ixgbe_update_mc_addr_list_vf(struct ixgbe_hw *hw, uint8_t *mc_addr_list,
> +				 uint32_t mc_addr_count, ixgbe_mc_addr_itr,
> +				 bool clear);
> +int32_t ixgbevf_update_xcast_mode(struct ixgbe_hw *hw, int xcast_mode);
> +int32_t ixgbe_get_link_state_vf(struct ixgbe_hw *hw, bool *link_state);
> +int32_t ixgbe_set_vfta_vf(struct ixgbe_hw *hw, uint32_t vlan, uint32_t vind,
> +		      bool vlan_on, bool vlvf_bypass);
> +int32_t ixgbevf_rlpml_set_vf(struct ixgbe_hw *hw, uint16_t max_size);
> +int ixgbevf_negotiate_api_version(struct ixgbe_hw *hw, int api);
> +int ixgbevf_get_queues(struct ixgbe_hw *hw, unsigned int *num_tcs,
> +		       unsigned int *default_tc);
> +
>  /* MBX */
>  int32_t ixgbe_read_mbx(struct ixgbe_hw *, uint32_t *, uint16_t, uint16_t);
> +int32_t ixgbe_poll_mbx(struct ixgbe_hw *, uint32_t *, uint16_t, uint16_t);
>  int32_t ixgbe_write_mbx(struct ixgbe_hw *, uint32_t *, uint16_t, uint16_t);
>  int32_t ixgbe_read_posted_mbx(struct ixgbe_hw *, uint32_t *, uint16_t, uint16_t);
>  int32_t ixgbe_write_posted_mbx(struct ixgbe_hw *, uint32_t *, uint16_t, uint16_t);
> @@ -363,6 +393,9 @@ int32_t ixgbe_check_for_msg(struct ixgbe
>  int32_t ixgbe_check_for_ack(struct ixgbe_hw *, uint16_t);
>  int32_t ixgbe_check_for_rst(struct ixgbe_hw *, uint16_t);
>  void ixgbe_init_mbx_ops_generic(struct ixgbe_hw *hw);
> +void ixgbe_init_mbx_params_vf(struct ixgbe_hw *);
> +void ixgbe_upgrade_mbx_params_vf(struct ixgbe_hw *);
>  void ixgbe_init_mbx_params_pf(struct ixgbe_hw *);
> +void ixgbe_upgrade_mbx_params_pf(struct ixgbe_hw *, uint16_t);
>  
>  #endif /* _IXGBE_H_ */
> Index: sys/dev/pci/ixgbe_type.h
> ===================================================================
> RCS file: /disk/cvs/openbsd/src/sys/dev/pci/ixgbe_type.h,v
> diff -u -p -u -p -r1.39 ixgbe_type.h
> --- sys/dev/pci/ixgbe_type.h	4 Sep 2024 07:54:52 -0000	1.39
> +++ sys/dev/pci/ixgbe_type.h	24 Oct 2024 05:55:26 -0000
> @@ -463,8 +463,14 @@ struct ixgbe_nvm_version {
>  #define IXGBE_PFMAILBOX(_i)	(0x04B00 + (4 * (_i))) /* 64 total */
>  /* 64 Mailboxes, 16 DW each */
>  #define IXGBE_PFMBMEM(_i)	(0x13000 + (64 * (_i)))
> +#define IXGBE_PFMBICR_INDEX(_i)	((_i) >> 4)
> +#define IXGBE_PFMBICR_SHIFT(_i)	((_i) % 16)
>  #define IXGBE_PFMBICR(_i)	(0x00710 + (4 * (_i))) /* 4 total */
>  #define IXGBE_PFMBIMR(_i)	(0x00720 + (4 * (_i))) /* 4 total */
> +#define IXGBE_PFVFLRE(_i)	((((_i) & 1) ? 0x001C0 : 0x00600))
> +#define IXGBE_PFVFLREC(_i)	(0x00700 + ((_i) * 4))
> +#define IXGBE_PFVFLRE_INDEX(_i)	((_i) >> 5)
> +#define IXGBE_PFVFLRE_SHIFT(_i)	((_i) % 32)
>  #define IXGBE_VFRE(_i)		(0x051E0 + ((_i) * 4))
>  #define IXGBE_VFTE(_i)		(0x08110 + ((_i) * 4))
>  #define IXGBE_VMECM(_i)		(0x08790 + ((_i) * 4))
> @@ -3949,6 +3955,7 @@ struct ixgbe_mac_operations {
>  	int32_t (*update_mc_addr_list)(struct ixgbe_hw *, uint8_t *, uint32_t,
>  				   ixgbe_mc_addr_itr, bool clear);
>  	int32_t (*update_xcast_mode)(struct ixgbe_hw *, int);
> +	int32_t (*get_link_state)(struct ixgbe_hw *hw, bool *link_state);
>  	int32_t (*enable_mc)(struct ixgbe_hw *);
>  	int32_t (*disable_mc)(struct ixgbe_hw *);
>  	int32_t (*clear_vfta)(struct ixgbe_hw *);
> @@ -3968,7 +3975,7 @@ struct ixgbe_mac_operations {
>  	/* Manageability interface */
>  	void (*disable_rx)(struct ixgbe_hw *hw);
>  	void (*enable_rx)(struct ixgbe_hw *hw);
> -  void (*stop_mac_link_on_d3)(struct ixgbe_hw *);
> +	void (*stop_mac_link_on_d3)(struct ixgbe_hw *);
>  	void (*set_source_address_pruning)(struct ixgbe_hw *, bool,
>  					   unsigned int);
>  	int32_t (*dmac_update_tcs)(struct ixgbe_hw *hw);
> @@ -4093,7 +4100,7 @@ struct ixgbe_phy_info {
>  };
>  
>  #define IXGBE_VFMAILBOX_SIZE	16 /* 16 32 bit words - 64 bytes */
> -#define IXGBE_ERR_MBX		-100
> +#define IXGBE_MAX_MULTICAST_ADDRESSES_VF  30
>  
>  #define IXGBE_VFMAILBOX		0x002FC
>  #define IXGBE_VFMBMEM		0x00200
> @@ -4115,22 +4122,25 @@ struct ixgbe_phy_info {
>  #define IXGBE_PFMAILBOX_PFU	0x00000008 /* PF owns the mailbox buffer */
>  #define IXGBE_PFMAILBOX_RVFU	0x00000010 /* Reset VFU - used when VF stuck */
>  
> -#define IXGBE_MBVFICR_VFREQ_MASK	0x0000FFFF /* bits for VF messages */
> -#define IXGBE_MBVFICR_VFREQ_VF1		0x00000001 /* bit for VF 1 message */
> -#define IXGBE_MBVFICR_VFACK_MASK	0xFFFF0000 /* bits for VF acks */
> -#define IXGBE_MBVFICR_VFACK_VF1		0x00010000 /* bit for VF 1 ack */
> +#define IXGBE_PFMBICR_VFREQ_MASK	0x0000FFFF /* bits for VF messages */
> +#define IXGBE_PFMBICR_VFREQ_VF1		0x00000001 /* bit for VF 1 message */
> +#define IXGBE_PFMBICR_VFACK_MASK	0xFFFF0000 /* bits for VF acks */
> +#define IXGBE_PFMBICR_VFACK_VF1		0x00010000 /* bit for VF 1 ack */
>  
>  
>  /* If it's a IXGBE_VF_* msg then it originates in the VF and is sent to the
>   * PF.  The reverse is TRUE if it is IXGBE_PF_*.
> - * Message ACK's are the value or'd with 0xF0000000
> + * Message results are the value or'd with 0xF0000000
>   */
> -#define IXGBE_VT_MSGTYPE_ACK	0x80000000 /* Messages below or'd with
> -					    * this are the ACK */
> -#define IXGBE_VT_MSGTYPE_NACK	0x40000000 /* Messages below or'd with
> -					    * this are the NACK */
> -#define IXGBE_VT_MSGTYPE_CTS	0x20000000 /* Indicates that VF is still
> -					    * clear to send requests */
> +#define IXGBE_VT_MSGTYPE_SUCCESS	0x80000000 /* Messages or'd with this
> +						    * have succeeded
> +						    */
> +#define IXGBE_VT_MSGTYPE_FAILURE	0x40000000 /* Messages or'd with this
> +						    * have failed
> +						    */
> +#define IXGBE_VT_MSGTYPE_CTS		0x20000000 /* Indicates that VF is still
> +						    * clear to send requests
> +						    */
>  #define IXGBE_VT_MSGINFO_SHIFT	16
>  /* bits 23:16 are used for extra info for certain messages */
>  #define IXGBE_VT_MSGINFO_MASK	(0xFF << IXGBE_VT_MSGINFO_SHIFT)
> @@ -4147,6 +4157,9 @@ enum ixgbe_pfvf_api_rev {
>  	ixgbe_mbox_api_11,	/* API version 1.1, linux/freebsd VF driver */
>  	ixgbe_mbox_api_12,	/* API version 1.2, linux/freebsd VF driver */
>  	ixgbe_mbox_api_13,	/* API version 1.3, linux/freebsd VF driver */
> +	/* API 1.4 is being used in the upstream for IPsec */
> +	ixgbe_mbox_api_14,	/* API version 1.4, linux/freebsd VF driver */
> +	ixgbe_mbox_api_15,	/* API version 1.5, linux/freebsd VF driver */
>  	/* This value should always be last */
>  	ixgbe_mbox_api_unknown,	/* indicates that API version is not known */
>  };
> @@ -4169,6 +4182,7 @@ enum ixgbe_pfvf_api_rev {
>  #define IXGBE_VF_GET_RETA      0x0a    /* VF request for RETA */
>  #define IXGBE_VF_GET_RSS_KEY	0x0b    /* get RSS key */
>  #define IXGBE_VF_UPDATE_XCAST_MODE	0x0c
> +#define IXGBE_VF_GET_LINK_STATE 0x10 /* get vf link state */
>  
>  /* mode choices for IXGBE_VF_UPDATE_XCAST_MODE */
>  enum ixgbevf_xcast_modes {
> @@ -4207,9 +4221,61 @@ enum ixgbevf_xcast_modes {
>  #define IXGBE_VF_MBX_INIT_TIMEOUT	2000 /* number of retries on mailbox */
>  #define IXGBE_VF_MBX_INIT_DELAY		500  /* microseconds between retries */
>  
> +#define IXGBE_VF_IRQ_CLEAR_MASK	7
> +#define IXGBE_VF_MAX_TX_QUEUES	8
> +#define IXGBE_VF_MAX_RX_QUEUES	8
> +/* DCB define */
> +#define IXGBE_VF_MAX_TRAFFIC_CLASS	8
> +
> +#define IXGBE_VFCTRL		0x00000
> +#define IXGBE_VFSTATUS		0x00008
> +#define IXGBE_VFLINKS		0x00010
> +#define IXGBE_VFFRTIMER		0x00048
> +#define IXGBE_VFRXMEMWRAP	0x03190
> +#define IXGBE_VTEICR		0x00100
> +#define IXGBE_VTEICS		0x00104
> +#define IXGBE_VTEIMS		0x00108
> +#define IXGBE_VTEIMC		0x0010C
> +#define IXGBE_VTEIAC		0x00110
> +#define IXGBE_VTEIAM		0x00114
> +#define IXGBE_VTEITR(x)		(0x00820 + (4 * (x)))
> +#define IXGBE_VTIVAR(x)		(0x00120 + (4 * (x)))
> +#define IXGBE_VTIVAR_MISC	0x00140
> +#define IXGBE_VTRSCINT(x)	(0x00180 + (4 * (x)))
> +/* define IXGBE_VFPBACL  still says TBD in EAS */
> +#define IXGBE_VFRDBAL(x)	(0x01000 + (0x40 * (x)))
> +#define IXGBE_VFRDBAH(x)	(0x01004 + (0x40 * (x)))
> +#define IXGBE_VFRDLEN(x)	(0x01008 + (0x40 * (x)))
> +#define IXGBE_VFRDH(x)		(0x01010 + (0x40 * (x)))
> +#define IXGBE_VFRDT(x)		(0x01018 + (0x40 * (x)))
> +#define IXGBE_VFRXDCTL(x)	(0x01028 + (0x40 * (x)))
> +#define IXGBE_VFSRRCTL(x)	(0x01014 + (0x40 * (x)))
> +#define IXGBE_VFRSCCTL(x)	(0x0102C + (0x40 * (x)))
> +#define IXGBE_VFPSRTYPE		0x00300
> +#define IXGBE_VFTDBAL(x)	(0x02000 + (0x40 * (x)))
> +#define IXGBE_VFTDBAH(x)	(0x02004 + (0x40 * (x)))
> +#define IXGBE_VFTDLEN(x)	(0x02008 + (0x40 * (x)))
> +#define IXGBE_VFTDH(x)		(0x02010 + (0x40 * (x)))
> +#define IXGBE_VFTDT(x)		(0x02018 + (0x40 * (x)))
> +#define IXGBE_VFTXDCTL(x)	(0x02028 + (0x40 * (x)))
> +#define IXGBE_VFTDWBAL(x)	(0x02038 + (0x40 * (x)))
> +#define IXGBE_VFTDWBAH(x)	(0x0203C + (0x40 * (x)))
> +#define IXGBE_VFDCA_RXCTRL(x)	(0x0100C + (0x40 * (x)))
> +#define IXGBE_VFDCA_TXCTRL(x)	(0x0200c + (0x40 * (x)))
> +#define IXGBE_VFGPRC		0x0101C
> +#define IXGBE_VFGPTC		0x0201C
> +#define IXGBE_VFGORC_LSB	0x01020
> +#define IXGBE_VFGORC_MSB	0x01024
> +#define IXGBE_VFGOTC_LSB	0x02020
> +#define IXGBE_VFGOTC_MSB	0x02024
> +#define IXGBE_VFMPRC		0x01034
> +#define IXGBE_VFMRQC		0x3000
> +#define IXGBE_VFRSSRK(x)	(0x3100 + ((x) * 4))
> +#define IXGBE_VFRETA(x)		(0x3200 + ((x) * 4))
>  
>  struct ixgbe_mbx_operations {
>  	void (*init_params)(struct ixgbe_hw *hw);
> +	void (*release)(struct ixgbe_hw *, uint16_t);
>  	int32_t  (*read)(struct ixgbe_hw *, uint32_t *, uint16_t,  uint16_t);
>  	int32_t  (*write)(struct ixgbe_hw *, uint32_t *, uint16_t, uint16_t);
>  	int32_t  (*read_posted)(struct ixgbe_hw *, uint32_t *, uint16_t,  uint16_t);
> @@ -4217,6 +4283,7 @@ struct ixgbe_mbx_operations {
>  	int32_t  (*check_for_msg)(struct ixgbe_hw *, uint16_t);
>  	int32_t  (*check_for_ack)(struct ixgbe_hw *, uint16_t);
>  	int32_t  (*check_for_rst)(struct ixgbe_hw *, uint16_t);
> +	int32_t  (*clear)(struct ixgbe_hw *, uint16_t);
>  };
>  
>  struct ixgbe_mbx_stats {
> @@ -4233,7 +4300,7 @@ struct ixgbe_mbx_info {
>  	struct ixgbe_mbx_stats stats;
>  	uint32_t timeout;
>  	uint32_t usec_delay;
> -	uint32_t v2p_mailbox;
> +	uint32_t vf_mailbox;
>  	uint16_t size;
>  };
>  
> @@ -4307,6 +4374,9 @@ struct ixgbe_hw {
>  #define IXGBE_ERR_FDIR_CMD_INCOMPLETE		-38
>  #define IXGBE_ERR_FW_RESP_INVALID		-39
>  #define IXGBE_ERR_TOKEN_RETRY			-40
> +#define IXGBE_ERR_MBX				-41
> +#define IXGBE_ERR_MBX_NOMSG			-42
> +#define IXGBE_ERR_TIMEOUT			-43
>  
>  #define IXGBE_NOT_IMPLEMENTED			0x7FFFFFFF
>  
> 
>