Re: [dpdk-dev] [PATCH v5 13/24] net/ngbe: support link update

[Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>]

On 6/2/21 12:40 PM, Jiawen Wu wrote:
> Register to handle device interrupt.
> 
> Signed-off-by: Jiawen Wu <jiawenwu@trustnetic.com>
> ---
>   doc/guides/nics/features/ngbe.ini  |   2 +
>   doc/guides/nics/ngbe.rst           |   5 +
>   drivers/net/ngbe/base/ngbe_dummy.h |   6 +
>   drivers/net/ngbe/base/ngbe_type.h  |  11 +
>   drivers/net/ngbe/ngbe_ethdev.c     | 364 +++++++++++++++++++++++++++++
>   drivers/net/ngbe/ngbe_ethdev.h     |  28 +++
>   6 files changed, 416 insertions(+)
> 
> diff --git a/doc/guides/nics/features/ngbe.ini b/doc/guides/nics/features/ngbe.ini
> index ca03a255de..291a542a42 100644
> --- a/doc/guides/nics/features/ngbe.ini
> +++ b/doc/guides/nics/features/ngbe.ini
> @@ -5,6 +5,8 @@
>   ;
>   [Features]
>   Speed capabilities   = Y
> +Link status          = Y
> +Link status event    = Y
>   Multiprocess aware   = Y
>   Linux                = Y
>   ARMv8                = Y
> diff --git a/doc/guides/nics/ngbe.rst b/doc/guides/nics/ngbe.rst
> index c274a15aab..de2ef65664 100644
> --- a/doc/guides/nics/ngbe.rst
> +++ b/doc/guides/nics/ngbe.rst
> @@ -7,6 +7,11 @@ NGBE Poll Mode Driver
>   The NGBE PMD (librte_pmd_ngbe) provides poll mode driver support
>   for Wangxun 1 Gigabit Ethernet NICs.
>   

Two empty lines before the section.

> +Features
> +--------
> +
> +- Link state information
> +
>   Prerequisites
>   -------------
>   
> diff --git a/drivers/net/ngbe/base/ngbe_dummy.h b/drivers/net/ngbe/base/ngbe_dummy.h
> index 8462d6d1cb..4273e5af36 100644
> --- a/drivers/net/ngbe/base/ngbe_dummy.h
> +++ b/drivers/net/ngbe/base/ngbe_dummy.h
> @@ -64,6 +64,11 @@ static inline void ngbe_mac_release_swfw_sync_dummy(struct ngbe_hw *TUP0,
>   					u32 TUP1)
>   {
>   }
> +static inline s32 ngbe_mac_check_link_dummy(struct ngbe_hw *TUP0, u32 *TUP1,
> +					bool *TUP3, bool TUP4)
> +{
> +	return NGBE_ERR_OPS_DUMMY;
> +}
>   static inline s32 ngbe_mac_set_rar_dummy(struct ngbe_hw *TUP0, u32 TUP1,
>   					u8 *TUP2, u32 TUP3, u32 TUP4)
>   {
> @@ -135,6 +140,7 @@ static inline void ngbe_init_ops_dummy(struct ngbe_hw *hw)
>   	hw->mac.get_mac_addr = ngbe_mac_get_mac_addr_dummy;
>   	hw->mac.acquire_swfw_sync = ngbe_mac_acquire_swfw_sync_dummy;
>   	hw->mac.release_swfw_sync = ngbe_mac_release_swfw_sync_dummy;
> +	hw->mac.check_link = ngbe_mac_check_link_dummy;
>   	hw->mac.set_rar = ngbe_mac_set_rar_dummy;
>   	hw->mac.clear_rar = ngbe_mac_clear_rar_dummy;
>   	hw->mac.set_vmdq = ngbe_mac_set_vmdq_dummy;
> diff --git a/drivers/net/ngbe/base/ngbe_type.h b/drivers/net/ngbe/base/ngbe_type.h
> index 5add9ec2a3..d05d2ff28a 100644
> --- a/drivers/net/ngbe/base/ngbe_type.h
> +++ b/drivers/net/ngbe/base/ngbe_type.h
> @@ -96,6 +96,8 @@ struct ngbe_mac_info {
>   	s32 (*acquire_swfw_sync)(struct ngbe_hw *hw, u32 mask);
>   	void (*release_swfw_sync)(struct ngbe_hw *hw, u32 mask);
>   
> +	s32 (*check_link)(struct ngbe_hw *hw, u32 *speed,
> +			       bool *link_up, bool link_up_wait_to_complete);
>   	/* RAR */
>   	s32 (*set_rar)(struct ngbe_hw *hw, u32 index, u8 *addr, u32 vmdq,
>   			  u32 enable_addr);
> @@ -116,6 +118,7 @@ struct ngbe_mac_info {
>   	u32 num_rar_entries;
>   	u32 max_tx_queues;
>   	u32 max_rx_queues;
> +	bool get_link_status;
>   	struct ngbe_thermal_sensor_data  thermal_sensor_data;
>   	bool set_lben;
>   };
> @@ -141,6 +144,14 @@ struct ngbe_phy_info {
>   	bool reset_disable;
>   };
>   
> +enum ngbe_isb_idx {
> +	NGBE_ISB_HEADER,
> +	NGBE_ISB_MISC,
> +	NGBE_ISB_VEC0,
> +	NGBE_ISB_VEC1,
> +	NGBE_ISB_MAX
> +};
> +
>   struct ngbe_hw {
>   	void IOMEM *hw_addr;
>   	void *back;
> diff --git a/drivers/net/ngbe/ngbe_ethdev.c b/drivers/net/ngbe/ngbe_ethdev.c
> index 07df677b64..97b6de3aa4 100644
> --- a/drivers/net/ngbe/ngbe_ethdev.c
> +++ b/drivers/net/ngbe/ngbe_ethdev.c
> @@ -6,6 +6,8 @@
>   #include <rte_common.h>
>   #include <ethdev_pci.h>
>   
> +#include <rte_alarm.h>
> +
>   #include "ngbe_logs.h"
>   #include "base/ngbe.h"
>   #include "ngbe_ethdev.h"
> @@ -13,6 +15,9 @@
>   
>   static int ngbe_dev_close(struct rte_eth_dev *dev);
>   
> +static void ngbe_dev_interrupt_handler(void *param);
> +static void ngbe_dev_interrupt_delayed_handler(void *param);
> +
>   /*
>    * The set of PCI devices this driver supports
>    */
> @@ -47,6 +52,26 @@ static const struct rte_eth_desc_lim tx_desc_lim = {
>   };
>   
>   static const struct eth_dev_ops ngbe_eth_dev_ops;
> +static inline void
> +ngbe_enable_intr(struct rte_eth_dev *dev)
> +{
> +	struct ngbe_interrupt *intr = NGBE_DEV_INTR(dev);
> +	struct ngbe_hw *hw = NGBE_DEV_HW(dev);
> +
> +	wr32(hw, NGBE_IENMISC, intr->mask_misc);
> +	wr32(hw, NGBE_IMC(0), intr->mask & BIT_MASK32);
> +	ngbe_flush(hw);
> +}
> +
> +static void
> +ngbe_disable_intr(struct ngbe_hw *hw)
> +{
> +	PMD_INIT_FUNC_TRACE();
> +
> +	wr32(hw, NGBE_IMS(0), NGBE_IMS_MASK);
> +	ngbe_flush(hw);
> +}
> +
>   
>   /*
>    * Ensure that all locks are released before first NVM or PHY access
> @@ -76,7 +101,9 @@ eth_ngbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params __rte_unused)
>   {
>   	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
>   	struct ngbe_hw *hw = NGBE_DEV_HW(eth_dev);
> +	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
>   	const struct rte_memzone *mz;
> +	uint32_t ctrl_ext;
>   	int err;
>   
>   	PMD_INIT_FUNC_TRACE();
> @@ -135,6 +162,9 @@ eth_ngbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params __rte_unused)
>   		return -EIO;
>   	}
>   
> +	/* disable interrupt */
> +	ngbe_disable_intr(hw);
> +
>   	/* Allocate memory for storing MAC addresses */
>   	eth_dev->data->mac_addrs = rte_zmalloc("ngbe", RTE_ETHER_ADDR_LEN *
>   					       hw->mac.num_rar_entries, 0);
> @@ -160,6 +190,22 @@ eth_ngbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params __rte_unused)
>   		return -ENOMEM;
>   	}
>   
> +	ctrl_ext = rd32(hw, NGBE_PORTCTL);
> +	/* let hardware know driver is loaded */
> +	ctrl_ext |= NGBE_PORTCTL_DRVLOAD;
> +	wr32(hw, NGBE_PORTCTL, ctrl_ext);
> +	ngbe_flush(hw);
> +
> +	rte_intr_callback_register(intr_handle,
> +				   ngbe_dev_interrupt_handler, eth_dev);
> +
> +	/* enable uio/vfio intr/eventfd mapping */
> +	rte_intr_enable(intr_handle);
> +
> +	/* enable support intr */
> +	ngbe_enable_intr(eth_dev);
> +
> +
>   	return 0;
>   }
>   
> @@ -212,6 +258,19 @@ static struct rte_pci_driver rte_ngbe_pmd = {
>   	.remove = eth_ngbe_pci_remove,
>   };
>   
> +static int
> +ngbe_dev_configure(struct rte_eth_dev *dev)
> +{
> +	struct ngbe_interrupt *intr = NGBE_DEV_INTR(dev);
> +
> +	PMD_INIT_FUNC_TRACE();
> +
> +	/* set flag to update link status after init */
> +	intr->flags |= NGBE_FLAG_NEED_LINK_UPDATE;
> +
> +	return 0;
> +}
> +
>   /*
>    * Reset and stop device.
>    */
> @@ -288,8 +347,313 @@ ngbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
>   	return 0;
>   }
>   
> +/* return 0 means link status changed, -1 means not changed */
> +int
> +ngbe_dev_link_update_share(struct rte_eth_dev *dev,
> +			    int wait_to_complete)
> +{
> +	struct ngbe_hw *hw = NGBE_DEV_HW(dev);
> +	struct rte_eth_link link;
> +	u32 link_speed = NGBE_LINK_SPEED_UNKNOWN;
> +	u32 lan_speed = 0;
> +	struct ngbe_interrupt *intr = NGBE_DEV_INTR(dev);
> +	bool link_up;
> +	int err;
> +	int wait = 1;
> +
> +	memset(&link, 0, sizeof(link));
> +	link.link_status = ETH_LINK_DOWN;
> +	link.link_speed = ETH_SPEED_NUM_NONE;
> +	link.link_duplex = ETH_LINK_HALF_DUPLEX;
> +	link.link_autoneg = ETH_LINK_AUTONEG;
> +
> +	hw->mac.get_link_status = true;
> +
> +	if (intr->flags & NGBE_FLAG_NEED_LINK_CONFIG)
> +		return rte_eth_linkstatus_set(dev, &link);
> +
> +	/* check if it needs to wait to complete, if lsc interrupt is enabled */
> +	if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0)
> +		wait = 0;
> +
> +	err = hw->mac.check_link(hw, &link_speed, &link_up, wait);
> +
> +	if (err != 0) {
> +		link.link_speed = ETH_SPEED_NUM_100M;
> +		link.link_duplex = ETH_LINK_FULL_DUPLEX;
> +		return rte_eth_linkstatus_set(dev, &link);
> +	}
> +
> +	if (link_up == 0)

bool should not be compared with 0, it should be !link_up here

> +		return rte_eth_linkstatus_set(dev, &link);
> +
> +	intr->flags &= ~NGBE_FLAG_NEED_LINK_CONFIG;
> +	link.link_status = ETH_LINK_UP;
> +	link.link_duplex = ETH_LINK_FULL_DUPLEX;
> +
> +	switch (link_speed) {
> +	default:
> +	case NGBE_LINK_SPEED_UNKNOWN:
> +		link.link_duplex = ETH_LINK_FULL_DUPLEX;
> +		link.link_speed = ETH_SPEED_NUM_100M;
> +		break;
> +
> +	case NGBE_LINK_SPEED_10M_FULL:
> +		link.link_speed = ETH_SPEED_NUM_10M;
> +		lan_speed = 0;
> +		break;
> +
> +	case NGBE_LINK_SPEED_100M_FULL:
> +		link.link_speed = ETH_SPEED_NUM_100M;
> +		lan_speed = 1;
> +		break;
> +
> +	case NGBE_LINK_SPEED_1GB_FULL:
> +		link.link_speed = ETH_SPEED_NUM_1G;
> +		lan_speed = 2;
> +		break;
> +
> +	case NGBE_LINK_SPEED_2_5GB_FULL:
> +		link.link_speed = ETH_SPEED_NUM_2_5G;
> +		break;
> +
> +	case NGBE_LINK_SPEED_5GB_FULL:
> +		link.link_speed = ETH_SPEED_NUM_5G;
> +		break;
> +
> +	case NGBE_LINK_SPEED_10GB_FULL:
> +		link.link_speed = ETH_SPEED_NUM_10G;
> +		break;

It looks like it does not match speed_capa reported in
the dev_info path.

> +	}
> +
> +	if (hw->is_pf) {
> +		wr32m(hw, NGBE_LAN_SPEED, NGBE_LAN_SPEED_MASK, lan_speed);
> +		if (link_speed & (NGBE_LINK_SPEED_1GB_FULL |
> +			NGBE_LINK_SPEED_100M_FULL | NGBE_LINK_SPEED_10M_FULL)) {
> +			wr32m(hw, NGBE_MACTXCFG, NGBE_MACTXCFG_SPEED_MASK,
> +				NGBE_MACTXCFG_SPEED_1G | NGBE_MACTXCFG_TE);
> +		}
> +	}
> +
> +	return rte_eth_linkstatus_set(dev, &link);
> +}
> +
> +static int
> +ngbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
> +{
> +	return ngbe_dev_link_update_share(dev, wait_to_complete);
> +}
> +
> +/*
> + * It reads ICR and sets flag for the link_update.
> + *
> + * @param dev
> + *  Pointer to struct rte_eth_dev.
> + *
> + * @return
> + *  - On success, zero.
> + *  - On failure, a negative value.
> + */
> +static int
> +ngbe_dev_interrupt_get_status(struct rte_eth_dev *dev)
> +{
> +	uint32_t eicr;
> +	struct ngbe_hw *hw = NGBE_DEV_HW(dev);
> +	struct ngbe_interrupt *intr = NGBE_DEV_INTR(dev);
> +
> +	/* clear all cause mask */
> +	ngbe_disable_intr(hw);
> +
> +	/* read-on-clear nic registers here */
> +	eicr = ((u32 *)hw->isb_mem)[NGBE_ISB_MISC];
> +	PMD_DRV_LOG(DEBUG, "eicr %x", eicr);
> +
> +	intr->flags = 0;
> +
> +	/* set flag for async link update */
> +	if (eicr & NGBE_ICRMISC_PHY)
> +		intr->flags |= NGBE_FLAG_NEED_LINK_UPDATE;
> +
> +	if (eicr & NGBE_ICRMISC_VFMBX)
> +		intr->flags |= NGBE_FLAG_MAILBOX;
> +
> +	if (eicr & NGBE_ICRMISC_LNKSEC)
> +		intr->flags |= NGBE_FLAG_MACSEC;
> +
> +	if (eicr & NGBE_ICRMISC_GPIO)
> +		intr->flags |= NGBE_FLAG_NEED_LINK_UPDATE;
> +
> +	return 0;
> +}
> +
> +/**
> + * It gets and then prints the link status.
> + *
> + * @param dev
> + *  Pointer to struct rte_eth_dev.
> + *
> + * @return
> + *  - On success, zero.
> + *  - On failure, a negative value.
> + */
> +static void
> +ngbe_dev_link_status_print(struct rte_eth_dev *dev)
> +{
> +	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
> +	struct rte_eth_link link;
> +
> +	rte_eth_linkstatus_get(dev, &link);
> +
> +	if (link.link_status) {

Compare with ETH_LINK_UP

> +		PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
> +					(int)(dev->data->port_id),
> +					(unsigned int)link.link_speed,
> +			link.link_duplex == ETH_LINK_FULL_DUPLEX ?
> +					"full-duplex" : "half-duplex");
> +	} else {
> +		PMD_INIT_LOG(INFO, " Port %d: Link Down",
> +				(int)(dev->data->port_id));
> +	}
> +	PMD_INIT_LOG(DEBUG, "PCI Address: " PCI_PRI_FMT,
> +				pci_dev->addr.domain,
> +				pci_dev->addr.bus,
> +				pci_dev->addr.devid,
> +				pci_dev->addr.function);
> +}
> +
> +/*
> + * It executes link_update after knowing an interrupt occurred.
> + *
> + * @param dev
> + *  Pointer to struct rte_eth_dev.
> + *
> + * @return
> + *  - On success, zero.
> + *  - On failure, a negative value.
> + */
> +static int
> +ngbe_dev_interrupt_action(struct rte_eth_dev *dev)
> +{
> +	struct ngbe_interrupt *intr = NGBE_DEV_INTR(dev);
> +	int64_t timeout;
> +
> +	PMD_DRV_LOG(DEBUG, "intr action type %d", intr->flags);
> +
> +	if (intr->flags & NGBE_FLAG_NEED_LINK_UPDATE) {
> +		struct rte_eth_link link;
> +
> +		/*get the link status before link update, for predicting later*/
> +		rte_eth_linkstatus_get(dev, &link);
> +
> +		ngbe_dev_link_update(dev, 0);
> +
> +		/* likely to up */
> +		if (!link.link_status)

Compare with ETH_LINK_UP

> +			/* handle it 1 sec later, wait it being stable */
> +			timeout = NGBE_LINK_UP_CHECK_TIMEOUT;
> +		/* likely to down */
> +		else
> +			/* handle it 4 sec later, wait it being stable */
> +			timeout = NGBE_LINK_DOWN_CHECK_TIMEOUT;
> +
> +		ngbe_dev_link_status_print(dev);
> +		if (rte_eal_alarm_set(timeout * 1000,
> +				      ngbe_dev_interrupt_delayed_handler,
> +				      (void *)dev) < 0) {
> +			PMD_DRV_LOG(ERR, "Error setting alarm");
> +		} else {
> +			/* remember original mask */
> +			intr->mask_misc_orig = intr->mask_misc;
> +			/* only disable lsc interrupt */
> +			intr->mask_misc &= ~NGBE_ICRMISC_PHY;
> +
> +			intr->mask_orig = intr->mask;
> +			/* only disable all misc interrupts */
> +			intr->mask &= ~(1ULL << NGBE_MISC_VEC_ID);
> +		}
> +	}
> +
> +	PMD_DRV_LOG(DEBUG, "enable intr immediately");
> +	ngbe_enable_intr(dev);
> +
> +	return 0;
> +}
> +
> +/**
> + * Interrupt handler which shall be registered for alarm callback for delayed
> + * handling specific interrupt to wait for the stable nic state. As the
> + * NIC interrupt state is not stable for ngbe after link is just down,
> + * it needs to wait 4 seconds to get the stable status.
> + *
> + * @param handle
> + *  Pointer to interrupt handle.
> + * @param param
> + *  The address of parameter (struct rte_eth_dev *) registered before.
> + *
> + * @return
> + *  void
> + */
> +static void
> +ngbe_dev_interrupt_delayed_handler(void *param)
> +{
> +	struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
> +	struct ngbe_interrupt *intr = NGBE_DEV_INTR(dev);
> +	struct ngbe_hw *hw = NGBE_DEV_HW(dev);
> +	uint32_t eicr;
> +
> +	ngbe_disable_intr(hw);
> +
> +	eicr = ((u32 *)hw->isb_mem)[NGBE_ISB_MISC];
> +
> +	if (intr->flags & NGBE_FLAG_NEED_LINK_UPDATE) {
> +		ngbe_dev_link_update(dev, 0);
> +		intr->flags &= ~NGBE_FLAG_NEED_LINK_UPDATE;
> +		ngbe_dev_link_status_print(dev);
> +		rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
> +					      NULL);
> +	}
> +
> +	if (intr->flags & NGBE_FLAG_MACSEC) {
> +		rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_MACSEC,
> +					      NULL);
> +		intr->flags &= ~NGBE_FLAG_MACSEC;
> +	}
> +
> +	/* restore original mask */
> +	intr->mask_misc = intr->mask_misc_orig;
> +	intr->mask_misc_orig = 0;
> +	intr->mask = intr->mask_orig;
> +	intr->mask_orig = 0;
> +
> +	PMD_DRV_LOG(DEBUG, "enable intr in delayed handler S[%08x]", eicr);
> +	ngbe_enable_intr(dev);
> +}
> +
> +/**
> + * Interrupt handler triggered by NIC  for handling
> + * specific interrupt.
> + *
> + * @param handle
> + *  Pointer to interrupt handle.
> + * @param param
> + *  The address of parameter (struct rte_eth_dev *) registered before.
> + *
> + * @return
> + *  void
> + */
> +static void
> +ngbe_dev_interrupt_handler(void *param)
> +{
> +	struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
> +
> +	ngbe_dev_interrupt_get_status(dev);
> +	ngbe_dev_interrupt_action(dev);
> +}
> +
>   static const struct eth_dev_ops ngbe_eth_dev_ops = {
> +	.dev_configure              = ngbe_dev_configure,
>   	.dev_infos_get              = ngbe_dev_info_get,
> +	.link_update                = ngbe_dev_link_update,
>   };
>   
>   RTE_PMD_REGISTER_PCI(net_ngbe, rte_ngbe_pmd);
> diff --git a/drivers/net/ngbe/ngbe_ethdev.h b/drivers/net/ngbe/ngbe_ethdev.h
> index b4e2000dd3..10c23c41d1 100644
> --- a/drivers/net/ngbe/ngbe_ethdev.h
> +++ b/drivers/net/ngbe/ngbe_ethdev.h
> @@ -6,6 +6,13 @@
>   #ifndef _NGBE_ETHDEV_H_
>   #define _NGBE_ETHDEV_H_
>   
> +/* need update link, bit flag */
> +#define NGBE_FLAG_NEED_LINK_UPDATE (uint32_t)(1 << 0)
> +#define NGBE_FLAG_MAILBOX          (uint32_t)(1 << 1)
> +#define NGBE_FLAG_PHY_INTERRUPT    (uint32_t)(1 << 2)
> +#define NGBE_FLAG_MACSEC           (uint32_t)(1 << 3)
> +#define NGBE_FLAG_NEED_LINK_CONFIG (uint32_t)(1 << 4)
> +
>   #define NGBE_HKEY_MAX_INDEX 10
>   
>   #define NGBE_RSS_OFFLOAD_ALL ( \
> @@ -19,11 +26,23 @@
>   	ETH_RSS_IPV6_TCP_EX | \
>   	ETH_RSS_IPV6_UDP_EX)
>   
> +#define NGBE_MISC_VEC_ID               RTE_INTR_VEC_ZERO_OFFSET
> +
> +/* structure for interrupt relative data */
> +struct ngbe_interrupt {
> +	uint32_t flags;
> +	uint32_t mask_misc;
> +	uint32_t mask_misc_orig; /* save mask during delayed handler */
> +	uint64_t mask;
> +	uint64_t mask_orig; /* save mask during delayed handler */
> +};
> +
>   /*
>    * Structure to store private data for each driver instance (for each port).
>    */
>   struct ngbe_adapter {
>   	struct ngbe_hw             hw;
> +	struct ngbe_interrupt      intr;
>   };
>   
>   #define NGBE_DEV_ADAPTER(dev) \
> @@ -32,6 +51,15 @@ struct ngbe_adapter {
>   #define NGBE_DEV_HW(dev) \
>   	(&((struct ngbe_adapter *)(dev)->data->dev_private)->hw)
>   
> +#define NGBE_DEV_INTR(dev) \
> +	(&((struct ngbe_adapter *)(dev)->data->dev_private)->intr)
> +

Above should be a static inline function.

> +int
> +ngbe_dev_link_update_share(struct rte_eth_dev *dev,
> +		int wait_to_complete);
> +
> +#define NGBE_LINK_DOWN_CHECK_TIMEOUT 4000 /* ms */
> +#define NGBE_LINK_UP_CHECK_TIMEOUT   1000 /* ms */
>   #define NGBE_VMDQ_NUM_UC_MAC         4096 /* Maximum nb. of UC MAC addr. */
>   
>   /*
>