[dpdk-dev] [PATCH v2 05/51] net/mlx4: remove secondary process support

[Adrien Mazarguil <adrien.mazarguil@6wind.com>]

Current implementation is partial (Tx only), not convenient to use and
not of primary concern.

Remove this feature before refactoring the PMD.

Signed-off-by: Adrien Mazarguil <adrien.mazarguil@6wind.com>
---
 doc/guides/nics/features/mlx4.ini |   1 -
 doc/guides/nics/mlx4.rst          |   2 -
 drivers/net/mlx4/mlx4.c           | 349 +--------------------------------
 3 files changed, 8 insertions(+), 344 deletions(-)

diff --git a/doc/guides/nics/features/mlx4.ini b/doc/guides/nics/features/mlx4.ini
index 1d5f266..f6efd21 100644
--- a/doc/guides/nics/features/mlx4.ini
+++ b/doc/guides/nics/features/mlx4.ini
@@ -27,7 +27,6 @@ Inner L4 checksum    = Y
 Packet type parsing  = Y
 Basic stats          = Y
 Stats per queue      = Y
-Multiprocess aware   = Y
 Other kdrv           = Y
 Power8               = Y
 x86-32               = Y
diff --git a/doc/guides/nics/mlx4.rst b/doc/guides/nics/mlx4.rst
index 388aaf3..bba1034 100644
--- a/doc/guides/nics/mlx4.rst
+++ b/doc/guides/nics/mlx4.rst
@@ -88,7 +88,6 @@ Features
 - Scattered packets are supported for TX and RX.
 - Inner L3/L4 (IP, TCP and UDP) TX/RX checksum offloading and validation.
 - Outer L3 (IP) TX/RX checksum offloading and validation for VXLAN frames.
-- Secondary process TX is supported.
 - RX interrupts.
 
 Limitations
@@ -99,7 +98,6 @@ Limitations
 - RSS always includes L3 (IPv4/IPv6) and L4 (UDP/TCP). They cannot be
   dissociated.
 - Hardware counters are not implemented (they are software counters).
-- Secondary process RX is not supported.
 
 Configuration
 -------------
diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c
index cef024a..6f8d328 100644
--- a/drivers/net/mlx4/mlx4.c
+++ b/drivers/net/mlx4/mlx4.c
@@ -110,14 +110,6 @@ typedef union {
 	 (((val) & (from)) / ((from) / (to))) : \
 	 (((val) & (from)) * ((to) / (from))))
 
-/* Local storage for secondary process data. */
-struct mlx4_secondary_data {
-	struct rte_eth_dev_data data; /* Local device data. */
-	struct priv *primary_priv; /* Private structure from primary. */
-	struct rte_eth_dev_data *shared_dev_data; /* Shared device data. */
-	rte_spinlock_t lock; /* Port configuration lock. */
-} mlx4_secondary_data[RTE_MAX_ETHPORTS];
-
 /** Configuration structure for device arguments. */
 struct mlx4_conf {
 	struct {
@@ -145,38 +137,6 @@ static void
 priv_rx_intr_vec_disable(struct priv *priv);
 
 /**
- * Check if running as a secondary process.
- *
- * @return
- *   Nonzero if running as a secondary process.
- */
-static inline int
-mlx4_is_secondary(void)
-{
-	return rte_eal_process_type() != RTE_PROC_PRIMARY;
-}
-
-/**
- * Return private structure associated with an Ethernet device.
- *
- * @param dev
- *   Pointer to Ethernet device structure.
- *
- * @return
- *   Pointer to private structure.
- */
-static struct priv *
-mlx4_get_priv(struct rte_eth_dev *dev)
-{
-	struct mlx4_secondary_data *sd;
-
-	if (!mlx4_is_secondary())
-		return dev->data->dev_private;
-	sd = &mlx4_secondary_data[dev->data->port_id];
-	return sd->data.dev_private;
-}
-
-/**
  * Lock private structure to protect it from concurrent access in the
  * control path.
  *
@@ -734,8 +694,6 @@ mlx4_dev_configure(struct rte_eth_dev *dev)
 	struct priv *priv = dev->data->dev_private;
 	int ret;
 
-	if (mlx4_is_secondary())
-		return -E_RTE_SECONDARY;
 	priv_lock(priv);
 	ret = dev_configure(dev);
 	assert(ret >= 0);
@@ -746,157 +704,6 @@ mlx4_dev_configure(struct rte_eth_dev *dev)
 static uint16_t mlx4_tx_burst(void *, struct rte_mbuf **, uint16_t);
 static uint16_t removed_rx_burst(void *, struct rte_mbuf **, uint16_t);
 
-/**
- * Configure secondary process queues from a private data pointer (primary
- * or secondary) and update burst callbacks. Can take place only once.
- *
- * All queues must have been previously created by the primary process to
- * avoid undefined behavior.
- *
- * @param priv
- *   Private data pointer from either primary or secondary process.
- *
- * @return
- *   Private data pointer from secondary process, NULL in case of error.
- */
-static struct priv *
-mlx4_secondary_data_setup(struct priv *priv)
-{
-	unsigned int port_id = 0;
-	struct mlx4_secondary_data *sd;
-	void **tx_queues;
-	void **rx_queues;
-	unsigned int nb_tx_queues;
-	unsigned int nb_rx_queues;
-	unsigned int i;
-
-	/* priv must be valid at this point. */
-	assert(priv != NULL);
-	/* priv->dev must also be valid but may point to local memory from
-	 * another process, possibly with the same address and must not
-	 * be dereferenced yet. */
-	assert(priv->dev != NULL);
-	/* Determine port ID by finding out where priv comes from. */
-	while (1) {
-		sd = &mlx4_secondary_data[port_id];
-		rte_spinlock_lock(&sd->lock);
-		/* Primary process? */
-		if (sd->primary_priv == priv)
-			break;
-		/* Secondary process? */
-		if (sd->data.dev_private == priv)
-			break;
-		rte_spinlock_unlock(&sd->lock);
-		if (++port_id == RTE_DIM(mlx4_secondary_data))
-			port_id = 0;
-	}
-	/* Switch to secondary private structure. If private data has already
-	 * been updated by another thread, there is nothing else to do. */
-	priv = sd->data.dev_private;
-	if (priv->dev->data == &sd->data)
-		goto end;
-	/* Sanity checks. Secondary private structure is supposed to point
-	 * to local eth_dev, itself still pointing to the shared device data
-	 * structure allocated by the primary process. */
-	assert(sd->shared_dev_data != &sd->data);
-	assert(sd->data.nb_tx_queues == 0);
-	assert(sd->data.tx_queues == NULL);
-	assert(sd->data.nb_rx_queues == 0);
-	assert(sd->data.rx_queues == NULL);
-	assert(priv != sd->primary_priv);
-	assert(priv->dev->data == sd->shared_dev_data);
-	assert(priv->txqs_n == 0);
-	assert(priv->txqs == NULL);
-	assert(priv->rxqs_n == 0);
-	assert(priv->rxqs == NULL);
-	nb_tx_queues = sd->shared_dev_data->nb_tx_queues;
-	nb_rx_queues = sd->shared_dev_data->nb_rx_queues;
-	/* Allocate local storage for queues. */
-	tx_queues = rte_zmalloc("secondary ethdev->tx_queues",
-				sizeof(sd->data.tx_queues[0]) * nb_tx_queues,
-				RTE_CACHE_LINE_SIZE);
-	rx_queues = rte_zmalloc("secondary ethdev->rx_queues",
-				sizeof(sd->data.rx_queues[0]) * nb_rx_queues,
-				RTE_CACHE_LINE_SIZE);
-	if (tx_queues == NULL || rx_queues == NULL)
-		goto error;
-	/* Lock to prevent control operations during setup. */
-	priv_lock(priv);
-	/* TX queues. */
-	for (i = 0; i != nb_tx_queues; ++i) {
-		struct txq *primary_txq = (*sd->primary_priv->txqs)[i];
-		struct txq *txq;
-
-		if (primary_txq == NULL)
-			continue;
-		txq = rte_calloc_socket("TXQ", 1, sizeof(*txq), 0,
-					primary_txq->socket);
-		if (txq != NULL) {
-			if (txq_setup(priv->dev,
-				      txq,
-				      primary_txq->elts_n * MLX4_PMD_SGE_WR_N,
-				      primary_txq->socket,
-				      NULL) == 0) {
-				txq->stats.idx = primary_txq->stats.idx;
-				tx_queues[i] = txq;
-				continue;
-			}
-			rte_free(txq);
-		}
-		while (i) {
-			txq = tx_queues[--i];
-			txq_cleanup(txq);
-			rte_free(txq);
-		}
-		goto error;
-	}
-	/* RX queues. */
-	for (i = 0; i != nb_rx_queues; ++i) {
-		struct rxq *primary_rxq = (*sd->primary_priv->rxqs)[i];
-
-		if (primary_rxq == NULL)
-			continue;
-		/* Not supported yet. */
-		rx_queues[i] = NULL;
-	}
-	/* Update everything. */
-	priv->txqs = (void *)tx_queues;
-	priv->txqs_n = nb_tx_queues;
-	priv->rxqs = (void *)rx_queues;
-	priv->rxqs_n = nb_rx_queues;
-	sd->data.rx_queues = rx_queues;
-	sd->data.tx_queues = tx_queues;
-	sd->data.nb_rx_queues = nb_rx_queues;
-	sd->data.nb_tx_queues = nb_tx_queues;
-	sd->data.dev_link = sd->shared_dev_data->dev_link;
-	sd->data.mtu = sd->shared_dev_data->mtu;
-	memcpy(sd->data.rx_queue_state, sd->shared_dev_data->rx_queue_state,
-	       sizeof(sd->data.rx_queue_state));
-	memcpy(sd->data.tx_queue_state, sd->shared_dev_data->tx_queue_state,
-	       sizeof(sd->data.tx_queue_state));
-	sd->data.dev_flags = sd->shared_dev_data->dev_flags;
-	/* Use local data from now on. */
-	rte_mb();
-	priv->dev->data = &sd->data;
-	rte_mb();
-	priv->dev->tx_pkt_burst = mlx4_tx_burst;
-	priv->dev->rx_pkt_burst = removed_rx_burst;
-	priv_unlock(priv);
-end:
-	/* More sanity checks. */
-	assert(priv->dev->tx_pkt_burst == mlx4_tx_burst);
-	assert(priv->dev->rx_pkt_burst == removed_rx_burst);
-	assert(priv->dev->data == &sd->data);
-	rte_spinlock_unlock(&sd->lock);
-	return priv;
-error:
-	priv_unlock(priv);
-	rte_free(tx_queues);
-	rte_free(rx_queues);
-	rte_spinlock_unlock(&sd->lock);
-	return NULL;
-}
-
 /* TX queues handling. */
 
 /**
@@ -1704,46 +1511,6 @@ mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 }
 
 /**
- * DPDK callback for TX in secondary processes.
- *
- * This function configures all queues from primary process information
- * if necessary before reverting to the normal TX burst callback.
- *
- * @param dpdk_txq
- *   Generic pointer to TX queue structure.
- * @param[in] pkts
- *   Packets to transmit.
- * @param pkts_n
- *   Number of packets in array.
- *
- * @return
- *   Number of packets successfully transmitted (<= pkts_n).
- */
-static uint16_t
-mlx4_tx_burst_secondary_setup(void *dpdk_txq, struct rte_mbuf **pkts,
-			      uint16_t pkts_n)
-{
-	struct txq *txq = dpdk_txq;
-	struct priv *priv = mlx4_secondary_data_setup(txq->priv);
-	struct priv *primary_priv;
-	unsigned int index;
-
-	if (priv == NULL)
-		return 0;
-	primary_priv =
-		mlx4_secondary_data[priv->dev->data->port_id].primary_priv;
-	/* Look for queue index in both private structures. */
-	for (index = 0; index != priv->txqs_n; ++index)
-		if (((*primary_priv->txqs)[index] == txq) ||
-		    ((*priv->txqs)[index] == txq))
-			break;
-	if (index == priv->txqs_n)
-		return 0;
-	txq = (*priv->txqs)[index];
-	return priv->dev->tx_pkt_burst(txq, pkts, pkts_n);
-}
-
-/**
  * Configure a TX queue.
  *
  * @param dev
@@ -1764,7 +1531,7 @@ static int
 txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc,
 	  unsigned int socket, const struct rte_eth_txconf *conf)
 {
-	struct priv *priv = mlx4_get_priv(dev);
+	struct priv *priv = dev->data->dev_private;
 	struct txq tmpl = {
 		.priv = priv,
 		.socket = socket
@@ -1960,8 +1727,6 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 	struct txq *txq = (*priv->txqs)[idx];
 	int ret;
 
-	if (mlx4_is_secondary())
-		return -E_RTE_SECONDARY;
 	priv_lock(priv);
 	DEBUG("%p: configuring queue %u for %u descriptors",
 	      (void *)dev, idx, desc);
@@ -2017,8 +1782,6 @@ mlx4_tx_queue_release(void *dpdk_txq)
 	struct priv *priv;
 	unsigned int i;
 
-	if (mlx4_is_secondary())
-		return;
 	if (txq == NULL)
 		return;
 	priv = txq->priv;
@@ -3328,46 +3091,6 @@ mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 }
 
 /**
- * DPDK callback for RX in secondary processes.
- *
- * This function configures all queues from primary process information
- * if necessary before reverting to the normal RX burst callback.
- *
- * @param dpdk_rxq
- *   Generic pointer to RX queue structure.
- * @param[out] pkts
- *   Array to store received packets.
- * @param pkts_n
- *   Maximum number of packets in array.
- *
- * @return
- *   Number of packets successfully received (<= pkts_n).
- */
-static uint16_t
-mlx4_rx_burst_secondary_setup(void *dpdk_rxq, struct rte_mbuf **pkts,
-			      uint16_t pkts_n)
-{
-	struct rxq *rxq = dpdk_rxq;
-	struct priv *priv = mlx4_secondary_data_setup(rxq->priv);
-	struct priv *primary_priv;
-	unsigned int index;
-
-	if (priv == NULL)
-		return 0;
-	primary_priv =
-		mlx4_secondary_data[priv->dev->data->port_id].primary_priv;
-	/* Look for queue index in both private structures. */
-	for (index = 0; index != priv->rxqs_n; ++index)
-		if (((*primary_priv->rxqs)[index] == rxq) ||
-		    ((*priv->rxqs)[index] == rxq))
-			break;
-	if (index == priv->rxqs_n)
-		return 0;
-	rxq = (*priv->rxqs)[index];
-	return priv->dev->rx_pkt_burst(rxq, pkts, pkts_n);
-}
-
-/**
  * Allocate a Queue Pair.
  * Optionally setup inline receive if supported.
  *
@@ -3998,8 +3721,6 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 	int inactive = 0;
 	int ret;
 
-	if (mlx4_is_secondary())
-		return -E_RTE_SECONDARY;
 	priv_lock(priv);
 	DEBUG("%p: configuring queue %u for %u descriptors",
 	      (void *)dev, idx, desc);
@@ -4067,8 +3788,6 @@ mlx4_rx_queue_release(void *dpdk_rxq)
 	struct priv *priv;
 	unsigned int i;
 
-	if (mlx4_is_secondary())
-		return;
 	if (rxq == NULL)
 		return;
 	priv = rxq->priv;
@@ -4114,8 +3833,6 @@ mlx4_dev_start(struct rte_eth_dev *dev)
 	struct rxq *rxq;
 	int ret;
 
-	if (mlx4_is_secondary())
-		return -E_RTE_SECONDARY;
 	priv_lock(priv);
 	if (priv->started) {
 		priv_unlock(priv);
@@ -4206,8 +3923,6 @@ mlx4_dev_stop(struct rte_eth_dev *dev)
 	unsigned int r;
 	struct rxq *rxq;
 
-	if (mlx4_is_secondary())
-		return;
 	priv_lock(priv);
 	if (!priv->started) {
 		priv_unlock(priv);
@@ -4309,7 +4024,7 @@ priv_dev_link_interrupt_handler_uninstall(struct priv *, struct rte_eth_dev *);
 static void
 mlx4_dev_close(struct rte_eth_dev *dev)
 {
-	struct priv *priv = mlx4_get_priv(dev);
+	struct priv *priv = dev->data->dev_private;
 	void *tmp;
 	unsigned int i;
 
@@ -4462,7 +4177,7 @@ mlx4_set_link_up(struct rte_eth_dev *dev)
 static void
 mlx4_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
 {
-	struct priv *priv = mlx4_get_priv(dev);
+	struct priv *priv = dev->data->dev_private;
 	unsigned int max;
 	char ifname[IF_NAMESIZE];
 
@@ -4539,7 +4254,7 @@ mlx4_dev_supported_ptypes_get(struct rte_eth_dev *dev)
 static void
 mlx4_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 {
-	struct priv *priv = mlx4_get_priv(dev);
+	struct priv *priv = dev->data->dev_private;
 	struct rte_eth_stats tmp = {0};
 	unsigned int i;
 	unsigned int idx;
@@ -4604,7 +4319,7 @@ mlx4_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 static void
 mlx4_stats_reset(struct rte_eth_dev *dev)
 {
-	struct priv *priv = mlx4_get_priv(dev);
+	struct priv *priv = dev->data->dev_private;
 	unsigned int i;
 	unsigned int idx;
 
@@ -4644,8 +4359,6 @@ mlx4_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
 {
 	struct priv *priv = dev->data->dev_private;
 
-	if (mlx4_is_secondary())
-		return;
 	priv_lock(priv);
 	if (priv->isolated)
 		goto end;
@@ -4678,8 +4391,6 @@ mlx4_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 	struct priv *priv = dev->data->dev_private;
 	int re;
 
-	if (mlx4_is_secondary())
-		return -ENOTSUP;
 	(void)vmdq;
 	priv_lock(priv);
 	if (priv->isolated) {
@@ -4732,8 +4443,6 @@ mlx4_promiscuous_enable(struct rte_eth_dev *dev)
 	unsigned int i;
 	int ret;
 
-	if (mlx4_is_secondary())
-		return;
 	priv_lock(priv);
 	if (priv->isolated) {
 		DEBUG("%p: cannot enable promiscuous, "
@@ -4786,8 +4495,6 @@ mlx4_promiscuous_disable(struct rte_eth_dev *dev)
 	struct priv *priv = dev->data->dev_private;
 	unsigned int i;
 
-	if (mlx4_is_secondary())
-		return;
 	priv_lock(priv);
 	if (!priv->promisc || priv->isolated) {
 		priv_unlock(priv);
@@ -4818,8 +4525,6 @@ mlx4_allmulticast_enable(struct rte_eth_dev *dev)
 	unsigned int i;
 	int ret;
 
-	if (mlx4_is_secondary())
-		return;
 	priv_lock(priv);
 	if (priv->isolated) {
 		DEBUG("%p: cannot enable allmulticast, "
@@ -4872,8 +4577,6 @@ mlx4_allmulticast_disable(struct rte_eth_dev *dev)
 	struct priv *priv = dev->data->dev_private;
 	unsigned int i;
 
-	if (mlx4_is_secondary())
-		return;
 	priv_lock(priv);
 	if (!priv->allmulti || priv->isolated) {
 		priv_unlock(priv);
@@ -4902,7 +4605,7 @@ mlx4_allmulticast_disable(struct rte_eth_dev *dev)
 static int
 mlx4_link_update(struct rte_eth_dev *dev, int wait_to_complete)
 {
-	const struct priv *priv = mlx4_get_priv(dev);
+	const struct priv *priv = dev->data->dev_private;
 	struct ethtool_cmd edata = {
 		.cmd = ETHTOOL_GSET
 	};
@@ -4976,8 +4679,6 @@ mlx4_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 	uint16_t (*rx_func)(void *, struct rte_mbuf **, uint16_t) =
 		mlx4_rx_burst;
 
-	if (mlx4_is_secondary())
-		return -E_RTE_SECONDARY;
 	priv_lock(priv);
 	/* Set kernel interface MTU first. */
 	if (priv_set_mtu(priv, mtu)) {
@@ -5059,8 +4760,6 @@ mlx4_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 	};
 	int ret;
 
-	if (mlx4_is_secondary())
-		return -E_RTE_SECONDARY;
 	ifr.ifr_data = (void *)&ethpause;
 	priv_lock(priv);
 	if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) {
@@ -5109,8 +4808,6 @@ mlx4_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 	};
 	int ret;
 
-	if (mlx4_is_secondary())
-		return -E_RTE_SECONDARY;
 	ifr.ifr_data = (void *)&ethpause;
 	ethpause.autoneg = fc_conf->autoneg;
 	if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
@@ -5250,8 +4947,6 @@ mlx4_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 	struct priv *priv = dev->data->dev_private;
 	int ret;
 
-	if (mlx4_is_secondary())
-		return -E_RTE_SECONDARY;
 	priv_lock(priv);
 	if (priv->isolated) {
 		DEBUG("%p: cannot set vlan filter, "
@@ -6269,36 +5964,8 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 			goto port_error;
 		}
 
-		/* Secondary processes have to use local storage for their
-		 * private data as well as a copy of eth_dev->data, but this
-		 * pointer must not be modified before burst functions are
-		 * actually called. */
-		if (mlx4_is_secondary()) {
-			struct mlx4_secondary_data *sd =
-				&mlx4_secondary_data[eth_dev->data->port_id];
-
-			sd->primary_priv = eth_dev->data->dev_private;
-			if (sd->primary_priv == NULL) {
-				ERROR("no private data for port %u",
-				      eth_dev->data->port_id);
-				err = EINVAL;
-				goto port_error;
-			}
-			sd->shared_dev_data = eth_dev->data;
-			rte_spinlock_init(&sd->lock);
-			memcpy(sd->data.name, sd->shared_dev_data->name,
-			       sizeof(sd->data.name));
-			sd->data.dev_private = priv;
-			sd->data.rx_mbuf_alloc_failed = 0;
-			sd->data.mtu = ETHER_MTU;
-			sd->data.port_id = sd->shared_dev_data->port_id;
-			sd->data.mac_addrs = priv->mac;
-			eth_dev->tx_pkt_burst = mlx4_tx_burst_secondary_setup;
-			eth_dev->rx_pkt_burst = mlx4_rx_burst_secondary_setup;
-		} else {
-			eth_dev->data->dev_private = priv;
-			eth_dev->data->mac_addrs = priv->mac;
-		}
+		eth_dev->data->dev_private = priv;
+		eth_dev->data->mac_addrs = priv->mac;
 		eth_dev->device = &pci_dev->device;
 
 		rte_eth_copy_pci_info(eth_dev, pci_dev);
-- 
2.1.4