[dpdk-dev] [PATCH v3 0/2] ADD mode 5(tlb) to link bonding pmd

[dpdk-dev] [PATCH v3 0/2] ADD mode 5(tlb) to link bonding pmd

[Daniel Mrzyglod]

This mode provides an adaptive transmit load balancing.
It dynamically changes the transmitting slave, according to the computed load.
Statistics are collected in 100ms intervals and scheduled every 10ms.

Daniel Mrzyglod (2):
  This patch add support of mode 5 to link bonding pmd
  Unit tests for Mode 5 of Bonding Transmit Load balancing.

 app/test/test_link_bonding.c               | 499 ++++++++++++++++++++++++++++-
 app/test/virtual_pmd.c                     |   6 +-
 lib/librte_pmd_bond/rte_eth_bond.h         |  11 +
 lib/librte_pmd_bond/rte_eth_bond_args.c    |   1 +
 lib/librte_pmd_bond/rte_eth_bond_pmd.c     | 160 ++++++++-
 lib/librte_pmd_bond/rte_eth_bond_private.h |   2 +-
 6 files changed, 673 insertions(+), 6 deletions(-)

-- 
2.1.0

[dpdk-dev] [PATCH v3 1/2] This patch add support of mode 5 to link bonding pmd

[Daniel Mrzyglod]

v3 change:
Rebase patch version to HEAD of orgin/master.
Unit tests moved to the separate patch v3 2/2.

v2 change:
Add Unit Tests
Modification that updates obytes structure in virtualpmd driver.
change internals->slaves[i].last_obytes to have proper values.
Update codebase to Declan's patches.

v1 change:
Add support for mode 5 (Transmit load balancing) into pmd driver

This mode provides an adaptive transmit load balancing. 
It dynamically changes the transmitting slave, according to the computed load. 
Statistics are collected in 100ms intervals and scheduled every 10ms.

Signed-off-by: Daniel Mrzyglod <danielx.t.mrzyglod@intel.com>
---
 lib/librte_pmd_bond/rte_eth_bond.h         |  11 ++
 lib/librte_pmd_bond/rte_eth_bond_args.c    |   1 +
 lib/librte_pmd_bond/rte_eth_bond_pmd.c     | 160 ++++++++++++++++++++++++++++-
 lib/librte_pmd_bond/rte_eth_bond_private.h |   2 +-
 4 files changed, 171 insertions(+), 3 deletions(-)

diff --git a/lib/librte_pmd_bond/rte_eth_bond.h b/lib/librte_pmd_bond/rte_eth_bond.h
index 085500b..29b9a89 100644
--- a/lib/librte_pmd_bond/rte_eth_bond.h
+++ b/lib/librte_pmd_bond/rte_eth_bond.h
@@ -77,6 +77,17 @@ extern "C" {
  * In this mode all transmitted packets will be transmitted on all available
  * active slaves of the bonded. */
 #endif
+#define BONDING_MODE_ADAPTIVE_TRANSMIT_LOAD_BALANCING	(5)
+/**< Adaptive TLB (Mode 5)
+ * Adaptive transmit load balancing: channel bonding that
+ * does not require any special switch support.  The
+ * outgoing traffic is distributed according to the
+ * current load (computed relative to the speed) on each
+ * slave.  Incoming traffic is received by the current
+ * slave.  If the receiving slave fails, another slave
+ * takes over the MAC address of the failed receiving
+ * slave.*/
+
 /* Balance Mode Transmit Policies */
 #define BALANCE_XMIT_POLICY_LAYER2		(0)
 /**< Layer 2 (Ethernet MAC) */
diff --git a/lib/librte_pmd_bond/rte_eth_bond_args.c b/lib/librte_pmd_bond/rte_eth_bond_args.c
index d8ce681..2675cf6 100644
--- a/lib/librte_pmd_bond/rte_eth_bond_args.c
+++ b/lib/librte_pmd_bond/rte_eth_bond_args.c
@@ -173,6 +173,7 @@ bond_ethdev_parse_slave_mode_kvarg(const char *key __rte_unused,
 #ifdef RTE_MBUF_REFCNT
 	case BONDING_MODE_BROADCAST:
 #endif
+	case BONDING_MODE_ADAPTIVE_TRANSMIT_LOAD_BALANCING:
 		return 0;
 	default:
 		RTE_BOND_LOG(ERR, "Invalid slave mode value (%s) specified", value);
diff --git a/lib/librte_pmd_bond/rte_eth_bond_pmd.c b/lib/librte_pmd_bond/rte_eth_bond_pmd.c
index cf2fbab..7a5dae6 100644
--- a/lib/librte_pmd_bond/rte_eth_bond_pmd.c
+++ b/lib/librte_pmd_bond/rte_eth_bond_pmd.c
@@ -30,7 +30,7 @@
  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
-
+#include <stdlib.h>
 #include <rte_mbuf.h>
 #include <rte_malloc.h>
 #include <rte_ethdev.h>
@@ -41,10 +41,15 @@
 #include <rte_kvargs.h>
 #include <rte_dev.h>
 #include <rte_alarm.h>
+#include <rte_cycles.h>
 
 #include "rte_eth_bond.h"
 #include "rte_eth_bond_private.h"
 
+#define REORDER_PERIOD_MS 10
+/* Table for statistics in mode 5 TLB */
+static uint64_t tlb_last_obytets[RTE_MAX_ETHPORTS];
+
 static uint16_t
 bond_ethdev_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 {
@@ -288,6 +293,144 @@ xmit_slave_hash(const struct rte_mbuf *buf, uint8_t slave_count, uint8_t policy)
 	return hash % slave_count;
 }
 
+struct bwg_slave {
+	uint64_t bwg_left_int;
+	uint64_t bwg_left_remainder;
+	uint8_t slave;
+};
+
+static int
+bandwidth_cmp(const void *a, const void *b)
+{
+	const struct bwg_slave *bwg_a = a;
+	const struct bwg_slave *bwg_b = b;
+	int64_t diff = (int64_t)bwg_b->bwg_left_int - (int64_t)bwg_a->bwg_left_int;
+	int64_t diff2 = (int64_t)bwg_b->bwg_left_remainder -
+			(int64_t)bwg_a->bwg_left_remainder;
+	if (diff > 0)
+		return 1;
+	else if (diff < 0)
+		return -1;
+	else if (diff2 > 0)
+		return 1;
+	else if (diff2 < 0)
+		return -1;
+	else
+		return 0;
+}
+
+static void
+bandwidth_left(int port_id, uint64_t load, uint8_t update_idx,
+		struct bwg_slave *bwg_slave)
+{
+	struct rte_eth_link link_status;
+
+	rte_eth_link_get(port_id, &link_status);
+	uint64_t link_bwg = link_status.link_speed * 1000000ULL / 8;
+	if (link_bwg == 0)
+		return;
+	link_bwg = (link_bwg * (update_idx+1) * REORDER_PERIOD_MS);
+	bwg_slave->bwg_left_int = (link_bwg - 1000*load) / link_bwg;
+	bwg_slave->bwg_left_remainder = (link_bwg - 1000*load) % link_bwg;
+}
+
+static void
+bond_ethdev_update_tlb_slave_cb(void *arg)
+{
+	struct bond_dev_private *internals = arg;
+	struct rte_eth_stats slave_stats;
+	struct bwg_slave bwg_array[RTE_MAX_ETHPORTS];
+	uint8_t slave_count;
+	uint64_t tx_bytes;
+
+	uint8_t update_stats = 0;
+	uint8_t i, slave_id;
+
+	internals->slave_update_idx++;
+
+
+	if (internals->slave_update_idx >= REORDER_PERIOD_MS)
+		update_stats = 1;
+
+	for (i = 0; i < internals->active_slave_count; i++) {
+		slave_id = internals->active_slaves[i];
+		rte_eth_stats_get(slave_id, &slave_stats);
+		tx_bytes = slave_stats.obytes - tlb_last_obytets[slave_id];
+		bandwidth_left(slave_id, tx_bytes,
+				internals->slave_update_idx, &bwg_array[i]);
+		bwg_array[i].slave = slave_id;
+
+		if (update_stats)
+			tlb_last_obytets[slave_id] = slave_stats.obytes;
+	}
+
+	if (update_stats == 1)
+		internals->slave_update_idx = 0;
+
+	slave_count = i;
+	qsort(bwg_array, slave_count, sizeof(bwg_array[0]), bandwidth_cmp);
+	for (i = 0; i < slave_count; i++)
+		internals->active_slaves[i] = bwg_array[i].slave;
+
+	rte_eal_alarm_set(REORDER_PERIOD_MS * 1000, bond_ethdev_update_tlb_slave_cb,
+			(struct bond_dev_private *)internals);
+}
+
+static uint16_t
+bond_ethdev_tx_burst_tlb(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
+{
+	struct bond_tx_queue *bd_tx_q = (struct bond_tx_queue *)queue;
+	struct bond_dev_private *internals = bd_tx_q->dev_private;
+
+	struct rte_eth_dev *primary_port =
+			&rte_eth_devices[internals->primary_port];
+	uint16_t num_tx_total = 0;
+	uint8_t i, j;
+
+	uint8_t num_of_slaves = internals->active_slave_count;
+	uint8_t slaves[RTE_MAX_ETHPORTS];
+
+	struct ether_hdr *ether_hdr;
+	struct ether_addr primary_slave_addr;
+	struct ether_addr active_slave_addr;
+
+	if (num_of_slaves < 1)
+		return num_tx_total;
+
+	memcpy(slaves, internals->active_slaves,
+				sizeof(internals->active_slaves[0]) * num_of_slaves);
+
+
+	ether_addr_copy(primary_port->data->mac_addrs, &primary_slave_addr);
+
+	if (nb_pkts > 3) {
+		for (i = 0; i < 3; i++)
+			rte_prefetch0(rte_pktmbuf_mtod(bufs[i], void*));
+	}
+
+	for (i = 0; i < num_of_slaves; i++) {
+		ether_addr_copy(&internals->slaves[slaves[i]].persisted_mac_addr,
+				&active_slave_addr);
+
+		for (j = num_tx_total; j < nb_pkts; j++) {
+			if (j + 3 < nb_pkts)
+				rte_prefetch0(rte_pktmbuf_mtod(bufs[j+3], void*));
+
+			ether_hdr = rte_pktmbuf_mtod(bufs[j], struct ether_hdr *);
+			if (is_same_ether_addr(&ether_hdr->s_addr, &primary_slave_addr))
+				ether_addr_copy(&active_slave_addr, &ether_hdr->s_addr);
+		}
+
+		num_tx_total += rte_eth_tx_burst(slaves[i], bd_tx_q->queue_id,
+				bufs + num_tx_total, nb_pkts - num_tx_total);
+
+		if (num_tx_total == nb_pkts)
+			break;
+	}
+
+	return num_tx_total;
+}
+
 static uint16_t
 bond_ethdev_tx_burst_balance(void *queue, struct rte_mbuf **bufs,
 		uint16_t nb_pkts)
@@ -500,6 +643,7 @@ mac_address_slaves_update(struct rte_eth_dev *bonded_eth_dev)
 		}
 		break;
 	case BONDING_MODE_ACTIVE_BACKUP:
+	case BONDING_MODE_ADAPTIVE_TRANSMIT_LOAD_BALANCING:
 	default:
 		for (i = 0; i < internals->slave_count; i++) {
 			if (internals->slaves[i].port_id ==
@@ -551,6 +695,10 @@ bond_ethdev_mode_set(struct rte_eth_dev *eth_dev, int mode)
 		eth_dev->rx_pkt_burst = bond_ethdev_rx_burst;
 		break;
 #endif
+	case BONDING_MODE_ADAPTIVE_TRANSMIT_LOAD_BALANCING:
+		eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_tlb;
+		eth_dev->rx_pkt_burst = bond_ethdev_rx_burst_active_backup;
+		break;
 	default:
 		return -1;
 	}
@@ -676,7 +824,7 @@ slave_add(struct bond_dev_private *internals,
 	}
 
 	slave_details->link_status_wait_to_complete = 0;
-
+	/* clean tlb_last_obytes when adding port for bonding device */
 	memcpy(&(slave_details->persisted_mac_addr), slave_eth_dev->data->mac_addrs,
 			sizeof(struct ether_addr));
 }
@@ -762,6 +910,9 @@ bond_ethdev_start(struct rte_eth_dev *eth_dev)
 	if (internals->user_defined_primary_port)
 		bond_ethdev_primary_set(internals, internals->primary_port);
 
+	if (internals->mode == BONDING_MODE_ADAPTIVE_TRANSMIT_LOAD_BALANCING)
+		bond_ethdev_update_tlb_slave_cb(internals);
+
 	return 0;
 }
 
@@ -770,6 +921,9 @@ bond_ethdev_stop(struct rte_eth_dev *eth_dev)
 {
 	struct bond_dev_private *internals = eth_dev->data->dev_private;
 
+	if (internals->mode == BONDING_MODE_ADAPTIVE_TRANSMIT_LOAD_BALANCING) {
+		rte_eal_alarm_cancel(bond_ethdev_update_tlb_slave_cb, internals);
+    }
 	internals->active_slave_count = 0;
 	internals->link_status_polling_enabled = 0;
 
@@ -1016,6 +1170,7 @@ bond_ethdev_promiscuous_enable(struct rte_eth_dev *eth_dev)
 		break;
 	/* Promiscuous mode is propagated only to primary slave */
 	case BONDING_MODE_ACTIVE_BACKUP:
+	case BONDING_MODE_ADAPTIVE_TRANSMIT_LOAD_BALANCING:
 	default:
 		rte_eth_promiscuous_enable(internals->current_primary_port);
 
@@ -1042,6 +1197,7 @@ bond_ethdev_promiscuous_disable(struct rte_eth_dev *dev)
 		break;
 	/* Promiscuous mode is propagated only to primary slave */
 	case BONDING_MODE_ACTIVE_BACKUP:
+	case BONDING_MODE_ADAPTIVE_TRANSMIT_LOAD_BALANCING:
 	default:
 		rte_eth_promiscuous_disable(internals->current_primary_port);
 	}
diff --git a/lib/librte_pmd_bond/rte_eth_bond_private.h b/lib/librte_pmd_bond/rte_eth_bond_private.h
index 6254c84..2a4e129 100644
--- a/lib/librte_pmd_bond/rte_eth_bond_private.h
+++ b/lib/librte_pmd_bond/rte_eth_bond_private.h
@@ -102,7 +102,6 @@ struct bond_slave_details {
 	uint8_t link_status_poll_enabled;
 	uint8_t link_status_wait_to_complete;
 	uint8_t last_link_status;
-
 	/**< Port Id of slave eth_dev */
 	struct ether_addr persisted_mac_addr;
 };
@@ -145,6 +144,7 @@ struct bond_dev_private {
 	/**< Arary of bonded slaves details */
 
 	struct rte_kvargs *kvlist;
+	uint8_t slave_update_idx;
 };
 
 extern struct eth_dev_ops default_dev_ops;
-- 
2.1.0

[dpdk-dev] [PATCH v3 2/2] Unit tests for Mode 5 of Bonding Transmit Load balancing.

[Daniel Mrzyglod]

This patch add unit tests for mode 5 - tlb - to the others 
link bonding unit tests.

Signed-off-by: Daniel Mrzyglod <danielx.t.mrzyglod@intel.com>
---
 app/test/test_link_bonding.c | 499 ++++++++++++++++++++++++++++++++++++++++++-
 app/test/virtual_pmd.c       |   6 +-
 2 files changed, 502 insertions(+), 3 deletions(-)

diff --git a/app/test/test_link_bonding.c b/app/test/test_link_bonding.c
index 93449af..f62c490 100644
--- a/app/test/test_link_bonding.c
+++ b/app/test/test_link_bonding.c
@@ -41,7 +41,7 @@
 #include <errno.h>
 #include <sys/queue.h>
 #include <sys/time.h>
-
+#include <rte_cycles.h>
 #include <rte_byteorder.h>
 #include <rte_common.h>
 #include <rte_debug.h>
@@ -3856,6 +3856,498 @@ testsuite_teardown(void)
 	return remove_slaves_and_stop_bonded_device();
 }
 
+static int
+test_tlb_tx_burst(void)
+{
+	int i, burst_size, nb_tx;
+	uint64_t nb_tx2 = 0;
+	struct rte_mbuf *pkt_burst[MAX_PKT_BURST];
+	struct rte_eth_stats port_stats[32];
+	uint64_t sum_ports_opackets = 0, all_bond_opackets = 0, all_bond_obytes = 0;
+	uint16_t pktlen;
+	uint64_t floor_obytes = 0, ceiling_obytes = 0;
+
+	TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves
+			(BONDING_MODE_ADAPTIVE_TRANSMIT_LOAD_BALANCING, 1, 3, 1),
+			"Failed to initialise bonded device");
+
+	burst_size = 20 * test_params->bonded_slave_count;
+
+	TEST_ASSERT(burst_size < MAX_PKT_BURST,
+			"Burst size specified is greater than supported.\n");
+
+
+	/* Generate 400000 test bursts in 2s of packets to transmit  */
+	for (i = 0; i < 400000; i++) {
+		/*test two types of mac src own(bonding) and others */
+		if (i % 2 == 0) {
+			initialize_eth_header(test_params->pkt_eth_hdr,
+					(struct ether_addr *)src_mac,
+					(struct ether_addr *)dst_mac_0, 0, 0);
+		} else {
+			initialize_eth_header(test_params->pkt_eth_hdr,
+					(struct ether_addr *)test_params->default_slave_mac,
+					(struct ether_addr *)dst_mac_0, 0, 0);
+		}
+		pktlen = initialize_udp_header(test_params->pkt_udp_hdr, src_port,
+				dst_port_0, 16);
+		pktlen = initialize_ipv4_header(test_params->pkt_ipv4_hdr, src_addr,
+				dst_addr_0, pktlen);
+		generate_packet_burst(test_params->mbuf_pool, pkt_burst,
+				test_params->pkt_eth_hdr, 0, test_params->pkt_ipv4_hdr,
+				1, test_params->pkt_udp_hdr, burst_size, 60, 1);
+		/* Send burst on bonded port */
+		nb_tx = rte_eth_tx_burst(test_params->bonded_port_id, 0, pkt_burst,
+				burst_size);
+		nb_tx2 += nb_tx;
+
+		TEST_ASSERT_EQUAL(nb_tx, burst_size,
+				"number of packet not equal burst size");
+
+		rte_delay_us(5);
+	}
+
+
+	/* Verify bonded port tx stats */
+	rte_eth_stats_get(test_params->bonded_port_id, &port_stats[0]);
+
+	all_bond_opackets = port_stats[0].opackets;
+	all_bond_obytes = port_stats[0].obytes;
+
+	TEST_ASSERT_EQUAL(port_stats[0].opackets, (uint64_t)nb_tx2,
+			"Bonded Port (%d) opackets value (%u) not as expected (%d)\n",
+			test_params->bonded_port_id, (unsigned int)port_stats[0].opackets,
+			burst_size);
+
+
+	/* Verify slave ports tx stats */
+	for (i = 0; i < test_params->bonded_slave_count; i++) {
+		rte_eth_stats_get(test_params->slave_port_ids[i], &port_stats[i]);
+		sum_ports_opackets += port_stats[i].opackets;
+	}
+
+	TEST_ASSERT_EQUAL(sum_ports_opackets, (uint64_t)all_bond_opackets,
+			"Total packets sent by slaves is not equal to packets sent by bond interface");
+	/* distribution of packets on each slave within +/- 10% of the expected value. */
+	for (i = 0; i < test_params->bonded_slave_count; i++) {
+
+		floor_obytes = (all_bond_obytes*90)/(test_params->bonded_slave_count*100);
+		ceiling_obytes = (all_bond_obytes*110)/(test_params->bonded_slave_count*100);
+		TEST_ASSERT(port_stats[i].obytes >= floor_obytes &&
+				port_stats[i].obytes <= ceiling_obytes,
+						"Distribution is not even");
+	}
+	/* Put all slaves down and try and transmit */
+	for (i = 0; i < test_params->bonded_slave_count; i++) {
+		virtual_ethdev_simulate_link_status_interrupt(
+				test_params->slave_port_ids[i], 0);
+	}
+
+	/* Send burst on bonded port */
+	nb_tx = rte_eth_tx_burst(test_params->bonded_port_id, 0, pkt_burst,
+			burst_size);
+	TEST_ASSERT_EQUAL(nb_tx, 0, " bad number of packet in burst");
+
+	/* Clean ugit checkout masterp and remove slaves from bonded device */
+	return remove_slaves_and_stop_bonded_device();
+}
+
+#define TEST_ADAPTIVE_TRANSMIT_LOAD_BALANCING_RX_BURST_SLAVE_COUNT (4)
+
+static int
+test_tlb_rx_burst(void)
+{
+	struct rte_mbuf *gen_pkt_burst[MAX_PKT_BURST] = { NULL };
+	struct rte_mbuf *rx_pkt_burst[MAX_PKT_BURST] = { NULL };
+
+	struct rte_eth_stats port_stats;
+
+	int primary_port;
+
+	uint16_t i, j, nb_rx, burst_size = 17;
+
+	/* Initialize bonded device with 4 slaves in transmit load balancing mode */
+	TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves(
+			BONDING_MODE_ADAPTIVE_TRANSMIT_LOAD_BALANCING,
+			TEST_ADAPTIVE_TRANSMIT_LOAD_BALANCING_RX_BURST_SLAVE_COUNT, 1, 1),
+			"Failed to initialize bonded device");
+
+
+	primary_port = rte_eth_bond_primary_get(test_params->bonded_port_id);
+	TEST_ASSERT(primary_port >= 0,
+			"failed to get primary slave for bonded port (%d)",
+			test_params->bonded_port_id);
+
+	for (i = 0; i < test_params->bonded_slave_count; i++) {
+		/* Generate test bursts of packets to transmit */
+		TEST_ASSERT_EQUAL(generate_test_burst(
+				&gen_pkt_burst[0], burst_size, 0, 1, 0, 0, 0), burst_size,
+				"burst generation failed");
+
+		/* Add rx data to slave */
+		virtual_ethdev_add_mbufs_to_rx_queue(test_params->slave_port_ids[i],
+				&gen_pkt_burst[0], burst_size);
+
+		/* Call rx burst on bonded device */
+		nb_rx = rte_eth_rx_burst(test_params->bonded_port_id, 0,
+				&rx_pkt_burst[0], MAX_PKT_BURST);
+
+		TEST_ASSERT_EQUAL(nb_rx, burst_size, "rte_eth_rx_burst failed\n");
+
+		if (test_params->slave_port_ids[i] == primary_port) {
+			/* Verify bonded device rx count */
+			rte_eth_stats_get(test_params->bonded_port_id, &port_stats);
+			TEST_ASSERT_EQUAL(port_stats.ipackets, (uint64_t)burst_size,
+					"Bonded Port (%d) ipackets value (%u) not as expected (%d)\n",
+					test_params->bonded_port_id,
+					(unsigned int)port_stats.ipackets, burst_size);
+
+			/* Verify bonded slave devices rx count */
+			for (j = 0; j < test_params->bonded_slave_count; j++) {
+				rte_eth_stats_get(test_params->slave_port_ids[j], &port_stats);
+				if (i == j) {
+					TEST_ASSERT_EQUAL(port_stats.ipackets, (uint64_t)burst_size,
+							"Slave Port (%d) ipackets value (%u) not as expected (%d)\n",
+							test_params->slave_port_ids[i],
+							(unsigned int)port_stats.ipackets, burst_size);
+				} else {
+					TEST_ASSERT_EQUAL(port_stats.ipackets, (uint64_t)0,
+							"Slave Port (%d) ipackets value (%u) not as expected (%d)\n",
+							test_params->slave_port_ids[i],
+							(unsigned int)port_stats.ipackets, 0);
+				}
+			}
+		} else {
+			for (j = 0; j < test_params->bonded_slave_count; j++) {
+				rte_eth_stats_get(test_params->slave_port_ids[j], &port_stats);
+				TEST_ASSERT_EQUAL(port_stats.ipackets, (uint64_t)0,
+						"Slave Port (%d) ipackets value (%u) not as expected (%d)\n",
+						test_params->slave_port_ids[i],
+						(unsigned int)port_stats.ipackets, 0);
+			}
+		}
+
+		/* free mbufs */
+		for (i = 0; i < burst_size; i++)
+			rte_pktmbuf_free(rx_pkt_burst[i]);
+
+		/* reset bonded device stats */
+		rte_eth_stats_reset(test_params->bonded_port_id);
+	}
+
+	/* Clean up and remove slaves from bonded device */
+	return remove_slaves_and_stop_bonded_device();
+}
+
+static int
+test_tlb_verify_promiscuous_enable_disable(void)
+{
+	int i, primary_port, promiscuous_en;
+
+	/* Initialize bonded device with 4 slaves in transmit load balancing mode */
+	TEST_ASSERT_SUCCESS( initialize_bonded_device_with_slaves(
+			BONDING_MODE_ADAPTIVE_TRANSMIT_LOAD_BALANCING, 0, 4, 1),
+			"Failed to initialize bonded device");
+
+	primary_port = rte_eth_bond_primary_get(test_params->bonded_port_id);
+	TEST_ASSERT(primary_port >= 0,
+			"failed to get primary slave for bonded port (%d)",
+			test_params->bonded_port_id);
+
+	rte_eth_promiscuous_enable(test_params->bonded_port_id);
+
+	promiscuous_en = rte_eth_promiscuous_get(test_params->bonded_port_id);
+	TEST_ASSERT_EQUAL(promiscuous_en, (int)1,
+			"Port (%d) promiscuous mode not enabled\n",
+			test_params->bonded_port_id);
+	for (i = 0; i < test_params->bonded_slave_count; i++) {
+		promiscuous_en = rte_eth_promiscuous_get(
+				test_params->slave_port_ids[i]);
+		if (primary_port == test_params->slave_port_ids[i]) {
+			TEST_ASSERT_EQUAL(promiscuous_en, (int)1,
+					"Port (%d) promiscuous mode not enabled\n",
+					test_params->bonded_port_id);
+		} else {
+			TEST_ASSERT_EQUAL(promiscuous_en, (int)0,
+					"Port (%d) promiscuous mode enabled\n",
+					test_params->bonded_port_id);
+		}
+
+	}
+
+	rte_eth_promiscuous_disable(test_params->bonded_port_id);
+
+	promiscuous_en = rte_eth_promiscuous_get(test_params->bonded_port_id);
+	TEST_ASSERT_EQUAL(promiscuous_en, (int)0,
+			"Port (%d) promiscuous mode not disabled\n",
+			test_params->bonded_port_id);
+
+	for (i = 0; i < test_params->bonded_slave_count; i++) {
+		promiscuous_en = rte_eth_promiscuous_get(
+				test_params->slave_port_ids[i]);
+		TEST_ASSERT_EQUAL(promiscuous_en, (int)0,
+				"slave port (%d) promiscuous mode not disabled\n",
+				test_params->slave_port_ids[i]);
+	}
+
+	/* Clean up and remove slaves from bonded device */
+	return remove_slaves_and_stop_bonded_device();
+}
+
+static int
+test_tlb_verify_mac_assignment(void)
+{
+	struct ether_addr read_mac_addr, expected_mac_addr_0, expected_mac_addr_1;
+
+	rte_eth_macaddr_get(test_params->slave_port_ids[0], &expected_mac_addr_0);
+	rte_eth_macaddr_get(test_params->slave_port_ids[1], &expected_mac_addr_1);
+
+	/* Initialize bonded device with 2 slaves in active backup mode */
+	TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves(
+			BONDING_MODE_ADAPTIVE_TRANSMIT_LOAD_BALANCING, 0, 2, 1),
+			"Failed to initialize bonded device");
+
+	/* Verify that bonded MACs is that of first slave and that the other slave
+	 * MAC hasn't been changed */
+	rte_eth_macaddr_get(test_params->bonded_port_id, &read_mac_addr);
+	TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_0, &read_mac_addr,
+			sizeof(read_mac_addr)),
+			"bonded port (%d) mac address not set to that of primary port",
+			test_params->bonded_port_id);
+
+	rte_eth_macaddr_get(test_params->slave_port_ids[0], &read_mac_addr);
+	TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_0, &read_mac_addr,
+			sizeof(read_mac_addr)),
+			"slave port (%d) mac address not set to that of primary port",
+			test_params->slave_port_ids[0]);
+
+	rte_eth_macaddr_get(test_params->slave_port_ids[1], &read_mac_addr);
+	TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_1, &read_mac_addr,
+			sizeof(read_mac_addr)),
+			"slave port (%d) mac address not as expected",
+			test_params->slave_port_ids[1]);
+
+	/* change primary and verify that MAC addresses haven't changed */
+	TEST_ASSERT_EQUAL(rte_eth_bond_primary_set(test_params->bonded_port_id,
+			test_params->slave_port_ids[1]), 0,
+			"Failed to set bonded port (%d) primary port to (%d)",
+			test_params->bonded_port_id, test_params->slave_port_ids[1]);
+
+	rte_eth_macaddr_get(test_params->bonded_port_id, &read_mac_addr);
+	TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_0, &read_mac_addr,
+			sizeof(read_mac_addr)),
+			"bonded port (%d) mac address not set to that of primary port",
+			test_params->bonded_port_id);
+
+	rte_eth_macaddr_get(test_params->slave_port_ids[0], &read_mac_addr);
+	TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_0, &read_mac_addr,
+			sizeof(read_mac_addr)),
+			"slave port (%d) mac address not set to that of primary port",
+			test_params->slave_port_ids[0]);
+
+	rte_eth_macaddr_get(test_params->slave_port_ids[1], &read_mac_addr);
+	TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_1, &read_mac_addr,
+			sizeof(read_mac_addr)),
+			"slave port (%d) mac address not as expected",
+			test_params->slave_port_ids[1]);
+
+	/* stop / start bonded device and verify that primary MAC address is
+	 * propagated to bonded device and slaves */
+
+	rte_eth_dev_stop(test_params->bonded_port_id);
+
+	TEST_ASSERT_SUCCESS(rte_eth_dev_start(test_params->bonded_port_id),
+			"Failed to start device");
+
+	rte_eth_macaddr_get(test_params->bonded_port_id, &read_mac_addr);
+	TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_1, &read_mac_addr,
+			sizeof(read_mac_addr)),
+			"bonded port (%d) mac address not set to that of primary port",
+			test_params->bonded_port_id);
+
+	rte_eth_macaddr_get(test_params->slave_port_ids[0], &read_mac_addr);
+	TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_0, &read_mac_addr,
+			sizeof(read_mac_addr)),
+			"slave port (%d) mac address not as expected",
+			test_params->slave_port_ids[0]);
+
+	rte_eth_macaddr_get(test_params->slave_port_ids[1], &read_mac_addr);
+	TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_1, &read_mac_addr,
+			sizeof(read_mac_addr)),
+			"slave port (%d) mac address not set to that of primary port",
+			test_params->slave_port_ids[1]);
+
+
+	/* Set explicit MAC address */
+	TEST_ASSERT_SUCCESS(rte_eth_bond_mac_address_set(
+			test_params->bonded_port_id, (struct ether_addr *)bonded_mac),
+			"failed to set MAC addres");
+
+	rte_eth_macaddr_get(test_params->bonded_port_id, &read_mac_addr);
+	TEST_ASSERT_SUCCESS(memcmp(&bonded_mac, &read_mac_addr,
+			sizeof(read_mac_addr)),
+			"bonded port (%d) mac address not set to that of bonded port",
+			test_params->bonded_port_id);
+
+	rte_eth_macaddr_get(test_params->slave_port_ids[0], &read_mac_addr);
+	TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_0, &read_mac_addr,
+			sizeof(read_mac_addr)),
+			"slave port (%d) mac address not as expected",
+			test_params->slave_port_ids[0]);
+
+	rte_eth_macaddr_get(test_params->slave_port_ids[1], &read_mac_addr);
+	TEST_ASSERT_SUCCESS(memcmp(&bonded_mac, &read_mac_addr,
+			sizeof(read_mac_addr)),
+			"slave port (%d) mac address not set to that of bonded port",
+			test_params->slave_port_ids[1]);
+
+	/* Clean up and remove slaves from bonded device */
+	return remove_slaves_and_stop_bonded_device();
+}
+
+static int
+test_tlb_verify_slave_link_status_change_failover(void)
+{
+	struct rte_mbuf *pkt_burst[TEST_ADAPTIVE_TRANSMIT_LOAD_BALANCING_RX_BURST_SLAVE_COUNT][MAX_PKT_BURST];
+	struct rte_mbuf *rx_pkt_burst[MAX_PKT_BURST] = { NULL };
+	struct rte_eth_stats port_stats;
+
+	uint8_t slaves[RTE_MAX_ETHPORTS];
+
+	int i, j, burst_size, slave_count, primary_port;
+
+	burst_size = 21;
+
+	memset(pkt_burst, 0, sizeof(pkt_burst));
+
+
+
+	/* Initialize bonded device with 4 slaves in round robin mode */
+	TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves(
+			BONDING_MODE_ADAPTIVE_TRANSMIT_LOAD_BALANCING, 0,
+			TEST_ADAPTIVE_TRANSMIT_LOAD_BALANCING_RX_BURST_SLAVE_COUNT, 1),
+			"Failed to initialize bonded device with slaves");
+
+	/* Verify Current Slaves Count /Active Slave Count is */
+	slave_count = rte_eth_bond_slaves_get(test_params->bonded_port_id, slaves,
+			RTE_MAX_ETHPORTS);
+	TEST_ASSERT_EQUAL(slave_count, 4,
+			"Number of slaves (%d) is not as expected (%d).\n",
+			slave_count, 4);
+
+	slave_count = rte_eth_bond_active_slaves_get(test_params->bonded_port_id,
+			slaves, RTE_MAX_ETHPORTS);
+	TEST_ASSERT_EQUAL(slave_count, (int)4,
+			"Number of slaves (%d) is not as expected (%d).\n",
+			slave_count, 4);
+
+	primary_port = rte_eth_bond_primary_get(test_params->bonded_port_id);
+	TEST_ASSERT_EQUAL(primary_port, test_params->slave_port_ids[0],
+			"Primary port not as expected");
+
+	/* Bring 2 slaves down and verify active slave count */
+	virtual_ethdev_simulate_link_status_interrupt(
+			test_params->slave_port_ids[1], 0);
+	virtual_ethdev_simulate_link_status_interrupt(
+			test_params->slave_port_ids[3], 0);
+
+	TEST_ASSERT_EQUAL(rte_eth_bond_active_slaves_get(
+			test_params->bonded_port_id, slaves, RTE_MAX_ETHPORTS), 2,
+			"Number of active slaves (%d) is not as expected (%d).",
+			slave_count, 2);
+
+	virtual_ethdev_simulate_link_status_interrupt(
+			test_params->slave_port_ids[1], 1);
+	virtual_ethdev_simulate_link_status_interrupt(
+			test_params->slave_port_ids[3], 1);
+
+
+	/* Bring primary port down, verify that active slave count is 3 and primary
+	 *  has changed */
+	virtual_ethdev_simulate_link_status_interrupt(
+			test_params->slave_port_ids[0], 0);
+
+	TEST_ASSERT_EQUAL(rte_eth_bond_active_slaves_get(
+			test_params->bonded_port_id, slaves, RTE_MAX_ETHPORTS), 3,
+			"Number of active slaves (%d) is not as expected (%d).",
+			slave_count, 3);
+
+	primary_port = rte_eth_bond_primary_get(test_params->bonded_port_id);
+	TEST_ASSERT_EQUAL(primary_port, test_params->slave_port_ids[2],
+			"Primary port not as expected");
+	rte_delay_us(500000);
+	/* Verify that pkts are sent on new primary slave */
+	for (i = 0; i < 4; i++) {
+		TEST_ASSERT_EQUAL(generate_test_burst(
+				&pkt_burst[0][0], burst_size, 0, 1, 0, 0, 0), burst_size,
+				"generate_test_burst failed\n");
+		TEST_ASSERT_EQUAL(rte_eth_tx_burst(
+				test_params->bonded_port_id, 0, &pkt_burst[0][0], burst_size), burst_size,
+				"rte_eth_tx_burst failed\n");
+		rte_delay_us(11000);
+	}
+
+	rte_eth_stats_get(test_params->slave_port_ids[2], &port_stats);
+	TEST_ASSERT_NOT_EQUAL(port_stats.opackets, (int8_t)0,
+			"(%d) port_stats.opackets not as expected\n",
+			test_params->slave_port_ids[2]);
+
+	rte_eth_stats_get(test_params->slave_port_ids[0], &port_stats);
+	TEST_ASSERT_EQUAL(port_stats.opackets, (int8_t)0,
+			"(%d) port_stats.opackets not as expected\n",
+			test_params->slave_port_ids[0]);
+
+	rte_eth_stats_get(test_params->slave_port_ids[1], &port_stats);
+	TEST_ASSERT_NOT_EQUAL(port_stats.opackets, (int8_t)0,
+			"(%d) port_stats.opackets not as expected\n",
+			test_params->slave_port_ids[1]);
+
+	rte_eth_stats_get(test_params->slave_port_ids[3], &port_stats);
+	TEST_ASSERT_NOT_EQUAL(port_stats.opackets, (int8_t)0,
+			"(%d) port_stats.opackets not as expected\n",
+			test_params->slave_port_ids[3]);
+
+
+	/* Generate packet burst for testing */
+
+	for (i = 0; i < TEST_ADAPTIVE_TRANSMIT_LOAD_BALANCING_RX_BURST_SLAVE_COUNT; i++) {
+		if (generate_test_burst(&pkt_burst[i][0], burst_size, 0, 1, 0, 0, 0) !=
+				burst_size)
+			return -1;
+
+		virtual_ethdev_add_mbufs_to_rx_queue(
+				test_params->slave_port_ids[i], &pkt_burst[i][0], burst_size);
+	}
+
+	if (rte_eth_rx_burst(test_params->bonded_port_id, 0, rx_pkt_burst,
+			MAX_PKT_BURST) != burst_size) {
+		printf("rte_eth_rx_burst\n");
+		return -1;
+
+	}
+
+	/* Verify bonded device rx count */
+	rte_eth_stats_get(test_params->bonded_port_id, &port_stats);
+	TEST_ASSERT_EQUAL(port_stats.ipackets, (uint64_t)burst_size,
+			"(%d) port_stats.ipackets not as expected\n",
+			test_params->bonded_port_id);
+
+	/* free mbufs */
+
+	for (i = 0; i < TEST_ADAPTIVE_TRANSMIT_LOAD_BALANCING_RX_BURST_SLAVE_COUNT; i++) {
+		for (j = 0; j < MAX_PKT_BURST; j++) {
+			if (pkt_burst[i][j] != NULL) {
+				rte_pktmbuf_free(pkt_burst[i][j]);
+				pkt_burst[i][j] = NULL;
+			}
+		}
+	}
+
+
+	/* Clean up and remove slaves from bonded device */
+	return remove_slaves_and_stop_bonded_device();
+}
+
 
 static struct unit_test_suite link_bonding_test_suite  = {
 	.suite_name = "Link Bonding Unit Test Suite",
@@ -3909,6 +4401,11 @@ static struct unit_test_suite link_bonding_test_suite  = {
 		TEST_CASE(test_balance_verify_promiscuous_enable_disable),
 		TEST_CASE(test_balance_verify_mac_assignment),
 		TEST_CASE(test_balance_verify_slave_link_status_change_behaviour),
+		TEST_CASE(test_tlb_tx_burst),
+		TEST_CASE(test_tlb_rx_burst),
+		TEST_CASE(test_tlb_verify_mac_assignment),
+		TEST_CASE(test_tlb_verify_promiscuous_enable_disable),
+		TEST_CASE(test_tlb_verify_slave_link_status_change_failover),
 #ifdef RTE_MBUF_REFCNT
 		TEST_CASE(test_broadcast_tx_burst),
 		TEST_CASE(test_broadcast_tx_burst_slave_tx_fail),
diff --git a/app/test/virtual_pmd.c b/app/test/virtual_pmd.c
index e7be98b..ade6cb0 100644
--- a/app/test/virtual_pmd.c
+++ b/app/test/virtual_pmd.c
@@ -359,16 +359,18 @@ virtual_ethdev_tx_burst_success(void *queue, struct rte_mbuf **bufs,
 
 	struct rte_eth_dev *vrtl_eth_dev;
 	struct virtual_ethdev_private *dev_private;
-
+	uint64_t obytes = 0;
 	int i;
 
+	for (i = 0; i < nb_pkts; i++)
+		obytes += rte_pktmbuf_pkt_len(bufs[i]);
 	vrtl_eth_dev = &rte_eth_devices[tx_q->port_id];
 	dev_private = vrtl_eth_dev->data->dev_private;
 
 	if (vrtl_eth_dev->data->dev_link.link_status) {
 		/* increment opacket count */
 		dev_private->eth_stats.opackets += nb_pkts;
-
+		dev_private->eth_stats.obytes += obytes;
 		/* free packets in burst */
 		for (i = 0; i < nb_pkts; i++)
 			rte_pktmbuf_free(bufs[i]);
-- 
2.1.0

Re: [dpdk-dev] [PATCH v3 0/2] ADD mode 5(tlb) to link bonding pmd

[Mrzyglod, DanielX T]

v3 change:
Rebase patch version to HEAD of orgin/master.
Unit tests moved to the separate patch v3 2/2.

v2 change:
Add Unit Tests
Modification that updates obytes structure in virtualpmd driver.
change internals->slaves[i].last_obytes to have proper values.
Update codebase to Declan's patches.

v1 change:
Add support for mode 5 (Transmit load balancing) into pmd driver

This mode provides an adaptive transmit load balancing. 
It dynamically changes the transmitting slave, according to the computed load. 
Statistics are collected in 100ms intervals and scheduled every 10ms.

> -----Original Message-----
> From: Mrzyglod, DanielX T
> Sent: Wednesday, November 26, 2014 6:13 PM
> To: dev@dpdk.org
> Cc: Mrzyglod, DanielX T
> Subject: [PATCH v3 0/2] ADD mode 5(tlb) to link bonding pmd
> 
> This mode provides an adaptive transmit load balancing.
> It dynamically changes the transmitting slave, according to the computed load.
> Statistics are collected in 100ms intervals and scheduled every 10ms.
> 
> Daniel Mrzyglod (2):
>   This patch add support of mode 5 to link bonding pmd
>   Unit tests for Mode 5 of Bonding Transmit Load balancing.
> 
>  app/test/test_link_bonding.c               | 499 ++++++++++++++++++++++++++++-
>  app/test/virtual_pmd.c                     |   6 +-
>  lib/librte_pmd_bond/rte_eth_bond.h         |  11 +
>  lib/librte_pmd_bond/rte_eth_bond_args.c    |   1 +
>  lib/librte_pmd_bond/rte_eth_bond_pmd.c     | 160 ++++++++-
>  lib/librte_pmd_bond/rte_eth_bond_private.h |   2 +-
>  6 files changed, 673 insertions(+), 6 deletions(-)
> 
> --
> 2.1.0

Re: [dpdk-dev] [PATCH v3 0/2] ADD mode 5(tlb) to link bonding pmd

[Doherty, Declan]

> -----Original Message-----
> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Mrzyglod, DanielX T
> Sent: Wednesday, November 26, 2014 5:18 PM
> To: dev@dpdk.org
> Subject: Re: [dpdk-dev] [PATCH v3 0/2] ADD mode 5(tlb) to link bonding pmd
> 
> v3 change:
> Rebase patch version to HEAD of orgin/master.
> Unit tests moved to the separate patch v3 2/2.
> 
> v2 change:
> Add Unit Tests
> Modification that updates obytes structure in virtualpmd driver.
> change internals->slaves[i].last_obytes to have proper values.
> Update codebase to Declan's patches.
> 
> v1 change:
> Add support for mode 5 (Transmit load balancing) into pmd driver
> 
> This mode provides an adaptive transmit load balancing.
> It dynamically changes the transmitting slave, according to the computed load.
> Statistics are collected in 100ms intervals and scheduled every 10ms.
> 
> > -----Original Message-----
> > From: Mrzyglod, DanielX T
> > Sent: Wednesday, November 26, 2014 6:13 PM
> > To: dev@dpdk.org
> > Cc: Mrzyglod, DanielX T
> > Subject: [PATCH v3 0/2] ADD mode 5(tlb) to link bonding pmd
> >
> > This mode provides an adaptive transmit load balancing.
> > It dynamically changes the transmitting slave, according to the computed load.
> > Statistics are collected in 100ms intervals and scheduled every 10ms.
> >
> > Daniel Mrzyglod (2):
> >   This patch add support of mode 5 to link bonding pmd
> >   Unit tests for Mode 5 of Bonding Transmit Load balancing.
> >
> >  app/test/test_link_bonding.c               | 499
> ++++++++++++++++++++++++++++-
> >  app/test/virtual_pmd.c                     |   6 +-
> >  lib/librte_pmd_bond/rte_eth_bond.h         |  11 +
> >  lib/librte_pmd_bond/rte_eth_bond_args.c    |   1 +
> >  lib/librte_pmd_bond/rte_eth_bond_pmd.c     | 160 ++++++++-
> >  lib/librte_pmd_bond/rte_eth_bond_private.h |   2 +-
> >  6 files changed, 673 insertions(+), 6 deletions(-)
> >
> > --
> > 2.1.0

Acked by Declan Doherty <declan.doherty@intel.com>