DPDK patches and discussions
 help / color / mirror / Atom feed
* [dpdk-dev] [RFC PATCH v1] rte: LCore heartbeat example
@ 2015-09-15 12:16 Remy Horton
  2015-09-15 13:10 ` Thomas Monjalon
  2015-09-30  9:04 ` [dpdk-dev] [PATCH v2 0/3] Keepalive monitoring & reporting Remy Horton
  0 siblings, 2 replies; 44+ messages in thread
From: Remy Horton @ 2015-09-15 12:16 UTC (permalink / raw)
  To: dev

Provides a basic framework for detecting and reporting live-ness of 
LCores, the primary requirement of which is minimal overheads for the 
core(s) being checked. Core failures are notified via an application 
defined callback. As an example l2fwd with random failures is used.

Signed-off-by: Remy Horton <remy.horton@intel.com>
---
 examples/l2fwd-heartbeat/Makefile |  50 +++
 examples/l2fwd-heartbeat/hbeat.c  | 121 ++++++
 examples/l2fwd-heartbeat/hbeat.h  | 143 +++++++
 examples/l2fwd-heartbeat/main.c   | 809 ++++++++++++++++++++++++++++++++++++++
 4 files changed, 1123 insertions(+)
 create mode 100644 examples/l2fwd-heartbeat/Makefile
 create mode 100644 examples/l2fwd-heartbeat/hbeat.c
 create mode 100644 examples/l2fwd-heartbeat/hbeat.h
 create mode 100644 examples/l2fwd-heartbeat/main.c

diff --git a/examples/l2fwd-heartbeat/Makefile b/examples/l2fwd-heartbeat/Makefile
new file mode 100644
index 0000000..8b89476
--- /dev/null
+++ b/examples/l2fwd-heartbeat/Makefile
@@ -0,0 +1,50 @@
+#   BSD LICENSE
+#
+#   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+#   All rights reserved.
+#
+#   Redistribution and use in source and binary forms, with or without
+#   modification, are permitted provided that the following conditions
+#   are met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in
+#       the documentation and/or other materials provided with the
+#       distribution.
+#     * Neither the name of Intel Corporation nor the names of its
+#       contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+#
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ifeq ($(RTE_SDK),)
+$(error "Please define RTE_SDK environment variable")
+endif
+
+# Default target, can be overridden by command line or environment
+RTE_TARGET ?= x86_64-native-linuxapp-gcc
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# binary name
+APP = l2fwd-heartbeat
+
+# all source are stored in SRCS-y
+SRCS-y := main.c hbeat.c
+
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS)
+
+include $(RTE_SDK)/mk/rte.extapp.mk
diff --git a/examples/l2fwd-heartbeat/hbeat.c b/examples/l2fwd-heartbeat/hbeat.c
new file mode 100644
index 0000000..755f3e8
--- /dev/null
+++ b/examples/l2fwd-heartbeat/hbeat.c
@@ -0,0 +1,121 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright 2015 Intel Shannon Ltd. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <inttypes.h>
+
+#include <rte_timer.h>
+#include <rte_cycles.h>
+#include <rte_lcore.h>
+
+#include "hbeat.h"
+
+#ifdef HBEAT_DEBUG_MSGS
+static void
+print_trace(const char *msg, struct rte_heartbeat *keepcfg, int idx_core)
+{
+	printf("%sLast seen %" PRId64  "ms ago.\n",
+		msg,
+		((rte_rdtsc() - keepcfg->core[idx_core].last_alive)*1000)
+		/ rte_get_tsc_hz()
+	      );
+}
+#else
+static void
+print_trace(__attribute__((unused)) const char *msg,
+	__attribute__((unused)) struct rte_heartbeat *keepcfg,
+	__attribute__((unused)) int idx_core)
+{
+}
+#endif
+
+
+
+void
+hbeat_dispatch_pings(__attribute__((unused)) struct rte_timer *ptr_timer,
+	void *ptr_data)
+{
+	struct rte_heartbeat *keepcfg = (struct rte_heartbeat *)ptr_data;
+	int idx_core;
+
+	for (idx_core = 0; idx_core < RTE_HBEAT_MAXCORES; idx_core++) {
+		if (keepcfg->active_cores[idx_core] == 0)
+			continue;
+		switch (keepcfg->core[idx_core].state_flags) {
+		case 1: /* Alive */
+			keepcfg->core[idx_core].state_flags = 0;
+			keepcfg->core[idx_core].last_alive = rte_rdtsc();
+			break;
+		case 0: /* MIA */
+			print_trace("Core MIA. ", keepcfg, idx_core);
+			keepcfg->core[idx_core].state_flags = 2;
+			break;
+		case 2: /* Dead */
+			keepcfg->core[idx_core].state_flags = 3;
+			print_trace("Core died. ", keepcfg, idx_core);
+			if (keepcfg->callback)
+				keepcfg->callback(
+					keepcfg->callback_data,
+					idx_core
+					);
+			break;
+		case 3: /* Buried */
+			break;
+		}
+	}
+}
+
+
+int
+hbeat_init(struct rte_heartbeat *keepcfg,
+	rte_heartbeat_failure_callback_t callback,
+	void *data)
+{
+	int idx_core;
+
+	for (idx_core = 0; idx_core < RTE_HBEAT_MAXCORES; idx_core++) {
+		keepcfg->core[idx_core].state_flags = 0;
+		keepcfg->active_cores[idx_core] = 0;
+	}
+	keepcfg->callback = callback;
+	keepcfg->callback_data = data;
+	keepcfg->tsc_initial = rte_rdtsc();
+	keepcfg->tsc_mhz = rte_get_tsc_hz() / 1000;
+	return 0;
+}
+
+
+void
+hbeat_register_core(struct rte_heartbeat *keepcfg, const int id_core)
+{
+	if (id_core < RTE_HBEAT_MAXCORES)
+		keepcfg->active_cores[id_core] = 1;
+}
diff --git a/examples/l2fwd-heartbeat/hbeat.h b/examples/l2fwd-heartbeat/hbeat.h
new file mode 100644
index 0000000..b70503e
--- /dev/null
+++ b/examples/l2fwd-heartbeat/hbeat.h
@@ -0,0 +1,143 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright 2015 Intel Shannon Ltd. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * @file hbeat.h
+ * DPDK RTE LCore Heartbeat Monitor.
+ *
+ **/
+
+#ifndef _HBEAT_H_
+#define _HBEAT_H_
+
+#include <rte_memory.h>
+
+#ifndef RTE_HBEAT_MAXCORES
+/**
+ * Number of cores to track.
+ * @note Must be larger than the highest core id. */
+#define RTE_HBEAT_MAXCORES RTE_MAX_LCORE
+#endif
+
+
+/**
+ * Heartbeat failure callback.
+ *
+ *  Receives a data pointer passed to hbeat_init() and the id of the
+ *  failed core.
+ */
+typedef void (*rte_heartbeat_failure_callback_t)(
+	void *data,
+	const int id_core);
+
+
+/**
+ * Heartbeat state structure.
+ * @internal
+ */
+struct rte_heartbeat {
+	/** Core-local information */
+	struct {
+		/** Core Liveness. */
+		uint32_t __rte_cache_aligned state_flags;
+		uint64_t last_alive;
+	} core[RTE_HBEAT_MAXCORES];
+
+	/**
+	 * Cores to check.
+	 * Indexed by core id, non-zero if the core should be checked.
+	 */
+	uint8_t active_cores[RTE_HBEAT_MAXCORES];
+
+	/** Dead core handler. */
+	rte_heartbeat_failure_callback_t callback;
+
+	/**
+	 * Dead core handler app data.
+	 * Pointer is passed to dead core handler.
+	 */
+	void *callback_data;
+	uint64_t tsc_initial;
+	uint64_t tsc_mhz;
+};
+
+
+/**
+ * Initialise Heartbeat sub-system.
+ * @param *keepcfg
+ *   Heartbeat structure pointer
+ * @param callback
+ *   Function called upon detection of a dead core.
+ * @param data
+ *   Data pointer to be passed to function callback.
+ * @return
+ *   0 on success, negative value on failure.
+ */
+int  hbeat_init(struct rte_heartbeat *keepcfg,
+	rte_heartbeat_failure_callback_t callback,
+	void *data);
+
+
+/**
+ * @param *ptr_timer Triggering timer
+ * @param *ptr_data  Data pointer (heartbeat structure)
+ */
+void hbeat_dispatch_pings(__attribute__((unused)) struct rte_timer *ptr_timer,
+	void *ptr_data);
+
+
+/**
+ * Registers a core for heartbeat checks.
+ * @param *keepcfg
+ *   Heartbeat structure pointer
+ * @param id_core
+ *   ID number of core to register.
+ */
+void hbeat_register_core(struct rte_heartbeat *keepcfg, const int id_core);
+
+
+/**
+ * Per-core heartbeat check.
+ * @param *keepcfg
+ *   Heartbeat structure pointer
+ *
+ * This function needs to be called from within the main process loop of
+ * the LCore to be checked.
+ */
+static inline void
+hbeat_mark_alive(struct rte_heartbeat *keepcfg)
+{
+	keepcfg->core[rte_lcore_id()].state_flags = 1;
+}
+
+
+#endif /* _HBEAT_H_ */
diff --git a/examples/l2fwd-heartbeat/main.c b/examples/l2fwd-heartbeat/main.c
new file mode 100644
index 0000000..96b6531
--- /dev/null
+++ b/examples/l2fwd-heartbeat/main.c
@@ -0,0 +1,809 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2015 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <inttypes.h>
+#include <sys/types.h>
+#include <sys/queue.h>
+#include <netinet/in.h>
+#include <setjmp.h>
+#include <stdarg.h>
+#include <ctype.h>
+#include <errno.h>
+#include <getopt.h>
+
+#include <rte_common.h>
+#include <rte_log.h>
+#include <rte_memory.h>
+#include <rte_memcpy.h>
+#include <rte_memzone.h>
+#include <rte_eal.h>
+#include <rte_per_lcore.h>
+#include <rte_launch.h>
+#include <rte_atomic.h>
+#include <rte_cycles.h>
+#include <rte_prefetch.h>
+#include <rte_lcore.h>
+#include <rte_per_lcore.h>
+#include <rte_branch_prediction.h>
+#include <rte_interrupts.h>
+#include <rte_pci.h>
+#include <rte_random.h>
+#include <rte_debug.h>
+#include <rte_ether.h>
+#include <rte_ethdev.h>
+#include <rte_ring.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+#include <rte_timer.h>
+
+#include "hbeat.h"
+
+#define RTE_LOGTYPE_L2FWD RTE_LOGTYPE_USER1
+
+#define NB_MBUF   8192
+
+#define MAX_PKT_BURST 32
+#define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */
+
+/*
+ * Configurable number of RX/TX ring descriptors
+ */
+#define RTE_TEST_RX_DESC_DEFAULT 128
+#define RTE_TEST_TX_DESC_DEFAULT 512
+static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT;
+static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT;
+
+/* ethernet addresses of ports */
+static struct ether_addr l2fwd_ports_eth_addr[RTE_MAX_ETHPORTS];
+
+/* mask of enabled ports */
+static uint32_t l2fwd_enabled_port_mask;
+
+/* list of enabled ports */
+static uint32_t l2fwd_dst_ports[RTE_MAX_ETHPORTS];
+
+static unsigned int l2fwd_rx_queue_per_lcore = 1;
+
+struct mbuf_table {
+	unsigned len;
+	struct rte_mbuf *m_table[MAX_PKT_BURST];
+};
+
+#define MAX_RX_QUEUE_PER_LCORE 16
+#define MAX_TX_QUEUE_PER_PORT 16
+struct lcore_queue_conf {
+	unsigned n_rx_port;
+	unsigned rx_port_list[MAX_RX_QUEUE_PER_LCORE];
+	struct mbuf_table tx_mbufs[RTE_MAX_ETHPORTS];
+
+} __rte_cache_aligned;
+struct lcore_queue_conf lcore_queue_conf[RTE_MAX_LCORE];
+
+static const struct rte_eth_conf port_conf = {
+	.rxmode = {
+		.split_hdr_size = 0,
+		.header_split   = 0, /**< Header Split disabled */
+		.hw_ip_checksum = 0, /**< IP checksum offload disabled */
+		.hw_vlan_filter = 0, /**< VLAN filtering disabled */
+		.jumbo_frame    = 0, /**< Jumbo Frame Support disabled */
+		.hw_strip_crc   = 0, /**< CRC stripped by hardware */
+	},
+	.txmode = {
+		.mq_mode = ETH_MQ_TX_NONE,
+	},
+};
+
+struct rte_mempool *l2fwd_pktmbuf_pool = NULL;
+
+/* Per-port statistics struct */
+struct l2fwd_port_statistics {
+	uint64_t tx;
+	uint64_t rx;
+	uint64_t dropped;
+} __rte_cache_aligned;
+struct l2fwd_port_statistics port_statistics[RTE_MAX_ETHPORTS];
+
+/* A tsc-based timer responsible for triggering statistics printout */
+#define TIMER_MILLISECOND 2000000ULL /* around 1ms at 2 Ghz */
+#define MAX_TIMER_PERIOD 86400 /* 1 day max */
+static int64_t timer_period = 10 * 1000; /* default period is 10 seconds */
+static int64_t check_period = 5; /* default check cycle is 5ms */
+
+
+/* Heartbeat structure */
+struct rte_heartbeat rte_global_hbeat_info;
+
+
+
+/* Print out statistics on packets dropped */
+static void
+print_stats(void)
+{
+	uint64_t total_packets_dropped, total_packets_tx, total_packets_rx;
+	unsigned portid;
+
+	total_packets_dropped = 0;
+	total_packets_tx = 0;
+	total_packets_rx = 0;
+
+	const char clr[] = { 27, '[', '2', 'J', '\0' };
+	const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0'};
+
+		/* Clear screen and move to top left */
+	printf("%s%s", clr, topLeft);
+
+	printf("\nPort statistics ====================================");
+
+	for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) {
+		/* skip disabled ports */
+		if ((l2fwd_enabled_port_mask & (1 << portid)) == 0)
+			continue;
+		printf("\nStatistics for port %u ------------------------------"
+			   "\nPackets sent: %24"PRIu64
+			   "\nPackets received: %20"PRIu64
+			   "\nPackets dropped: %21"PRIu64,
+			   portid,
+			   port_statistics[portid].tx,
+			   port_statistics[portid].rx,
+			   port_statistics[portid].dropped);
+
+		total_packets_dropped += port_statistics[portid].dropped;
+		total_packets_tx += port_statistics[portid].tx;
+		total_packets_rx += port_statistics[portid].rx;
+	}
+	printf("\nAggregate statistics ==============================="
+		   "\nTotal packets sent: %18"PRIu64
+		   "\nTotal packets received: %14"PRIu64
+		   "\nTotal packets dropped: %15"PRIu64,
+		   total_packets_tx,
+		   total_packets_rx,
+		   total_packets_dropped);
+	printf("\n====================================================\n");
+}
+
+/* Send the burst of packets on an output interface */
+static int
+l2fwd_send_burst(struct lcore_queue_conf *qconf, unsigned n, uint8_t port)
+{
+	struct rte_mbuf **m_table;
+	unsigned ret;
+	unsigned queueid = 0;
+
+	m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table;
+
+	ret = rte_eth_tx_burst(port, (uint16_t) queueid, m_table, (uint16_t) n);
+	port_statistics[port].tx += ret;
+	if (unlikely(ret < n)) {
+		port_statistics[port].dropped += (n - ret);
+		do {
+			rte_pktmbuf_free(m_table[ret]);
+		} while (++ret < n);
+	}
+
+	return 0;
+}
+
+/* Enqueue packets for TX and prepare them to be sent */
+static int
+l2fwd_send_packet(struct rte_mbuf *m, uint8_t port)
+{
+	unsigned lcore_id, len;
+	struct lcore_queue_conf *qconf;
+
+	lcore_id = rte_lcore_id();
+
+	qconf = &lcore_queue_conf[lcore_id];
+	len = qconf->tx_mbufs[port].len;
+	qconf->tx_mbufs[port].m_table[len] = m;
+	len++;
+
+	/* enough pkts to be sent */
+	if (unlikely(len == MAX_PKT_BURST)) {
+		l2fwd_send_burst(qconf, MAX_PKT_BURST, port);
+		len = 0;
+	}
+
+	qconf->tx_mbufs[port].len = len;
+	return 0;
+}
+
+static void
+l2fwd_simple_forward(struct rte_mbuf *m, unsigned portid)
+{
+	struct ether_hdr *eth;
+	void *tmp;
+	unsigned dst_port;
+
+	dst_port = l2fwd_dst_ports[portid];
+	eth = rte_pktmbuf_mtod(m, struct ether_hdr *);
+
+	/* 02:00:00:00:00:xx */
+	tmp = &eth->d_addr.addr_bytes[0];
+	*((uint64_t *)tmp) = 0x000000000002 + ((uint64_t)dst_port << 40);
+
+	/* src addr */
+	ether_addr_copy(&l2fwd_ports_eth_addr[dst_port], &eth->s_addr);
+
+	l2fwd_send_packet(m, (uint8_t) dst_port);
+}
+
+/* main processing loop */
+static void
+l2fwd_main_loop(void)
+{
+	struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
+	struct rte_mbuf *m;
+	unsigned lcore_id;
+	uint64_t prev_tsc, diff_tsc, cur_tsc;
+	unsigned i, j, portid, nb_rx;
+	struct lcore_queue_conf *qconf;
+	const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) /
+		US_PER_S * BURST_TX_DRAIN_US;
+
+	prev_tsc = 0;
+
+	lcore_id = rte_lcore_id();
+	qconf = &lcore_queue_conf[lcore_id];
+
+	if (qconf->n_rx_port == 0) {
+		RTE_LOG(INFO, L2FWD, "lcore %u has nothing to do\n", lcore_id);
+		return;
+	}
+
+	RTE_LOG(INFO, L2FWD, "entering main loop on lcore %u\n", lcore_id);
+
+	for (i = 0; i < qconf->n_rx_port; i++) {
+
+		portid = qconf->rx_port_list[i];
+		RTE_LOG(INFO, L2FWD, " -- lcoreid=%u portid=%u\n", lcore_id,
+			portid);
+	}
+
+	uint64_t tsc_initial = rte_rdtsc();
+	uint64_t tsc_lifetime = (rand()&0x07) * rte_get_tsc_hz();
+
+	while (1) {
+		/* Keepalive heartbeat */
+		hbeat_mark_alive(&rte_global_hbeat_info);
+
+		cur_tsc = rte_rdtsc();
+
+		/* Die randomly within 7 secs for demo purposes.. */
+		if (cur_tsc - tsc_initial > tsc_lifetime)
+			break;
+
+		/*
+		 * TX burst queue drain
+		 */
+		diff_tsc = cur_tsc - prev_tsc;
+		if (unlikely(diff_tsc > drain_tsc)) {
+
+			for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) {
+				if (qconf->tx_mbufs[portid].len == 0)
+					continue;
+				l2fwd_send_burst(&lcore_queue_conf[lcore_id],
+						 qconf->tx_mbufs[portid].len,
+						 (uint8_t) portid);
+				qconf->tx_mbufs[portid].len = 0;
+			}
+
+			prev_tsc = cur_tsc;
+		}
+
+		/*
+		 * Read packet from RX queues
+		 */
+		for (i = 0; i < qconf->n_rx_port; i++) {
+
+			portid = qconf->rx_port_list[i];
+			nb_rx = rte_eth_rx_burst((uint8_t) portid, 0,
+						 pkts_burst, MAX_PKT_BURST);
+
+			port_statistics[portid].rx += nb_rx;
+
+			for (j = 0; j < nb_rx; j++) {
+				m = pkts_burst[j];
+				rte_prefetch0(rte_pktmbuf_mtod(m, void *));
+				l2fwd_simple_forward(m, portid);
+			}
+		}
+	}
+}
+
+static int
+l2fwd_launch_one_lcore(__attribute__((unused)) void *dummy)
+{
+	l2fwd_main_loop();
+	return 0;
+}
+
+/* display usage */
+static void
+l2fwd_usage(const char *prgname)
+{
+	printf("%s [EAL options] -- -p PORTMASK [-q NQ]\n"
+	       "  -p PORTMASK: hexadecimal bitmask of ports to configure\n"
+	       "  -q NQ: number of queue (=ports) per lcore (default is 1)\n"
+	       "  -K PERIOD: Heartbeat check period (5 default; 86400 max\n)"
+		   "  -T PERIOD: statistics will be refreshed each PERIOD seconds (0 to disable, 10 default, 86400 maximum)\n",
+	       prgname);
+}
+
+static int
+l2fwd_parse_portmask(const char *portmask)
+{
+	char *end = NULL;
+	unsigned long pm;
+
+	/* parse hexadecimal string */
+	pm = strtoul(portmask, &end, 16);
+	if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0'))
+		return -1;
+
+	if (pm == 0)
+		return -1;
+
+	return pm;
+}
+
+static unsigned int
+l2fwd_parse_nqueue(const char *q_arg)
+{
+	char *end = NULL;
+	unsigned long n;
+
+	/* parse hexadecimal string */
+	n = strtoul(q_arg, &end, 10);
+	if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0'))
+		return 0;
+	if (n == 0)
+		return 0;
+	if (n >= MAX_RX_QUEUE_PER_LCORE)
+		return 0;
+
+	return n;
+}
+
+static int
+l2fwd_parse_timer_period(const char *q_arg)
+{
+	char *end = NULL;
+	int n;
+
+	/* parse number string */
+	n = strtol(q_arg, &end, 10);
+	if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0'))
+		return -1;
+	if (n >= MAX_TIMER_PERIOD)
+		return -1;
+
+	return n;
+}
+
+static int
+l2fwd_parse_check_period(const char *q_arg)
+{
+	char *end = NULL;
+	int n;
+
+	/* parse number string */
+	n = strtol(q_arg, &end, 10);
+	if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0'))
+		return -1;
+	if (n >= MAX_TIMER_PERIOD)
+		return -1;
+
+	return n;
+}
+
+/* Parse the argument given in the command line of the application */
+static int
+l2fwd_parse_args(int argc, char **argv)
+{
+	int opt, ret;
+	char **argvopt;
+	int option_index;
+	char *prgname = argv[0];
+	static struct option lgopts[] = {
+		{NULL, 0, 0, 0}
+	};
+
+	argvopt = argv;
+
+	while ((opt = getopt_long(argc, argvopt, "p:q:T:K:",
+				  lgopts, &option_index)) != EOF) {
+
+		switch (opt) {
+		/* portmask */
+		case 'p':
+			l2fwd_enabled_port_mask = l2fwd_parse_portmask(optarg);
+			if (l2fwd_enabled_port_mask == 0) {
+				printf("invalid portmask\n");
+				l2fwd_usage(prgname);
+				return -1;
+			}
+			break;
+
+		/* nqueue */
+		case 'q':
+			l2fwd_rx_queue_per_lcore = l2fwd_parse_nqueue(optarg);
+			if (l2fwd_rx_queue_per_lcore == 0) {
+				printf("invalid queue number\n");
+				l2fwd_usage(prgname);
+				return -1;
+			}
+			break;
+
+		/* timer period */
+		case 'T':
+			timer_period = l2fwd_parse_timer_period(optarg) * 1000;
+			if (timer_period < 0) {
+				printf("invalid timer period\n");
+				l2fwd_usage(prgname);
+				return -1;
+			}
+			break;
+
+		/* Check period */
+		case 'K':
+			check_period = l2fwd_parse_check_period(optarg);
+			if (check_period < 0) {
+				printf("invalid check period\n");
+				l2fwd_usage(prgname);
+				return -1;
+			}
+			break;
+
+		/* long options */
+		case 0:
+			l2fwd_usage(prgname);
+			return -1;
+
+		default:
+			l2fwd_usage(prgname);
+			return -1;
+		}
+	}
+
+	if (optind >= 0)
+		argv[optind-1] = prgname;
+
+	ret = optind-1;
+	optind = 0; /* reset getopt lib */
+	return ret;
+}
+
+/* Check the link status of all ports in up to 9s, and print them finally */
+static void
+check_all_ports_link_status(uint8_t port_num, uint32_t port_mask)
+{
+#define CHECK_INTERVAL 100 /* 100ms */
+#define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
+	uint8_t portid, count, all_ports_up, print_flag = 0;
+	struct rte_eth_link link;
+
+	printf("\nChecking link status");
+	fflush(stdout);
+	for (count = 0; count <= MAX_CHECK_TIME; count++) {
+		all_ports_up = 1;
+		for (portid = 0; portid < port_num; portid++) {
+			if ((port_mask & (1 << portid)) == 0)
+				continue;
+			memset(&link, 0, sizeof(link));
+			rte_eth_link_get_nowait(portid, &link);
+			/* print link status if flag set */
+			if (print_flag == 1) {
+				if (link.link_status)
+					printf("Port %d Link Up - speed %u "
+						"Mbps - %s\n", (uint8_t)portid,
+						(unsigned)link.link_speed,
+				(link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
+					("full-duplex") : ("half-duplex\n"));
+				else
+					printf("Port %d Link Down\n",
+						(uint8_t)portid);
+				continue;
+			}
+			/* clear all_ports_up flag if any link down */
+			if (link.link_status == 0) {
+				all_ports_up = 0;
+				break;
+			}
+		}
+		/* after finally printing all link status, get out */
+		if (print_flag == 1)
+			break;
+
+		if (all_ports_up == 0) {
+			printf(".");
+			fflush(stdout);
+			rte_delay_ms(CHECK_INTERVAL);
+		}
+
+		/* set the print_flag if all ports up or timeout */
+		if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
+			print_flag = 1;
+			printf("done\n");
+		}
+	}
+}
+
+static void
+dead_core(__attribute__((unused)) void *ptr_data, const int id_core)
+{
+	printf("Dead core %i - restarting..\n", id_core);
+	if (rte_eal_get_lcore_state(id_core) == FINISHED) {
+		rte_eal_wait_lcore(id_core);
+		rte_eal_remote_launch(l2fwd_launch_one_lcore, NULL, id_core);
+	} else {
+		printf("..false positive!\n");
+	}
+}
+
+static void
+dispatch_stats(
+	__attribute__((unused)) struct rte_timer *ptr_timer,
+	__attribute__((unused)) void *ptr_data)
+{
+	print_stats();
+}
+
+int
+main(int argc, char **argv)
+{
+	struct lcore_queue_conf *qconf;
+	struct rte_eth_dev_info dev_info;
+	int ret;
+	uint8_t nb_ports;
+	uint8_t nb_ports_available;
+	uint8_t portid, last_port;
+	unsigned lcore_id, rx_lcore_id;
+	unsigned nb_ports_in_mask = 0;
+
+	l2fwd_enabled_port_mask = 0;
+
+
+	/* init EAL */
+	ret = rte_eal_init(argc, argv);
+	if (ret < 0)
+		rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n");
+	argc -= ret;
+	argv += ret;
+
+	/* parse application arguments (after the EAL ones) */
+	ret = l2fwd_parse_args(argc, argv);
+	if (ret < 0)
+		rte_exit(EXIT_FAILURE, "Invalid L2FWD arguments\n");
+
+	/* create the mbuf pool */
+	l2fwd_pktmbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", NB_MBUF, 32,
+		0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
+	if (l2fwd_pktmbuf_pool == NULL)
+		rte_exit(EXIT_FAILURE, "Cannot init mbuf pool\n");
+
+	nb_ports = rte_eth_dev_count();
+	if (nb_ports == 0)
+		rte_exit(EXIT_FAILURE, "No Ethernet ports - bye\n");
+
+	if (nb_ports > RTE_MAX_ETHPORTS)
+		nb_ports = RTE_MAX_ETHPORTS;
+
+	/* reset l2fwd_dst_ports */
+	for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++)
+		l2fwd_dst_ports[portid] = 0;
+	last_port = 0;
+
+	/*
+	 * Each logical core is assigned a dedicated TX queue on each port.
+	 */
+	for (portid = 0; portid < nb_ports; portid++) {
+		/* skip ports that are not enabled */
+		if ((l2fwd_enabled_port_mask & (1 << portid)) == 0)
+			continue;
+
+		if (nb_ports_in_mask % 2) {
+			l2fwd_dst_ports[portid] = last_port;
+			l2fwd_dst_ports[last_port] = portid;
+		} else
+			last_port = portid;
+
+		nb_ports_in_mask++;
+
+		rte_eth_dev_info_get(portid, &dev_info);
+	}
+	if (nb_ports_in_mask % 2) {
+		printf("Notice: odd number of ports in portmask.\n");
+		l2fwd_dst_ports[last_port] = last_port;
+	}
+
+	rx_lcore_id = 1;
+	qconf = NULL;
+
+	/* Initialize the port/queue configuration of each logical core */
+	for (portid = 0; portid < nb_ports; portid++) {
+		/* skip ports that are not enabled */
+		if ((l2fwd_enabled_port_mask & (1 << portid)) == 0)
+			continue;
+
+		/* get the lcore_id for this port */
+		while (rte_lcore_is_enabled(rx_lcore_id) == 0 ||
+		       lcore_queue_conf[rx_lcore_id].n_rx_port ==
+		       l2fwd_rx_queue_per_lcore) {
+			rx_lcore_id++;
+			if (rx_lcore_id >= RTE_MAX_LCORE)
+				rte_exit(EXIT_FAILURE, "Not enough cores\n");
+		}
+
+		if (qconf != &lcore_queue_conf[rx_lcore_id])
+			/* Assigned a new logical core in the loop above. */
+			qconf = &lcore_queue_conf[rx_lcore_id];
+
+		qconf->rx_port_list[qconf->n_rx_port] = portid;
+		qconf->n_rx_port++;
+		printf("Lcore %u: RX port %u\n",
+			rx_lcore_id, (unsigned) portid
+			);
+	}
+
+	nb_ports_available = nb_ports;
+
+	/* Initialise each port */
+	for (portid = 0; portid < nb_ports; portid++) {
+		/* skip ports that are not enabled */
+		if ((l2fwd_enabled_port_mask & (1 << portid)) == 0) {
+			printf("Skipping disabled port %u\n",
+				(unsigned) portid);
+			nb_ports_available--;
+			continue;
+		}
+		/* init port */
+		printf("Initializing port %u... ", (unsigned) portid);
+		fflush(stdout);
+		ret = rte_eth_dev_configure(portid, 1, 1, &port_conf);
+		if (ret < 0)
+			rte_exit(EXIT_FAILURE,
+				"Cannot configure device: err=%d, port=%u\n",
+				ret, (unsigned) portid);
+
+		rte_eth_macaddr_get(portid, &l2fwd_ports_eth_addr[portid]);
+
+		/* init one RX queue */
+		fflush(stdout);
+		ret = rte_eth_rx_queue_setup(portid, 0, nb_rxd,
+					     rte_eth_dev_socket_id(portid),
+					     NULL,
+					     l2fwd_pktmbuf_pool);
+		if (ret < 0)
+			rte_exit(EXIT_FAILURE,
+				"rte_eth_rx_queue_setup:err=%d, port=%u\n",
+				  ret, (unsigned) portid);
+
+		/* init one TX queue on each port */
+		fflush(stdout);
+		ret = rte_eth_tx_queue_setup(portid, 0, nb_txd,
+				rte_eth_dev_socket_id(portid),
+				NULL);
+		if (ret < 0)
+			rte_exit(EXIT_FAILURE,
+				"rte_eth_tx_queue_setup:err=%d, port=%u\n",
+				ret, (unsigned) portid);
+
+		/* Start device */
+		ret = rte_eth_dev_start(portid);
+		if (ret < 0)
+			rte_exit(EXIT_FAILURE,
+				"rte_eth_dev_start:err=%d, port=%u\n",
+				ret, (unsigned) portid);
+
+		rte_eth_promiscuous_enable(portid);
+
+		printf("Port %u,"
+			" MAC address: %02X:%02X:%02X:%02X:%02X:%02X\n\n",
+				(unsigned) portid,
+				l2fwd_ports_eth_addr[portid].addr_bytes[0],
+				l2fwd_ports_eth_addr[portid].addr_bytes[1],
+				l2fwd_ports_eth_addr[portid].addr_bytes[2],
+				l2fwd_ports_eth_addr[portid].addr_bytes[3],
+				l2fwd_ports_eth_addr[portid].addr_bytes[4],
+				l2fwd_ports_eth_addr[portid].addr_bytes[5]);
+
+		/* initialize port stats */
+		memset(&port_statistics, 0, sizeof(port_statistics));
+	}
+
+	if (!nb_ports_available) {
+		rte_exit(EXIT_FAILURE,
+			"All available ports are disabled. Please set portmask.\n");
+	}
+
+	check_all_ports_link_status(nb_ports, l2fwd_enabled_port_mask);
+
+	struct rte_timer hb_timer, stats_timer;
+
+	rte_timer_subsystem_init();
+	if (hbeat_init(&rte_global_hbeat_info, &dead_core, NULL) < 0)
+		rte_exit(EXIT_FAILURE, "init_keep_alive() failed");
+	rte_timer_init(&hb_timer);
+	rte_timer_init(&stats_timer);
+
+
+	if (rte_timer_reset(&hb_timer,
+			(check_period * rte_get_timer_hz()) / 1000,
+			PERIODICAL,
+			rte_lcore_id(),
+			&hbeat_dispatch_pings, &rte_global_hbeat_info
+			) != 0 )
+		rte_exit(EXIT_FAILURE, "Heartbeat setup failure.\n");
+
+	if (timer_period > 0) {
+		if (rte_timer_reset(&stats_timer,
+				(timer_period * rte_get_timer_hz()) / 1000,
+				PERIODICAL,
+				rte_lcore_id(),
+				&dispatch_stats, NULL
+				) != 0 )
+			rte_exit(EXIT_FAILURE, "Stats setup failure.\n");
+	}
+	/* launch per-lcore init on every slave lcore */
+	RTE_LCORE_FOREACH_SLAVE(lcore_id) {
+		struct lcore_queue_conf *qconf = &lcore_queue_conf[lcore_id];
+
+		if (qconf->n_rx_port == 0)
+			RTE_LOG(INFO, L2FWD,
+				"lcore %u has nothing to do\n",
+				lcore_id
+				);
+		else {
+			rte_eal_remote_launch(
+				l2fwd_launch_one_lcore,
+				NULL,
+				lcore_id
+				);
+			hbeat_register_core(&rte_global_hbeat_info, lcore_id);
+		}
+	}
+	for (;;) {
+		rte_timer_manage();
+		rte_delay_ms(5);
+		}
+
+	RTE_LCORE_FOREACH_SLAVE(lcore_id) {
+		if (rte_eal_wait_lcore(lcore_id) < 0)
+			return -1;
+	}
+
+	return 0;
+}
-- 
1.9.3

^ permalink raw reply	[flat|nested] 44+ messages in thread
end of thread, other threads:[~2015-11-19 15:04 UTC | newest]

Thread overview: 44+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2015-09-15 12:16 [dpdk-dev] [RFC PATCH v1] rte: LCore heartbeat example Remy Horton
2015-09-15 13:10 ` Thomas Monjalon
2015-09-23  9:00   ` Remy Horton
2015-10-05 11:16   ` Mcnamara, John
2015-09-30  9:04 ` [dpdk-dev] [PATCH v2 0/3] Keepalive monitoring & reporting Remy Horton
2015-09-30  9:04   ` [dpdk-dev] [PATCH v2 1/3] rte: add keep alive functionality Remy Horton
2015-10-23 11:40     ` Tahhan, Maryam
2015-10-23 14:27     ` Wiles, Keith
2015-10-26 16:36       ` Remy Horton
2015-09-30  9:04   ` [dpdk-dev] [PATCH v2 2/3] l2fwd: keep alive sample application Remy Horton
2015-10-23 14:23     ` Tahhan, Maryam
2015-10-27  8:48       ` Remy Horton
2015-10-27 10:10         ` Bruce Richardson
2015-09-30  9:04   ` [dpdk-dev] [PATCH v2 3/3] docs: add keep alive sample app guide Remy Horton
2015-10-28  8:52   ` [dpdk-dev] [PATCH v3 0/3] Keepalive monitoring & reporting Remy Horton
2015-10-28  8:52     ` [dpdk-dev] [PATCH v3 1/3] rte: add keep alive functionality Remy Horton
2015-10-28 12:25       ` Tahhan, Maryam
2015-11-04  1:48       ` Thomas Monjalon
2015-11-04  1:54       ` Thomas Monjalon
2015-10-28  8:52     ` [dpdk-dev] [PATCH v3 2/3] docs: add keep alive sample app guide Remy Horton
2015-10-28 11:22       ` Van Haaren, Harry
2015-10-28  8:52     ` [dpdk-dev] [PATCH v3 3/3] example: add keep alive sample application Remy Horton
2015-10-28 12:28       ` Tahhan, Maryam
2015-11-05 11:32     ` [dpdk-dev] [PATCH v4 0/3] Keepalive monitoring & reporting Remy Horton
2015-11-05 11:32       ` [dpdk-dev] [PATCH v4 1/3] rte: add keep alive functionality Remy Horton
2015-11-05 16:43         ` Tahhan, Maryam
2015-11-10 14:02         ` Thomas Monjalon
2015-11-11  9:21           ` Remy Horton
2015-11-11 16:28         ` Stephen Hemminger
2015-11-18 10:21           ` Remy Horton
2015-11-13 16:09         ` Thomas Monjalon
2015-11-05 11:32       ` [dpdk-dev] [PATCH v4 2/3] docs: add keep alive sample app guide & release notes Remy Horton
2015-11-05 16:40         ` Van Haaren, Harry
2015-11-05 11:32       ` [dpdk-dev] [PATCH v4 3/3] example: add keep alive sample application Remy Horton
2015-11-05 16:44         ` Tahhan, Maryam
2015-11-11  6:52       ` [dpdk-dev] [PATCH v4 0/3] Keepalive monitoring & reporting Cao, Min
2015-11-18 14:05       ` [dpdk-dev] [PATCH v5 " Remy Horton
2015-11-18 14:05         ` [dpdk-dev] [PATCH v5 1/3] rte: add keep alive functionality Remy Horton
2015-11-18 14:05         ` [dpdk-dev] [PATCH v5 2/3] docs: add keep alive sample app guide & release notes Remy Horton
2015-11-19 11:32           ` Thomas Monjalon
2015-11-19 11:43             ` Remy Horton
2015-11-18 14:05         ` [dpdk-dev] [PATCH v5 3/3] example: add keep alive sample application Remy Horton
2015-11-19 11:31           ` Thomas Monjalon
2015-11-19 14:49         ` [dpdk-dev] [PATCH v5 0/3] Keepalive monitoring & reporting Thomas Monjalon

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).