From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 1346BA00C2; Thu, 17 Nov 2022 06:11:01 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 2255042D61; Thu, 17 Nov 2022 06:10:17 +0100 (CET) Received: from mga07.intel.com (mga07.intel.com [134.134.136.100]) by mails.dpdk.org (Postfix) with ESMTP id 2E71840141 for ; Thu, 17 Nov 2022 06:10:15 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1668661815; x=1700197815; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=Qkb7HlVZ1naKviVdq7cAXa9hv+BILazNiQWu/CklEO0=; b=V9DCFVhZfJJrrf4l46sfdsSyFjH0OI2u7ukQpwXjrqpAy16n5rzISDbm lXmIlJMgMqIz5XBBBpqg/c+h35wr4u8s6YxKbdhJBNDqkjEOQvUWwwu/T RZ760zcgzxOkMmN1xNI+VHlJ6LtNJbth2yYxPO3ONWiUjAzcpLEQD+189 Os/9S5AFXZllES5BR1sJQHGPUI2AUpgdcNrOLqTUghMAE9XJHmxfN1D1L CqMlEZdxoVUh0CnBMqfXmmF7BLN+bvnG/ReogF/B21z7sQsMkKWROuLI7 mMWb/B2UtuxTY2AfrjndMi4N/x10xxT2fHI9FfbfL6uLgVIEd0M0oq7N1 w==; X-IronPort-AV: E=McAfee;i="6500,9779,10533"; a="377026724" X-IronPort-AV: E=Sophos;i="5.96,169,1665471600"; d="scan'208";a="377026724" Received: from orsmga006.jf.intel.com ([10.7.209.51]) by orsmga105.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 16 Nov 2022 21:10:14 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=McAfee;i="6500,9779,10533"; a="617466253" X-IronPort-AV: E=Sophos;i="5.96,169,1665471600"; d="scan'208";a="617466253" Received: from dpdk-zhirun-lmm.sh.intel.com ([10.67.118.230]) by orsmga006.jf.intel.com with ESMTP; 16 Nov 2022 21:10:11 -0800 From: Zhirun Yan To: dev@dpdk.org, jerinj@marvell.com, kirankumark@marvell.com, ndabilpuram@marvell.com Cc: cunming.liang@intel.com, haiyue.wang@intel.com, Zhirun Yan Subject: [PATCH v1 13/13] examples/l3fwd-graph: introduce generic worker model Date: Thu, 17 Nov 2022 13:09:26 +0800 Message-Id: <20221117050926.136974-14-zhirun.yan@intel.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20221117050926.136974-1-zhirun.yan@intel.com> References: <20221117050926.136974-1-zhirun.yan@intel.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Add new parameter "model" to choose generic or rtc worker model. And in generic model, the node will affinity to worker core successively. Note: only support one RX node for remote model in current implementation. ./dpdk-l3fwd-graph -l 8,9,10,11 -n 4 -- -p 0x1 --config="(0,0,9)" -P --model="generic" Signed-off-by: Haiyue Wang Signed-off-by: Cunming Liang Signed-off-by: Zhirun Yan --- examples/l3fwd-graph/main.c | 218 +++++++++++++++++++++++++++++------- 1 file changed, 179 insertions(+), 39 deletions(-) diff --git a/examples/l3fwd-graph/main.c b/examples/l3fwd-graph/main.c index 6dcb6ee92b..c145a3e3e8 100644 --- a/examples/l3fwd-graph/main.c +++ b/examples/l3fwd-graph/main.c @@ -147,6 +147,19 @@ static struct ipv4_l3fwd_lpm_route ipv4_l3fwd_lpm_route_array[] = { {RTE_IPV4(198, 18, 6, 0), 24, 6}, {RTE_IPV4(198, 18, 7, 0), 24, 7}, }; +static int +check_worker_model_params(void) +{ + if (rte_graph_worker_model_get() == RTE_GRAPH_MODEL_GENERIC && + nb_lcore_params > 1) { + printf("Exceeded max number of lcore params for remote model: %hu\n", + nb_lcore_params); + return -1; + } + + return 0; +} + static int check_lcore_params(void) { @@ -291,6 +304,20 @@ parse_max_pkt_len(const char *pktlen) return len; } +static int +parse_worker_model(const char *model) +{ + if (strcmp(model, WORKER_MODEL_DEFAULT) == 0) + return RTE_GRAPH_MODEL_DEFAULT; + else if (strcmp(model, WORKER_MODEL_GENERIC) == 0) { + rte_graph_worker_model_set(RTE_GRAPH_MODEL_GENERIC); + return RTE_GRAPH_MODEL_GENERIC; + } + rte_exit(EXIT_FAILURE, "Invalid worker model: %s", model); + + return RTE_GRAPH_MODEL_MAX; +} + static int parse_portmask(const char *portmask) { @@ -404,6 +431,7 @@ static const char short_options[] = "p:" /* portmask */ #define CMD_LINE_OPT_NO_NUMA "no-numa" #define CMD_LINE_OPT_MAX_PKT_LEN "max-pkt-len" #define CMD_LINE_OPT_PER_PORT_POOL "per-port-pool" +#define CMD_LINE_OPT_WORKER_MODEL "model" enum { /* Long options mapped to a short option */ @@ -416,6 +444,7 @@ enum { CMD_LINE_OPT_NO_NUMA_NUM, CMD_LINE_OPT_MAX_PKT_LEN_NUM, CMD_LINE_OPT_PARSE_PER_PORT_POOL, + CMD_LINE_OPT_WORKER_MODEL_TYPE, }; static const struct option lgopts[] = { @@ -424,6 +453,7 @@ static const struct option lgopts[] = { {CMD_LINE_OPT_NO_NUMA, 0, 0, CMD_LINE_OPT_NO_NUMA_NUM}, {CMD_LINE_OPT_MAX_PKT_LEN, 1, 0, CMD_LINE_OPT_MAX_PKT_LEN_NUM}, {CMD_LINE_OPT_PER_PORT_POOL, 0, 0, CMD_LINE_OPT_PARSE_PER_PORT_POOL}, + {CMD_LINE_OPT_WORKER_MODEL, 1, 0, CMD_LINE_OPT_WORKER_MODEL_TYPE}, {NULL, 0, 0, 0}, }; @@ -498,6 +528,11 @@ parse_args(int argc, char **argv) per_port_pool = 1; break; + case CMD_LINE_OPT_WORKER_MODEL_TYPE: + printf("Use new worker model: %s\n", optarg); + parse_worker_model(optarg); + break; + default: print_usage(prgname); return -1; @@ -735,6 +770,140 @@ config_port_max_pkt_len(struct rte_eth_conf *conf, return 0; } +static void +graph_config_generic(struct rte_graph_param graph_conf) +{ + uint16_t nb_patterns = graph_conf.nb_node_patterns; + int worker_count = rte_lcore_count() - 1; + int main_lcore_id = rte_get_main_lcore(); + int worker_lcore = main_lcore_id; + rte_graph_t main_graph_id = 0; + struct rte_node *node_tmp; + struct lcore_conf *qconf; + struct rte_graph *graph; + rte_graph_t graph_id; + rte_graph_off_t off; + int n_rx_node = 0; + rte_node_t count; + rte_edge_t i; + int ret; + + for (int j = 0; j < nb_lcore_params; j++) { + qconf = &lcore_conf[lcore_params[j].lcore_id]; + /* Add rx node patterns of all lcore */ + for (i = 0; i < qconf->n_rx_queue; i++) { + char *node_name = qconf->rx_queue_list[i].node_name; + + graph_conf.node_patterns[nb_patterns + n_rx_node + i] = node_name; + n_rx_node++; + ret = rte_node_model_generic_set_lcore_affinity(node_name, + lcore_params[j].lcore_id); + if (ret == 0) + printf("Set node %s affinity to lcore %u\n", node_name, + lcore_params[j].lcore_id); + } + } + + graph_conf.nb_node_patterns = nb_patterns + n_rx_node; + graph_conf.socket_id = rte_lcore_to_socket_id(main_lcore_id); + + snprintf(qconf->name, sizeof(qconf->name), "worker_%u", + main_lcore_id); + + /* create main graph */ + main_graph_id = rte_graph_create(qconf->name, &graph_conf); + if (main_graph_id == RTE_GRAPH_ID_INVALID) + rte_exit(EXIT_FAILURE, + "rte_graph_create(): main_graph_id invalid for lcore %u\n", + main_lcore_id); + + qconf->graph_id = main_graph_id; + qconf->graph = rte_graph_lookup(qconf->name); + /* >8 End of graph initialization. */ + if (!qconf->graph) + rte_exit(EXIT_FAILURE, + "rte_graph_lookup(): graph %s not found\n", + qconf->name); + + graph = qconf->graph; + rte_graph_foreach_node(count, off, graph, node_tmp) { + worker_lcore = rte_get_next_lcore(worker_lcore, true, 1); + + /* Need to set the node Lcore affinity before clone graph for each lcore */ + if (node_tmp->lcore_id == RTE_MAX_LCORE) { + ret = rte_node_model_generic_set_lcore_affinity(node_tmp->name, + worker_lcore); + if (ret == 0) + printf("Set node %s affinity to lcore %u\n", + node_tmp->name, worker_lcore); + } + } + + worker_lcore = main_lcore_id; + for (int i = 0; i < worker_count; i++) { + worker_lcore = rte_get_next_lcore(worker_lcore, true, 1); + + qconf = &lcore_conf[worker_lcore]; + snprintf(qconf->name, sizeof(qconf->name), "cloned-%u", worker_lcore); + graph_id = rte_graph_clone(main_graph_id, qconf->name); + ret = rte_graph_bind_core(graph_id, worker_lcore); + if (ret == 0) + printf("bind graph %d to lcore %u\n", graph_id, worker_lcore); + + /* full cloned graph name */ + snprintf(qconf->name, sizeof(qconf->name), "%s", + rte_graph_id_to_name(graph_id)); + qconf->graph_id = graph_id; + qconf->graph = rte_graph_lookup(qconf->name); + if (!qconf->graph) + rte_exit(EXIT_FAILURE, + "Failed to lookup graph %s\n", + qconf->name); + continue; + } +} + +static void +graph_config_rtc(struct rte_graph_param graph_conf) +{ + uint16_t nb_patterns = graph_conf.nb_node_patterns; + struct lcore_conf *qconf; + rte_graph_t graph_id; + uint32_t lcore_id; + rte_edge_t i; + + for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { + if (rte_lcore_is_enabled(lcore_id) == 0) + continue; + + qconf = &lcore_conf[lcore_id]; + /* Skip graph creation if no source exists */ + if (!qconf->n_rx_queue) + continue; + /* Add rx node patterns of this lcore */ + for (i = 0; i < qconf->n_rx_queue; i++) { + graph_conf.node_patterns[nb_patterns + i] = + qconf->rx_queue_list[i].node_name; + } + graph_conf.nb_node_patterns = nb_patterns + i; + graph_conf.socket_id = rte_lcore_to_socket_id(lcore_id); + snprintf(qconf->name, sizeof(qconf->name), "worker_%u", + lcore_id); + graph_id = rte_graph_create(qconf->name, &graph_conf); + if (graph_id == RTE_GRAPH_ID_INVALID) + rte_exit(EXIT_FAILURE, + "rte_graph_create(): graph_id invalid for lcore %u\n", + lcore_id); + qconf->graph_id = graph_id; + qconf->graph = rte_graph_lookup(qconf->name); + /* >8 End of graph initialization. */ + if (!qconf->graph) + rte_exit(EXIT_FAILURE, + "rte_graph_lookup(): graph %s not found\n", + qconf->name); + } +} + int main(int argc, char **argv) { @@ -759,6 +928,7 @@ main(int argc, char **argv) uint16_t nb_patterns; uint8_t rewrite_len; uint32_t lcore_id; + uint16_t model; int ret; /* Init EAL */ @@ -787,6 +957,9 @@ main(int argc, char **argv) if (check_lcore_params() < 0) rte_exit(EXIT_FAILURE, "check_lcore_params() failed\n"); + if (check_worker_model_params() < 0) + rte_exit(EXIT_FAILURE, "check_worker_model_params() failed\n"); + ret = init_lcore_rx_queues(); if (ret < 0) rte_exit(EXIT_FAILURE, "init_lcore_rx_queues() failed\n"); @@ -1026,46 +1199,13 @@ main(int argc, char **argv) memset(&graph_conf, 0, sizeof(graph_conf)); graph_conf.node_patterns = node_patterns; + graph_conf.nb_node_patterns = nb_patterns; - for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { - rte_graph_t graph_id; - rte_edge_t i; - - if (rte_lcore_is_enabled(lcore_id) == 0) - continue; - - qconf = &lcore_conf[lcore_id]; - - /* Skip graph creation if no source exists */ - if (!qconf->n_rx_queue) - continue; - - /* Add rx node patterns of this lcore */ - for (i = 0; i < qconf->n_rx_queue; i++) { - graph_conf.node_patterns[nb_patterns + i] = - qconf->rx_queue_list[i].node_name; - } - - graph_conf.nb_node_patterns = nb_patterns + i; - graph_conf.socket_id = rte_lcore_to_socket_id(lcore_id); - - snprintf(qconf->name, sizeof(qconf->name), "worker_%u", - lcore_id); - - graph_id = rte_graph_create(qconf->name, &graph_conf); - if (graph_id == RTE_GRAPH_ID_INVALID) - rte_exit(EXIT_FAILURE, - "rte_graph_create(): graph_id invalid" - " for lcore %u\n", lcore_id); - - qconf->graph_id = graph_id; - qconf->graph = rte_graph_lookup(qconf->name); - /* >8 End of graph initialization. */ - if (!qconf->graph) - rte_exit(EXIT_FAILURE, - "rte_graph_lookup(): graph %s not found\n", - qconf->name); - } + model = rte_graph_worker_model_get(); + if (model == RTE_GRAPH_MODEL_DEFAULT) + graph_config_rtc(graph_conf); + else if (model == RTE_GRAPH_MODEL_GENERIC) + graph_config_generic(graph_conf); memset(&rewrite_data, 0, sizeof(rewrite_data)); rewrite_len = sizeof(rewrite_data); -- 2.25.1