From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 9FAD54281E; Fri, 24 Mar 2023 04:17:59 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 75D8B42D4F; Fri, 24 Mar 2023 04:17:02 +0100 (CET) Received: from mga06.intel.com (mga06b.intel.com [134.134.136.31]) by mails.dpdk.org (Postfix) with ESMTP id 87DA342D52 for ; Fri, 24 Mar 2023 04:17:00 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1679627820; x=1711163820; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=eHLXI4NTC2tnJ48tUvIBBjy5etf2ju7CP7f28DoE7vk=; b=G8DG8+ioKKXHipPmxDnI5RqvHBgVB4BwuzvdBtNTSoaXyeX+n/07yumT Gko1fX/IJIa9aP91uE4ZnJyOk/SslaKsjvaoXb2/ziF+n6yHOvwJ7xG+v UaFwa/0t8Hk4S8Pg1iYEWeH0pj0Tgmbkaw3uhyEbmP+qFOxoP8WBQFWRW iRdf7UyaAykJvmx8iHtlmj/s/6Qfoyxs0eqY+C7yX89JNDTaR9ChcN79X oRtToukLenSAltBT74SZoqGbrItCkfkSw3GM8HeFQANBW2xrDfkF78DbZ WoF0oUO1VpAhMeqqJ63/ejyYQNCc8IXhHEV7bQCZmUXLABjDKPVRAbNTm g==; X-IronPort-AV: E=McAfee;i="6600,9927,10658"; a="402266491" X-IronPort-AV: E=Sophos;i="5.98,286,1673942400"; d="scan'208";a="402266491" Received: from fmsmga007.fm.intel.com ([10.253.24.52]) by orsmga104.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 23 Mar 2023 20:17:00 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=McAfee;i="6600,9927,10658"; a="684998596" X-IronPort-AV: E=Sophos;i="5.98,286,1673942400"; d="scan'208";a="684998596" Received: from dpdk-zhirun-lmm.sh.intel.com ([10.67.119.68]) by fmsmga007.fm.intel.com with ESMTP; 23 Mar 2023 20:16:57 -0700 From: Zhirun Yan To: dev@dpdk.org, jerinj@marvell.com, kirankumark@marvell.com, ndabilpuram@marvell.com Cc: cunming.liang@intel.com, haiyue.wang@intel.com, Zhirun Yan Subject: [PATCH v2 14/15] examples/l3fwd-graph: introduce multicore dispatch worker model Date: Fri, 24 Mar 2023 11:16:21 +0900 Message-Id: <20230324021622.1369006-15-zhirun.yan@intel.com> X-Mailer: git-send-email 2.37.2 In-Reply-To: <20230324021622.1369006-1-zhirun.yan@intel.com> References: <20221117050926.136974-1-zhirun.yan@intel.com> <20230324021622.1369006-1-zhirun.yan@intel.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Add new parameter "model" to choose dispatch or rtc worker model. And in dispatch model, the node will affinity to worker core successively. Note: only support one RX node for remote model in current implementation. ./dpdk-l3fwd-graph -l 8,9,10,11 -n 4 -- -p 0x1 --config="(0,0,9)" -P --model="dispatch" Signed-off-by: Haiyue Wang Signed-off-by: Cunming Liang Signed-off-by: Zhirun Yan --- examples/l3fwd-graph/main.c | 237 +++++++++++++++++++++++++++++------- 1 file changed, 195 insertions(+), 42 deletions(-) diff --git a/examples/l3fwd-graph/main.c b/examples/l3fwd-graph/main.c index 5feeab4f0f..cfa78003f4 100644 --- a/examples/l3fwd-graph/main.c +++ b/examples/l3fwd-graph/main.c @@ -55,6 +55,9 @@ #define NB_SOCKETS 8 +/* Graph module */ +#define WORKER_MODEL_RTC "rtc" +#define WORKER_MODEL_MCORE_DISPATCH "dispatch" /* Static global variables used within this file. */ static uint16_t nb_rxd = RX_DESC_DEFAULT; static uint16_t nb_txd = TX_DESC_DEFAULT; @@ -88,6 +91,10 @@ struct lcore_rx_queue { char node_name[RTE_NODE_NAMESIZE]; }; +struct model_conf { + enum rte_graph_worker_model model; +}; + /* Lcore conf */ struct lcore_conf { uint16_t n_rx_queue; @@ -153,6 +160,19 @@ static struct ipv4_l3fwd_lpm_route ipv4_l3fwd_lpm_route_array[] = { {RTE_IPV4(198, 18, 6, 0), 24, 6}, {RTE_IPV4(198, 18, 7, 0), 24, 7}, }; +static int +check_worker_model_params(void) +{ + if (rte_graph_worker_model_get() == RTE_GRAPH_MODEL_MCORE_DISPATCH && + nb_lcore_params > 1) { + printf("Exceeded max number of lcore params for remote model: %hu\n", + nb_lcore_params); + return -1; + } + + return 0; +} + static int check_lcore_params(void) { @@ -276,6 +296,7 @@ print_usage(const char *prgname) " --eth-dest=X,MM:MM:MM:MM:MM:MM: Ethernet destination for " "port X\n" " --max-pkt-len PKTLEN: maximum packet length in decimal (64-9600)\n" + " --model NAME: walking model name, dispatch or rtc(by default)\n" " --no-numa: Disable numa awareness\n" " --per-port-pool: Use separate buffer pool per port\n" " --pcap-enable: Enables pcap capture\n" @@ -318,6 +339,20 @@ parse_max_pkt_len(const char *pktlen) return len; } +static int +parse_worker_model(const char *model) +{ + if (strcmp(model, WORKER_MODEL_MCORE_DISPATCH) == 0) { + rte_graph_worker_model_set(RTE_GRAPH_MODEL_MCORE_DISPATCH); + return RTE_GRAPH_MODEL_MCORE_DISPATCH; + } else if (strcmp(model, WORKER_MODEL_RTC) == 0) + return RTE_GRAPH_MODEL_RTC; + + rte_exit(EXIT_FAILURE, "Invalid worker model: %s", model); + + return RTE_GRAPH_MODEL_LIST_END; +} + static int parse_portmask(const char *portmask) { @@ -434,6 +469,8 @@ static const char short_options[] = "p:" /* portmask */ #define CMD_LINE_OPT_PCAP_ENABLE "pcap-enable" #define CMD_LINE_OPT_NUM_PKT_CAP "pcap-num-cap" #define CMD_LINE_OPT_PCAP_FILENAME "pcap-file-name" +#define CMD_LINE_OPT_WORKER_MODEL "model" + enum { /* Long options mapped to a short option */ @@ -449,6 +486,7 @@ enum { CMD_LINE_OPT_PARSE_PCAP_ENABLE, CMD_LINE_OPT_PARSE_NUM_PKT_CAP, CMD_LINE_OPT_PCAP_FILENAME_CAP, + CMD_LINE_OPT_WORKER_MODEL_TYPE, }; static const struct option lgopts[] = { @@ -460,6 +498,7 @@ static const struct option lgopts[] = { {CMD_LINE_OPT_PCAP_ENABLE, 0, 0, CMD_LINE_OPT_PARSE_PCAP_ENABLE}, {CMD_LINE_OPT_NUM_PKT_CAP, 1, 0, CMD_LINE_OPT_PARSE_NUM_PKT_CAP}, {CMD_LINE_OPT_PCAP_FILENAME, 1, 0, CMD_LINE_OPT_PCAP_FILENAME_CAP}, + {CMD_LINE_OPT_WORKER_MODEL, 1, 0, CMD_LINE_OPT_WORKER_MODEL_TYPE}, {NULL, 0, 0, 0}, }; @@ -551,6 +590,11 @@ parse_args(int argc, char **argv) printf("Pcap file name: %s\n", pcap_filename); break; + case CMD_LINE_OPT_WORKER_MODEL_TYPE: + printf("Use new worker model: %s\n", optarg); + parse_worker_model(optarg); + break; + default: print_usage(prgname); return -1; @@ -726,15 +770,15 @@ print_stats(void) static int graph_main_loop(void *conf) { + struct model_conf *mconf = conf; struct lcore_conf *qconf; struct rte_graph *graph; uint32_t lcore_id; - RTE_SET_USED(conf); - lcore_id = rte_lcore_id(); qconf = &lcore_conf[lcore_id]; graph = qconf->graph; + rte_graph_worker_model_set(mconf->model); if (!graph) { RTE_LOG(INFO, L3FWD_GRAPH, "Lcore %u has nothing to do\n", @@ -788,6 +832,141 @@ config_port_max_pkt_len(struct rte_eth_conf *conf, return 0; } +static void +graph_config_mcore_dispatch(struct rte_graph_param graph_conf) +{ + uint16_t nb_patterns = graph_conf.nb_node_patterns; + int worker_count = rte_lcore_count() - 1; + int main_lcore_id = rte_get_main_lcore(); + int worker_lcore = main_lcore_id; + rte_graph_t main_graph_id = 0; + struct rte_node *node_tmp; + struct lcore_conf *qconf; + struct rte_graph *graph; + rte_graph_t graph_id; + rte_graph_off_t off; + int n_rx_node = 0; + rte_node_t count; + int i, j; + int ret; + + for (j = 0; j < nb_lcore_params; j++) { + qconf = &lcore_conf[lcore_params[j].lcore_id]; + /* Add rx node patterns of all lcore */ + for (i = 0; i < qconf->n_rx_queue; i++) { + char *node_name = qconf->rx_queue_list[i].node_name; + + graph_conf.node_patterns[nb_patterns + n_rx_node + i] = node_name; + n_rx_node++; + ret = rte_graph_model_dispatch_lcore_affinity_set(node_name, + lcore_params[j].lcore_id); + if (ret == 0) + printf("Set node %s affinity to lcore %u\n", node_name, + lcore_params[j].lcore_id); + } + } + + graph_conf.nb_node_patterns = nb_patterns + n_rx_node; + graph_conf.socket_id = rte_lcore_to_socket_id(main_lcore_id); + + qconf = &lcore_conf[main_lcore_id]; + snprintf(qconf->name, sizeof(qconf->name), "worker_%u", + main_lcore_id); + + /* create main graph */ + main_graph_id = rte_graph_create(qconf->name, &graph_conf); + if (main_graph_id == RTE_GRAPH_ID_INVALID) + rte_exit(EXIT_FAILURE, + "rte_graph_create(): main_graph_id invalid for lcore %u\n", + main_lcore_id); + + qconf->graph_id = main_graph_id; + qconf->graph = rte_graph_lookup(qconf->name); + /* >8 End of graph initialization. */ + if (!qconf->graph) + rte_exit(EXIT_FAILURE, + "rte_graph_lookup(): graph %s not found\n", + qconf->name); + + graph = qconf->graph; + rte_graph_foreach_node(count, off, graph, node_tmp) { + worker_lcore = rte_get_next_lcore(worker_lcore, true, 1); + + /* Need to set the node Lcore affinity before clone graph for each lcore */ + if (node_tmp->lcore_id == RTE_MAX_LCORE) { + ret = rte_graph_model_dispatch_lcore_affinity_set(node_tmp->name, + worker_lcore); + if (ret == 0) + printf("Set node %s affinity to lcore %u\n", + node_tmp->name, worker_lcore); + } + } + + worker_lcore = main_lcore_id; + for (i = 0; i < worker_count; i++) { + worker_lcore = rte_get_next_lcore(worker_lcore, true, 1); + + qconf = &lcore_conf[worker_lcore]; + snprintf(qconf->name, sizeof(qconf->name), "cloned-%u", worker_lcore); + graph_id = rte_graph_clone(main_graph_id, qconf->name); + ret = rte_graph_model_dispatch_core_bind(graph_id, worker_lcore); + if (ret == 0) + printf("bind graph %d to lcore %u\n", graph_id, worker_lcore); + + /* full cloned graph name */ + snprintf(qconf->name, sizeof(qconf->name), "%s", + rte_graph_id_to_name(graph_id)); + qconf->graph_id = graph_id; + qconf->graph = rte_graph_lookup(qconf->name); + if (!qconf->graph) + rte_exit(EXIT_FAILURE, + "Failed to lookup graph %s\n", + qconf->name); + continue; + } +} + +static void +graph_config_rtc(struct rte_graph_param graph_conf) +{ + uint16_t nb_patterns = graph_conf.nb_node_patterns; + struct lcore_conf *qconf; + rte_graph_t graph_id; + uint32_t lcore_id; + rte_edge_t i; + + for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { + if (rte_lcore_is_enabled(lcore_id) == 0) + continue; + + qconf = &lcore_conf[lcore_id]; + /* Skip graph creation if no source exists */ + if (!qconf->n_rx_queue) + continue; + /* Add rx node patterns of this lcore */ + for (i = 0; i < qconf->n_rx_queue; i++) { + graph_conf.node_patterns[nb_patterns + i] = + qconf->rx_queue_list[i].node_name; + } + graph_conf.nb_node_patterns = nb_patterns + i; + graph_conf.socket_id = rte_lcore_to_socket_id(lcore_id); + snprintf(qconf->name, sizeof(qconf->name), "worker_%u", + lcore_id); + graph_id = rte_graph_create(qconf->name, &graph_conf); + if (graph_id == RTE_GRAPH_ID_INVALID) + rte_exit(EXIT_FAILURE, + "rte_graph_create(): graph_id invalid for lcore %u\n", + lcore_id); + qconf->graph_id = graph_id; + qconf->graph = rte_graph_lookup(qconf->name); + /* >8 End of graph initialization. */ + if (!qconf->graph) + rte_exit(EXIT_FAILURE, + "rte_graph_lookup(): graph %s not found\n", + qconf->name); + } +} + int main(int argc, char **argv) { @@ -808,10 +987,12 @@ main(int argc, char **argv) uint16_t queueid, portid, i; const char **node_patterns; struct lcore_conf *qconf; + struct model_conf mconf; uint16_t nb_graphs = 0; uint16_t nb_patterns; uint8_t rewrite_len; uint32_t lcore_id; + uint16_t model; int ret; /* Init EAL */ @@ -840,6 +1021,9 @@ main(int argc, char **argv) if (check_lcore_params() < 0) rte_exit(EXIT_FAILURE, "check_lcore_params() failed\n"); + if (check_worker_model_params() < 0) + rte_exit(EXIT_FAILURE, "check_worker_model_params() failed\n"); + ret = init_lcore_rx_queues(); if (ret < 0) rte_exit(EXIT_FAILURE, "init_lcore_rx_queues() failed\n"); @@ -1079,51 +1263,18 @@ main(int argc, char **argv) memset(&graph_conf, 0, sizeof(graph_conf)); graph_conf.node_patterns = node_patterns; + graph_conf.nb_node_patterns = nb_patterns; /* Pcap config */ graph_conf.pcap_enable = pcap_trace_enable; graph_conf.num_pkt_to_capture = packet_to_capture; graph_conf.pcap_filename = pcap_filename; - for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { - rte_graph_t graph_id; - rte_edge_t i; - - if (rte_lcore_is_enabled(lcore_id) == 0) - continue; - - qconf = &lcore_conf[lcore_id]; - - /* Skip graph creation if no source exists */ - if (!qconf->n_rx_queue) - continue; - - /* Add rx node patterns of this lcore */ - for (i = 0; i < qconf->n_rx_queue; i++) { - graph_conf.node_patterns[nb_patterns + i] = - qconf->rx_queue_list[i].node_name; - } - - graph_conf.nb_node_patterns = nb_patterns + i; - graph_conf.socket_id = rte_lcore_to_socket_id(lcore_id); - - snprintf(qconf->name, sizeof(qconf->name), "worker_%u", - lcore_id); - - graph_id = rte_graph_create(qconf->name, &graph_conf); - if (graph_id == RTE_GRAPH_ID_INVALID) - rte_exit(EXIT_FAILURE, - "rte_graph_create(): graph_id invalid" - " for lcore %u\n", lcore_id); - - qconf->graph_id = graph_id; - qconf->graph = rte_graph_lookup(qconf->name); - /* >8 End of graph initialization. */ - if (!qconf->graph) - rte_exit(EXIT_FAILURE, - "rte_graph_lookup(): graph %s not found\n", - qconf->name); - } + model = rte_graph_worker_model_get(); + if (model == RTE_GRAPH_MODEL_MCORE_DISPATCH) + graph_config_mcore_dispatch(graph_conf); + else + graph_config_rtc(graph_conf); memset(&rewrite_data, 0, sizeof(rewrite_data)); rewrite_len = sizeof(rewrite_data); @@ -1174,8 +1325,10 @@ main(int argc, char **argv) } /* >8 End of adding route to ip4 graph infa. */ + mconf.model = model; /* Launch per-lcore init on every worker lcore */ - rte_eal_mp_remote_launch(graph_main_loop, NULL, SKIP_MAIN); + rte_eal_mp_remote_launch(graph_main_loop, &mconf, + SKIP_MAIN); /* Accumulate and print stats on main until exit */ if (rte_graph_has_stats_feature()) -- 2.37.2