Re: [PATCH v1 13/13] examples/l3fwd-graph: introduce generic worker model

[Jerin Jacob <jerinjacobk@gmail.com>]

On Thu, Nov 17, 2022 at 10:41 AM Zhirun Yan <zhirun.yan@intel.com> wrote:
>
> Add new parameter "model" to choose generic or rtc worker model.
> And in generic model, the node will affinity to worker core successively.
>
> Note:
> only support one RX node for remote model in current implementation.
>
> ./dpdk-l3fwd-graph  -l 8,9,10,11 -n 4 -- -p 0x1 --config="(0,0,9)" -P
> --model="generic"

Patch apply issue, please rebase with main.
See https://patches.dpdk.org/project/dpdk/patch/20221117050926.136974-14-zhirun.yan@intel.com/

>
> Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
> Signed-off-by: Cunming Liang <cunming.liang@intel.com>
> Signed-off-by: Zhirun Yan <zhirun.yan@intel.com>
> ---
>  examples/l3fwd-graph/main.c | 218 +++++++++++++++++++++++++++++-------
>  1 file changed, 179 insertions(+), 39 deletions(-)
>
> diff --git a/examples/l3fwd-graph/main.c b/examples/l3fwd-graph/main.c
> index 6dcb6ee92b..c145a3e3e8 100644
> --- a/examples/l3fwd-graph/main.c
> +++ b/examples/l3fwd-graph/main.c
> @@ -147,6 +147,19 @@ static struct ipv4_l3fwd_lpm_route ipv4_l3fwd_lpm_route_array[] = {
>         {RTE_IPV4(198, 18, 6, 0), 24, 6}, {RTE_IPV4(198, 18, 7, 0), 24, 7},
>  };
>
> +static int
> +check_worker_model_params(void)
> +{
> +       if (rte_graph_worker_model_get() == RTE_GRAPH_MODEL_GENERIC &&
> +           nb_lcore_params > 1) {
> +               printf("Exceeded max number of lcore params for remote model: %hu\n",
> +                      nb_lcore_params);
> +               return -1;
> +       }
> +
> +       return 0;
> +}
> +
>  static int
>  check_lcore_params(void)
>  {
> @@ -291,6 +304,20 @@ parse_max_pkt_len(const char *pktlen)
>         return len;
>  }
>
> +static int
> +parse_worker_model(const char *model)
> +{
> +       if (strcmp(model, WORKER_MODEL_DEFAULT) == 0)
> +               return RTE_GRAPH_MODEL_DEFAULT;
> +       else if (strcmp(model, WORKER_MODEL_GENERIC) == 0) {
> +               rte_graph_worker_model_set(RTE_GRAPH_MODEL_GENERIC);
> +               return RTE_GRAPH_MODEL_GENERIC;
> +       }
> +       rte_exit(EXIT_FAILURE, "Invalid worker model: %s", model);
> +
> +       return RTE_GRAPH_MODEL_MAX;
> +}
> +
>  static int
>  parse_portmask(const char *portmask)
>  {
> @@ -404,6 +431,7 @@ static const char short_options[] = "p:" /* portmask */
>  #define CMD_LINE_OPT_NO_NUMA      "no-numa"
>  #define CMD_LINE_OPT_MAX_PKT_LEN   "max-pkt-len"
>  #define CMD_LINE_OPT_PER_PORT_POOL "per-port-pool"
> +#define CMD_LINE_OPT_WORKER_MODEL  "model"
>  enum {
>         /* Long options mapped to a short option */
>
> @@ -416,6 +444,7 @@ enum {
>         CMD_LINE_OPT_NO_NUMA_NUM,
>         CMD_LINE_OPT_MAX_PKT_LEN_NUM,
>         CMD_LINE_OPT_PARSE_PER_PORT_POOL,
> +       CMD_LINE_OPT_WORKER_MODEL_TYPE,
>  };
>
>  static const struct option lgopts[] = {
> @@ -424,6 +453,7 @@ static const struct option lgopts[] = {
>         {CMD_LINE_OPT_NO_NUMA, 0, 0, CMD_LINE_OPT_NO_NUMA_NUM},
>         {CMD_LINE_OPT_MAX_PKT_LEN, 1, 0, CMD_LINE_OPT_MAX_PKT_LEN_NUM},
>         {CMD_LINE_OPT_PER_PORT_POOL, 0, 0, CMD_LINE_OPT_PARSE_PER_PORT_POOL},
> +       {CMD_LINE_OPT_WORKER_MODEL, 1, 0, CMD_LINE_OPT_WORKER_MODEL_TYPE},
>         {NULL, 0, 0, 0},
>  };
>
> @@ -498,6 +528,11 @@ parse_args(int argc, char **argv)
>                         per_port_pool = 1;
>                         break;
>
> +               case CMD_LINE_OPT_WORKER_MODEL_TYPE:
> +                       printf("Use new worker model: %s\n", optarg);
> +                       parse_worker_model(optarg);
> +                       break;
> +
>                 default:
>                         print_usage(prgname);
>                         return -1;
> @@ -735,6 +770,140 @@ config_port_max_pkt_len(struct rte_eth_conf *conf,
>         return 0;
>  }
>
> +static void
> +graph_config_generic(struct rte_graph_param graph_conf)
> +{
> +       uint16_t nb_patterns = graph_conf.nb_node_patterns;
> +       int worker_count = rte_lcore_count() - 1;
> +       int main_lcore_id = rte_get_main_lcore();
> +       int worker_lcore = main_lcore_id;
> +       rte_graph_t main_graph_id = 0;
> +       struct rte_node *node_tmp;
> +       struct lcore_conf *qconf;
> +       struct rte_graph *graph;
> +       rte_graph_t graph_id;
> +       rte_graph_off_t off;
> +       int n_rx_node = 0;
> +       rte_node_t count;
> +       rte_edge_t i;
> +       int ret;
> +
> +       for (int j = 0; j < nb_lcore_params; j++) {
> +               qconf = &lcore_conf[lcore_params[j].lcore_id];
> +               /* Add rx node patterns of all lcore */
> +               for (i = 0; i < qconf->n_rx_queue; i++) {
> +                       char *node_name = qconf->rx_queue_list[i].node_name;
> +
> +                       graph_conf.node_patterns[nb_patterns + n_rx_node + i] = node_name;
> +                       n_rx_node++;
> +                       ret = rte_node_model_generic_set_lcore_affinity(node_name,
> +                                                                       lcore_params[j].lcore_id);
> +                       if (ret == 0)
> +                               printf("Set node %s affinity to lcore %u\n", node_name,
> +                                      lcore_params[j].lcore_id);
> +               }
> +       }
> +
> +       graph_conf.nb_node_patterns = nb_patterns + n_rx_node;
> +       graph_conf.socket_id = rte_lcore_to_socket_id(main_lcore_id);
> +
> +       snprintf(qconf->name, sizeof(qconf->name), "worker_%u",
> +                main_lcore_id);
> +
> +       /* create main graph */
> +       main_graph_id = rte_graph_create(qconf->name, &graph_conf);
> +       if (main_graph_id == RTE_GRAPH_ID_INVALID)
> +               rte_exit(EXIT_FAILURE,
> +                        "rte_graph_create(): main_graph_id invalid for lcore %u\n",
> +                        main_lcore_id);
> +
> +       qconf->graph_id = main_graph_id;
> +       qconf->graph = rte_graph_lookup(qconf->name);
> +       /* >8 End of graph initialization. */
> +       if (!qconf->graph)
> +               rte_exit(EXIT_FAILURE,
> +                        "rte_graph_lookup(): graph %s not found\n",
> +                        qconf->name);
> +
> +       graph = qconf->graph;
> +       rte_graph_foreach_node(count, off, graph, node_tmp) {
> +               worker_lcore = rte_get_next_lcore(worker_lcore, true, 1);
> +
> +               /* Need to set the node Lcore affinity before clone graph for each lcore */
> +               if (node_tmp->lcore_id == RTE_MAX_LCORE) {
> +                       ret = rte_node_model_generic_set_lcore_affinity(node_tmp->name,
> +                                                                       worker_lcore);
> +                       if (ret == 0)
> +                               printf("Set node %s affinity to lcore %u\n",
> +                                      node_tmp->name, worker_lcore);
> +               }
> +       }
> +
> +       worker_lcore = main_lcore_id;
> +       for (int i = 0; i < worker_count; i++) {
> +               worker_lcore = rte_get_next_lcore(worker_lcore, true, 1);
> +
> +               qconf = &lcore_conf[worker_lcore];
> +               snprintf(qconf->name, sizeof(qconf->name), "cloned-%u", worker_lcore);
> +               graph_id = rte_graph_clone(main_graph_id, qconf->name);
> +               ret = rte_graph_bind_core(graph_id, worker_lcore);
> +               if (ret == 0)
> +                       printf("bind graph %d to lcore %u\n", graph_id, worker_lcore);
> +
> +               /* full cloned graph name */
> +               snprintf(qconf->name, sizeof(qconf->name), "%s",
> +                        rte_graph_id_to_name(graph_id));
> +               qconf->graph_id = graph_id;
> +               qconf->graph = rte_graph_lookup(qconf->name);
> +               if (!qconf->graph)
> +                       rte_exit(EXIT_FAILURE,
> +                                "Failed to lookup graph %s\n",
> +                                qconf->name);
> +               continue;
> +       }
> +}
> +
> +static void
> +graph_config_rtc(struct rte_graph_param graph_conf)
> +{
> +       uint16_t nb_patterns = graph_conf.nb_node_patterns;
> +       struct lcore_conf *qconf;
> +       rte_graph_t graph_id;
> +       uint32_t lcore_id;
> +       rte_edge_t i;
> +
> +       for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
> +               if (rte_lcore_is_enabled(lcore_id) == 0)
> +                       continue;
> +
> +               qconf = &lcore_conf[lcore_id];
> +               /* Skip graph creation if no source exists */
> +               if (!qconf->n_rx_queue)
> +                       continue;
> +               /* Add rx node patterns of this lcore */
> +               for (i = 0; i < qconf->n_rx_queue; i++) {
> +                       graph_conf.node_patterns[nb_patterns + i] =
> +                               qconf->rx_queue_list[i].node_name;
> +               }
> +               graph_conf.nb_node_patterns = nb_patterns + i;
> +               graph_conf.socket_id = rte_lcore_to_socket_id(lcore_id);
> +               snprintf(qconf->name, sizeof(qconf->name), "worker_%u",
> +                        lcore_id);
> +               graph_id = rte_graph_create(qconf->name, &graph_conf);
> +               if (graph_id == RTE_GRAPH_ID_INVALID)
> +                       rte_exit(EXIT_FAILURE,
> +                                "rte_graph_create(): graph_id invalid for lcore %u\n",
> +                                lcore_id);
> +               qconf->graph_id = graph_id;
> +               qconf->graph = rte_graph_lookup(qconf->name);
> +               /* >8 End of graph initialization. */
> +               if (!qconf->graph)
> +                       rte_exit(EXIT_FAILURE,
> +                                "rte_graph_lookup(): graph %s not found\n",
> +                                qconf->name);
> +       }
> +}
> +
>  int
>  main(int argc, char **argv)
>  {
> @@ -759,6 +928,7 @@ main(int argc, char **argv)
>         uint16_t nb_patterns;
>         uint8_t rewrite_len;
>         uint32_t lcore_id;
> +       uint16_t model;
>         int ret;
>
>         /* Init EAL */
> @@ -787,6 +957,9 @@ main(int argc, char **argv)
>         if (check_lcore_params() < 0)
>                 rte_exit(EXIT_FAILURE, "check_lcore_params() failed\n");
>
> +       if (check_worker_model_params() < 0)
> +               rte_exit(EXIT_FAILURE, "check_worker_model_params() failed\n");
> +
>         ret = init_lcore_rx_queues();
>         if (ret < 0)
>                 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues() failed\n");
> @@ -1026,46 +1199,13 @@ main(int argc, char **argv)
>
>         memset(&graph_conf, 0, sizeof(graph_conf));
>         graph_conf.node_patterns = node_patterns;
> +       graph_conf.nb_node_patterns = nb_patterns;
>
> -       for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
> -               rte_graph_t graph_id;
> -               rte_edge_t i;
> -
> -               if (rte_lcore_is_enabled(lcore_id) == 0)
> -                       continue;
> -
> -               qconf = &lcore_conf[lcore_id];
> -
> -               /* Skip graph creation if no source exists */
> -               if (!qconf->n_rx_queue)
> -                       continue;
> -
> -               /* Add rx node patterns of this lcore */
> -               for (i = 0; i < qconf->n_rx_queue; i++) {
> -                       graph_conf.node_patterns[nb_patterns + i] =
> -                               qconf->rx_queue_list[i].node_name;
> -               }
> -
> -               graph_conf.nb_node_patterns = nb_patterns + i;
> -               graph_conf.socket_id = rte_lcore_to_socket_id(lcore_id);
> -
> -               snprintf(qconf->name, sizeof(qconf->name), "worker_%u",
> -                        lcore_id);
> -
> -               graph_id = rte_graph_create(qconf->name, &graph_conf);
> -               if (graph_id == RTE_GRAPH_ID_INVALID)
> -                       rte_exit(EXIT_FAILURE,
> -                                "rte_graph_create(): graph_id invalid"
> -                                " for lcore %u\n", lcore_id);
> -
> -               qconf->graph_id = graph_id;
> -               qconf->graph = rte_graph_lookup(qconf->name);
> -               /* >8 End of graph initialization. */
> -               if (!qconf->graph)
> -                       rte_exit(EXIT_FAILURE,
> -                                "rte_graph_lookup(): graph %s not found\n",
> -                                qconf->name);
> -       }
> +       model = rte_graph_worker_model_get();
> +       if (model == RTE_GRAPH_MODEL_DEFAULT)
> +               graph_config_rtc(graph_conf);
> +       else if (model == RTE_GRAPH_MODEL_GENERIC)
> +               graph_config_generic(graph_conf);
>
>         memset(&rewrite_data, 0, sizeof(rewrite_data));
>         rewrite_len = sizeof(rewrite_data);
> --
> 2.25.1
>