Re: [PATCH v3 2/3] lib: add IPv6 rewrite node

[Nithin Dabilpuram <nithind1988@gmail.com>]

Acked-by: Nithin Dabilpuram <ndabilpuram@marvell.com>

On Wed, May 31, 2023 at 5:08 PM Amit Prakash Shukla
<amitprakashs@marvell.com> wrote:
>
> Similar to IPv4 rewrite node, patch adds IPv6 rewrite node.
>
> Signed-off-by: Amit Prakash Shukla <amitprakashs@marvell.com>
> ---
> v2:
> - Performance related changes
>
> v3:
> - Removing redundant dynamic variable
>
>  doc/guides/prog_guide/graph_lib.rst |   8 +
>  lib/node/ethdev_ctrl.c              |  13 ++
>  lib/node/ip6_rewrite.c              | 331 ++++++++++++++++++++++++++++
>  lib/node/ip6_rewrite_priv.h         |  69 ++++++
>  lib/node/meson.build                |   1 +
>  5 files changed, 422 insertions(+)
>  create mode 100644 lib/node/ip6_rewrite.c
>  create mode 100644 lib/node/ip6_rewrite_priv.h
>
> diff --git a/doc/guides/prog_guide/graph_lib.rst b/doc/guides/prog_guide/graph_lib.rst
> index 1f70d63628..841cabe259 100644
> --- a/doc/guides/prog_guide/graph_lib.rst
> +++ b/doc/guides/prog_guide/graph_lib.rst
> @@ -401,6 +401,14 @@ On LPM lookup failure, objects are redirected to pkt_drop node.
>  To achieve home run, node use ``rte_node_stream_move()`` as mentioned in above
>  sections.
>
> +ip6_rewrite
> +~~~~~~~~~~~
> +This node gets packets from ``ip6_lookup`` node with next-hop id for each
> +packet is embedded in ``node_mbuf_priv1(mbuf)->nh``. This id is used
> +to determine the L2 header to be written to the packet before sending
> +the packet out to a particular ethdev_tx node.
> +``rte_node_ip6_rewrite_add()`` is control path API to add next-hop info.
> +
>  null
>  ~~~~
>  This node ignores the set of objects passed to it and reports that all are
> diff --git a/lib/node/ethdev_ctrl.c b/lib/node/ethdev_ctrl.c
> index 37df0431b8..ea7dc8210b 100644
> --- a/lib/node/ethdev_ctrl.c
> +++ b/lib/node/ethdev_ctrl.c
> @@ -12,6 +12,7 @@
>  #include "ethdev_rx_priv.h"
>  #include "ethdev_tx_priv.h"
>  #include "ip4_rewrite_priv.h"
> +#include "ip6_rewrite_priv.h"
>  #include "node_private.h"
>
>  static struct ethdev_ctrl {
> @@ -23,6 +24,7 @@ rte_node_eth_config(struct rte_node_ethdev_config *conf, uint16_t nb_confs,
>                     uint16_t nb_graphs)
>  {
>         struct rte_node_register *ip4_rewrite_node;
> +       struct rte_node_register *ip6_rewrite_node;
>         struct ethdev_tx_node_main *tx_node_data;
>         uint16_t tx_q_used, rx_q_used, port_id;
>         struct rte_node_register *tx_node;
> @@ -33,6 +35,7 @@ rte_node_eth_config(struct rte_node_ethdev_config *conf, uint16_t nb_confs,
>         uint32_t id;
>
>         ip4_rewrite_node = ip4_rewrite_node_get();
> +       ip6_rewrite_node = ip6_rewrite_node_get();
>         tx_node_data = ethdev_tx_node_data_get();
>         tx_node = ethdev_tx_node_get();
>         for (i = 0; i < nb_confs; i++) {
> @@ -110,6 +113,16 @@ rte_node_eth_config(struct rte_node_ethdev_config *conf, uint16_t nb_confs,
>                         port_id, rte_node_edge_count(ip4_rewrite_node->id) - 1);
>                 if (rc < 0)
>                         return rc;
> +
> +               /* Add this tx port node as next to ip6_rewrite_node */
> +               rte_node_edge_update(ip6_rewrite_node->id, RTE_EDGE_ID_INVALID,
> +                                    &next_nodes, 1);
> +               /* Assuming edge id is the last one alloc'ed */
> +               rc = ip6_rewrite_set_next(
> +                       port_id, rte_node_edge_count(ip6_rewrite_node->id) - 1);
> +               if (rc < 0)
> +                       return rc;
> +
>         }
>
>         ctrl.nb_graphs = nb_graphs;
> diff --git a/lib/node/ip6_rewrite.c b/lib/node/ip6_rewrite.c
> new file mode 100644
> index 0000000000..198d8d8820
> --- /dev/null
> +++ b/lib/node/ip6_rewrite.c
> @@ -0,0 +1,331 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(C) 2023 Marvell.
> + */
> +
> +#include <rte_ethdev.h>
> +#include <rte_ether.h>
> +#include <rte_graph.h>
> +#include <rte_graph_worker.h>
> +#include <rte_ip.h>
> +#include <rte_malloc.h>
> +#include <rte_vect.h>
> +
> +#include "rte_node_ip6_api.h"
> +
> +#include "ip6_rewrite_priv.h"
> +#include "node_private.h"
> +
> +struct ip6_rewrite_node_ctx {
> +       /* Dynamic offset to mbuf priv1 */
> +       int mbuf_priv1_off;
> +       /* Cached next index */
> +       uint16_t next_index;
> +};
> +
> +static struct ip6_rewrite_node_main *ip6_rewrite_nm;
> +
> +#define IP6_REWRITE_NODE_LAST_NEXT(ctx) \
> +       (((struct ip6_rewrite_node_ctx *)ctx)->next_index)
> +
> +#define IP6_REWRITE_NODE_PRIV1_OFF(ctx) \
> +       (((struct ip6_rewrite_node_ctx *)ctx)->mbuf_priv1_off)
> +
> +static uint16_t
> +ip6_rewrite_node_process(struct rte_graph *graph, struct rte_node *node,
> +                        void **objs, uint16_t nb_objs)
> +{
> +       struct rte_mbuf *mbuf0, *mbuf1, *mbuf2, *mbuf3, **pkts;
> +       struct ip6_rewrite_nh_header *nh = ip6_rewrite_nm->nh;
> +       const int dyn = IP6_REWRITE_NODE_PRIV1_OFF(node->ctx);
> +       uint16_t next0, next1, next2, next3, next_index;
> +       uint16_t n_left_from, held = 0, last_spec = 0;
> +       struct rte_ipv6_hdr *ip0, *ip1, *ip2, *ip3;
> +       void *d0, *d1, *d2, *d3;
> +       void **to_next, **from;
> +       rte_xmm_t priv01;
> +       rte_xmm_t priv23;
> +       int i;
> +
> +       /* Speculative next as last next */
> +       next_index = IP6_REWRITE_NODE_LAST_NEXT(node->ctx);
> +       rte_prefetch0(nh);
> +
> +       pkts = (struct rte_mbuf **)objs;
> +       from = objs;
> +       n_left_from = nb_objs;
> +
> +       for (i = 0; i < 4 && i < n_left_from; i++)
> +               rte_prefetch0(pkts[i]);
> +
> +       /* Get stream for the speculated next node */
> +       to_next = rte_node_next_stream_get(graph, node, next_index, nb_objs);
> +       /* Update Ethernet header of pkts */
> +       while (n_left_from >= 4) {
> +               if (likely(n_left_from > 7)) {
> +                       /* Prefetch only next-mbuf struct and priv area.
> +                        * Data need not be prefetched as we only write.
> +                        */
> +                       rte_prefetch0(pkts[4]);
> +                       rte_prefetch0(pkts[5]);
> +                       rte_prefetch0(pkts[6]);
> +                       rte_prefetch0(pkts[7]);
> +               }
> +
> +               mbuf0 = pkts[0];
> +               mbuf1 = pkts[1];
> +               mbuf2 = pkts[2];
> +               mbuf3 = pkts[3];
> +
> +               pkts += 4;
> +               n_left_from -= 4;
> +               priv01.u64[0] = node_mbuf_priv1(mbuf0, dyn)->u;
> +               priv01.u64[1] = node_mbuf_priv1(mbuf1, dyn)->u;
> +               priv23.u64[0] = node_mbuf_priv1(mbuf2, dyn)->u;
> +               priv23.u64[1] = node_mbuf_priv1(mbuf3, dyn)->u;
> +
> +               /* Update next_hop rewrite ethernet hdr on mbuf0 */
> +               d0 = rte_pktmbuf_mtod(mbuf0, void *);
> +               rte_memcpy(d0, nh[priv01.u16[0]].rewrite_data,
> +                          nh[priv01.u16[0]].rewrite_len);
> +
> +               next0 = nh[priv01.u16[0]].tx_node;
> +               ip0 = (struct rte_ipv6_hdr *)((uint8_t *)d0 +
> +                                             sizeof(struct rte_ether_hdr));
> +               ip0->hop_limits = priv01.u16[1] - 1;
> +
> +               /* Update next_hop rewrite ethernet hdr on mbuf1 */
> +               d1 = rte_pktmbuf_mtod(mbuf1, void *);
> +               rte_memcpy(d1, nh[priv01.u16[4]].rewrite_data,
> +                          nh[priv01.u16[4]].rewrite_len);
> +
> +               next1 = nh[priv01.u16[4]].tx_node;
> +               ip1 = (struct rte_ipv6_hdr *)((uint8_t *)d1 +
> +                                             sizeof(struct rte_ether_hdr));
> +               ip1->hop_limits = priv01.u16[5] - 1;
> +
> +               /* Update next_hop rewrite ethernet hdr on mbuf2 */
> +               d2 = rte_pktmbuf_mtod(mbuf2, void *);
> +               rte_memcpy(d2, nh[priv23.u16[0]].rewrite_data,
> +                          nh[priv23.u16[0]].rewrite_len);
> +               next2 = nh[priv23.u16[0]].tx_node;
> +               ip2 = (struct rte_ipv6_hdr *)((uint8_t *)d2 +
> +                                             sizeof(struct rte_ether_hdr));
> +               ip2->hop_limits = priv23.u16[1] - 1;
> +
> +               /* Update next_hop rewrite ethernet hdr on mbuf3 */
> +               d3 = rte_pktmbuf_mtod(mbuf3, void *);
> +               rte_memcpy(d3, nh[priv23.u16[4]].rewrite_data,
> +                          nh[priv23.u16[4]].rewrite_len);
> +
> +               next3 = nh[priv23.u16[4]].tx_node;
> +               ip3 = (struct rte_ipv6_hdr *)((uint8_t *)d3 +
> +                                             sizeof(struct rte_ether_hdr));
> +               ip3->hop_limits = priv23.u16[5] - 1;
> +
> +               /* Enqueue four packets to next node */
> +               rte_edge_t fix_spec =
> +                       ((next_index == next0) && (next0 == next1) &&
> +                        (next1 == next2) && (next2 == next3));
> +
> +               if (unlikely(fix_spec == 0)) {
> +                       /* Copy things successfully speculated till now */
> +                       rte_memcpy(to_next, from, last_spec * sizeof(from[0]));
> +                       from += last_spec;
> +                       to_next += last_spec;
> +                       held += last_spec;
> +                       last_spec = 0;
> +
> +                       /* next0 */
> +                       if (next_index == next0) {
> +                               to_next[0] = from[0];
> +                               to_next++;
> +                               held++;
> +                       } else {
> +                               rte_node_enqueue_x1(graph, node, next0,
> +                                                   from[0]);
> +                       }
> +
> +                       /* next1 */
> +                       if (next_index == next1) {
> +                               to_next[0] = from[1];
> +                               to_next++;
> +                               held++;
> +                       } else {
> +                               rte_node_enqueue_x1(graph, node, next1,
> +                                                   from[1]);
> +                       }
> +
> +                       /* next2 */
> +                       if (next_index == next2) {
> +                               to_next[0] = from[2];
> +                               to_next++;
> +                               held++;
> +                       } else {
> +                               rte_node_enqueue_x1(graph, node, next2,
> +                                                   from[2]);
> +                       }
> +
> +                       /* next3 */
> +                       if (next_index == next3) {
> +                               to_next[0] = from[3];
> +                               to_next++;
> +                               held++;
> +                       } else {
> +                               rte_node_enqueue_x1(graph, node, next3,
> +                                                   from[3]);
> +                       }
> +
> +                       from += 4;
> +
> +                       /* Change speculation if last two are same */
> +                       if ((next_index != next3) && (next2 == next3)) {
> +                               /* Put the current speculated node */
> +                               rte_node_next_stream_put(graph, node,
> +                                                        next_index, held);
> +                               held = 0;
> +
> +                               /* Get next speculated stream */
> +                               next_index = next3;
> +                               to_next = rte_node_next_stream_get(
> +                                       graph, node, next_index, nb_objs);
> +                       }
> +               } else {
> +                       last_spec += 4;
> +               }
> +       }
> +
> +       while (n_left_from > 0) {
> +               mbuf0 = pkts[0];
> +
> +               pkts += 1;
> +               n_left_from -= 1;
> +
> +               d0 = rte_pktmbuf_mtod(mbuf0, void *);
> +               rte_memcpy(d0, nh[node_mbuf_priv1(mbuf0, dyn)->nh].rewrite_data,
> +                          nh[node_mbuf_priv1(mbuf0, dyn)->nh].rewrite_len);
> +
> +               next0 = nh[node_mbuf_priv1(mbuf0, dyn)->nh].tx_node;
> +               ip0 = (struct rte_ipv6_hdr *)((uint8_t *)d0 +
> +                                             sizeof(struct rte_ether_hdr));
> +               ip0->hop_limits = node_mbuf_priv1(mbuf0, dyn)->ttl - 1;
> +
> +               if (unlikely(next_index ^ next0)) {
> +                       /* Copy things successfully speculated till now */
> +                       rte_memcpy(to_next, from, last_spec * sizeof(from[0]));
> +                       from += last_spec;
> +                       to_next += last_spec;
> +                       held += last_spec;
> +                       last_spec = 0;
> +
> +                       rte_node_enqueue_x1(graph, node, next0, from[0]);
> +                       from += 1;
> +               } else {
> +                       last_spec += 1;
> +               }
> +       }
> +
> +       /* !!! Home run !!! */
> +       if (likely(last_spec == nb_objs)) {
> +               rte_node_next_stream_move(graph, node, next_index);
> +               return nb_objs;
> +       }
> +
> +       held += last_spec;
> +       rte_memcpy(to_next, from, last_spec * sizeof(from[0]));
> +       rte_node_next_stream_put(graph, node, next_index, held);
> +       /* Save the last next used */
> +       IP6_REWRITE_NODE_LAST_NEXT(node->ctx) = next_index;
> +
> +       return nb_objs;
> +}
> +
> +static int
> +ip6_rewrite_node_init(const struct rte_graph *graph, struct rte_node *node)
> +{
> +       static bool init_once;
> +
> +       RTE_SET_USED(graph);
> +       RTE_BUILD_BUG_ON(sizeof(struct ip6_rewrite_node_ctx) > RTE_NODE_CTX_SZ);
> +
> +       if (!init_once) {
> +               node_mbuf_priv1_dynfield_offset = rte_mbuf_dynfield_register(
> +                               &node_mbuf_priv1_dynfield_desc);
> +               if (node_mbuf_priv1_dynfield_offset < 0)
> +                       return -rte_errno;
> +               init_once = true;
> +       }
> +       IP6_REWRITE_NODE_PRIV1_OFF(node->ctx) = node_mbuf_priv1_dynfield_offset;
> +
> +       node_dbg("ip6_rewrite", "Initialized ip6_rewrite node");
> +
> +       return 0;
> +}
> +
> +int
> +ip6_rewrite_set_next(uint16_t port_id, uint16_t next_index)
> +{
> +       if (ip6_rewrite_nm == NULL) {
> +               ip6_rewrite_nm = rte_zmalloc(
> +                       "ip6_rewrite", sizeof(struct ip6_rewrite_node_main),
> +                       RTE_CACHE_LINE_SIZE);
> +               if (ip6_rewrite_nm == NULL)
> +                       return -ENOMEM;
> +       }
> +       ip6_rewrite_nm->next_index[port_id] = next_index;
> +
> +       return 0;
> +}
> +
> +int
> +rte_node_ip6_rewrite_add(uint16_t next_hop, uint8_t *rewrite_data,
> +                        uint8_t rewrite_len, uint16_t dst_port)
> +{
> +       struct ip6_rewrite_nh_header *nh;
> +
> +       if (next_hop >= RTE_GRAPH_IP6_REWRITE_MAX_NH)
> +               return -EINVAL;
> +
> +       if (rewrite_len > RTE_GRAPH_IP6_REWRITE_MAX_LEN)
> +               return -EINVAL;
> +
> +       if (ip6_rewrite_nm == NULL) {
> +               ip6_rewrite_nm = rte_zmalloc(
> +                       "ip6_rewrite", sizeof(struct ip6_rewrite_node_main),
> +                       RTE_CACHE_LINE_SIZE);
> +               if (ip6_rewrite_nm == NULL)
> +                       return -ENOMEM;
> +       }
> +
> +       /* Check if dst port doesn't exist as edge */
> +       if (!ip6_rewrite_nm->next_index[dst_port])
> +               return -EINVAL;
> +
> +       /* Update next hop */
> +       nh = &ip6_rewrite_nm->nh[next_hop];
> +
> +       memcpy(nh->rewrite_data, rewrite_data, rewrite_len);
> +       nh->tx_node = ip6_rewrite_nm->next_index[dst_port];
> +       nh->rewrite_len = rewrite_len;
> +       nh->enabled = true;
> +
> +       return 0;
> +}
> +
> +static struct rte_node_register ip6_rewrite_node = {
> +       .process = ip6_rewrite_node_process,
> +       .name = "ip6_rewrite",
> +       /* Default edge i.e '0' is pkt drop */
> +       .nb_edges = 1,
> +       .next_nodes = {
> +               [0] = "pkt_drop",
> +       },
> +       .init = ip6_rewrite_node_init,
> +};
> +
> +struct rte_node_register *
> +ip6_rewrite_node_get(void)
> +{
> +       return &ip6_rewrite_node;
> +}
> +
> +RTE_NODE_REGISTER(ip6_rewrite_node);
> diff --git a/lib/node/ip6_rewrite_priv.h b/lib/node/ip6_rewrite_priv.h
> new file mode 100644
> index 0000000000..93db28bec0
> --- /dev/null
> +++ b/lib/node/ip6_rewrite_priv.h
> @@ -0,0 +1,69 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(C) 2023 Marvell.
> + */
> +#ifndef __INCLUDE_IP6_REWRITE_PRIV_H__
> +#define __INCLUDE_IP6_REWRITE_PRIV_H__
> +
> +#include <rte_common.h>
> +
> +#define RTE_GRAPH_IP6_REWRITE_MAX_NH 64
> +#define RTE_GRAPH_IP6_REWRITE_MAX_LEN 56
> +
> +/**
> + * @internal
> + *
> + * Ipv6 rewrite next hop header data structure. Used to store port specific
> + * rewrite data.
> + */
> +struct ip6_rewrite_nh_header {
> +       uint16_t rewrite_len; /**< Header rewrite length. */
> +       uint16_t tx_node;     /**< Tx node next index identifier. */
> +       uint16_t enabled;     /**< NH enable flag */
> +       uint16_t rsvd;
> +       union {
> +               struct {
> +                       struct rte_ether_addr dst;
> +                       /**< Destination mac address. */
> +                       struct rte_ether_addr src;
> +                       /**< Source mac address. */
> +               };
> +               uint8_t rewrite_data[RTE_GRAPH_IP6_REWRITE_MAX_LEN];
> +               /**< Generic rewrite data */
> +       };
> +};
> +
> +/**
> + * @internal
> + *
> + * Ipv6 node main data structure.
> + */
> +struct ip6_rewrite_node_main {
> +       struct ip6_rewrite_nh_header nh[RTE_GRAPH_IP6_REWRITE_MAX_NH];
> +       /**< Array of next hop header data */
> +       uint16_t next_index[RTE_MAX_ETHPORTS];
> +       /**< Next index of each configured port. */
> +};
> +
> +/**
> + * @internal
> + *
> + * Get the ipv6 rewrite node.
> + *
> + * @return
> + *   Pointer to the ipv6 rewrite node.
> + */
> +struct rte_node_register *ip6_rewrite_node_get(void);
> +
> +/**
> + * @internal
> + *
> + * Set the Edge index of a given port_id.
> + *
> + * @param port_id
> + *   Ethernet port identifier.
> + * @param next_index
> + *   Edge index of the Given Tx node.
> + */
> +int ip6_rewrite_set_next(uint16_t port_id, uint16_t next_index);
> +
> +#endif /* __INCLUDE_IP6_REWRITE_PRIV_H__ */
> diff --git a/lib/node/meson.build b/lib/node/meson.build
> index cd30847a0b..b2f04269c5 100644
> --- a/lib/node/meson.build
> +++ b/lib/node/meson.build
> @@ -14,6 +14,7 @@ sources = files(
>          'ip4_lookup.c',
>          'ip6_lookup.c',
>          'ip4_rewrite.c',
> +        'ip6_rewrite.c',
>          'log.c',
>          'null.c',
>          'pkt_cls.c',
> --
> 2.25.1
>