* [dpdk-dev] [PATCH] examples/l2fwd-event: changes to use event vector
@ 2021-09-04 7:46 Shijith Thotton
2021-09-27 6:21 ` [dpdk-dev] [PATCH v2] " Shijith Thotton
0 siblings, 1 reply; 3+ messages in thread
From: Shijith Thotton @ 2021-09-04 7:46 UTC (permalink / raw)
To: dev
Cc: Shijith Thotton, jerinj, ndabilpuram, pbhagavatula, schalla,
Sunil Kumar Kori
Added changes to receive packets as event vector. By default this is
disabled and can be enabled using the option --enable-vector. Vector
size and timeout to form the vector can be configured using options
--vector-size and --vector-tmo-ns.
Example:
dpdk-l2fwd-event -l 0-3 -n 4 -- -p 0x03 --mode=eventdev \
--eventq-sched=ordered --enable-vector --vector-size 16
Signed-off-by: Shijith Thotton <sthotton@marvell.com>
---
Depends-on: series=18322 (eventdev: simplify Rx adapter event vector config)
doc/guides/sample_app_ug/l2_forward_event.rst | 13 +-
examples/l2fwd-event/l2fwd_common.h | 12 ++
examples/l2fwd-event/l2fwd_event.c | 204 ++++++++++++++++--
examples/l2fwd-event/l2fwd_event_generic.c | 21 ++
.../l2fwd-event/l2fwd_event_internal_port.c | 22 ++
examples/l2fwd-event/main.c | 48 +++++
6 files changed, 306 insertions(+), 14 deletions(-)
diff --git a/doc/guides/sample_app_ug/l2_forward_event.rst b/doc/guides/sample_app_ug/l2_forward_event.rst
index 540ca015db..1308f7e6d0 100644
--- a/doc/guides/sample_app_ug/l2_forward_event.rst
+++ b/doc/guides/sample_app_ug/l2_forward_event.rst
@@ -52,7 +52,12 @@ The application requires a number of command line options:
.. code-block:: console
- ./<build_dir>/examples/dpdk-l2fwd-event [EAL options] -- -p PORTMASK [-q NQ] --[no-]mac-updating --mode=MODE --eventq-sched=SCHED_MODE
+ ./<build_dir>/examples/dpdk-l2fwd-event [EAL options] -- -p PORTMASK
+ [-q NQ]
+ [--[no-]mac-updating]
+ [--mode=MODE]
+ [--eventq-sched=SCHED_MODE]
+ [--enable-vector [--vector-size SIZE] [--vector-tmo-ns NS]]
where,
@@ -68,6 +73,12 @@ where,
* --config: Configure forwarding port pair mapping. Alternate port pairs by default.
+* --enable-vector: Enable event vectorization. Only valid if --mode=eventdev.
+
+* --vector-size: Max vector size if event vectorization is enabled.
+
+* --vector-tmo-ns: Max timeout to form vector in nanoseconds if event vectorization is enabled.
+
Sample usage commands are given below to run the application into different mode:
Poll mode with 4 lcores, 16 ports and 8 RX queues per lcore and MAC address updating enabled,
diff --git a/examples/l2fwd-event/l2fwd_common.h b/examples/l2fwd-event/l2fwd_common.h
index 939221d45a..5e380ded16 100644
--- a/examples/l2fwd-event/l2fwd_common.h
+++ b/examples/l2fwd-event/l2fwd_common.h
@@ -56,6 +56,9 @@
#define DEFAULT_TIMER_PERIOD 10 /* default period is 10 seconds */
#define MAX_TIMER_PERIOD 86400 /* 1 day max */
+#define VECTOR_SIZE_DEFAULT MAX_PKT_BURST
+#define VECTOR_TMO_NS_DEFAULT 1E6 /* 1ms */
+
/* Per-port statistics struct */
struct l2fwd_port_statistics {
uint64_t dropped;
@@ -63,6 +66,13 @@ struct l2fwd_port_statistics {
uint64_t rx;
} __rte_cache_aligned;
+/* Event vector attributes */
+struct l2fwd_event_vector_params {
+ uint8_t enabled;
+ uint16_t size;
+ uint64_t timeout_ns;
+};
+
struct l2fwd_resources {
volatile uint8_t force_quit;
uint8_t event_mode;
@@ -75,9 +85,11 @@ struct l2fwd_resources {
uint32_t enabled_port_mask;
uint64_t timer_period;
struct rte_mempool *pktmbuf_pool;
+ struct rte_mempool *evt_vec_pool;
uint32_t dst_ports[RTE_MAX_ETHPORTS];
struct rte_ether_addr eth_addr[RTE_MAX_ETHPORTS];
struct l2fwd_port_statistics port_stats[RTE_MAX_ETHPORTS];
+ struct l2fwd_event_vector_params evt_vec;
void *evt_rsrc;
void *poll_rsrc;
} __rte_cache_aligned;
diff --git a/examples/l2fwd-event/l2fwd_event.c b/examples/l2fwd-event/l2fwd_event.c
index 7ba5311d66..09080fa666 100644
--- a/examples/l2fwd-event/l2fwd_event.c
+++ b/examples/l2fwd-event/l2fwd_event.c
@@ -346,19 +346,198 @@ l2fwd_event_main_loop_tx_q_brst_mac(struct l2fwd_resources *rsrc)
L2FWD_EVENT_TX_ENQ | L2FWD_EVENT_BURST);
}
+static __rte_always_inline void
+l2fwd_event_vector_fwd(struct l2fwd_resources *rsrc,
+ struct rte_event_vector *vec,
+ const uint64_t timer_period, const uint32_t flags)
+{
+ struct rte_mbuf **mbufs = vec->mbufs;
+ uint16_t i, j;
+
+ rte_prefetch0(rte_pktmbuf_mtod(mbufs[0], void *));
+
+ /* If vector attribute is valid, mbufs will be from same port/queue */
+ if (vec->attr_valid) {
+ vec->port = rsrc->dst_ports[mbufs[0]->port];
+ if (flags & L2FWD_EVENT_TX_DIRECT)
+ vec->queue = 0;
+
+ if (timer_period > 0)
+ __atomic_fetch_add(&rsrc->port_stats[mbufs[0]->port].rx,
+ vec->nb_elem, __ATOMIC_RELAXED);
+
+ for (i = 0, j = 1; i < vec->nb_elem; i++, j++) {
+ if (j < vec->nb_elem)
+ rte_prefetch0(
+ rte_pktmbuf_mtod(mbufs[j], void *));
+
+ if (flags & L2FWD_EVENT_UPDT_MAC)
+ l2fwd_mac_updating(
+ mbufs[i], vec->port,
+ &rsrc->eth_addr[vec->port]);
+ }
+
+ if (timer_period > 0)
+ __atomic_fetch_add(&rsrc->port_stats[vec->port].tx,
+ vec->nb_elem, __ATOMIC_RELAXED);
+ } else {
+ for (i = 0, j = 1; i < vec->nb_elem; i++, j++) {
+ if (timer_period > 0)
+ __atomic_fetch_add(
+ &rsrc->port_stats[mbufs[i]->port].rx, 1,
+ __ATOMIC_RELAXED);
+
+ if (j < vec->nb_elem)
+ rte_prefetch0(
+ rte_pktmbuf_mtod(mbufs[j], void *));
+
+ mbufs[i]->port = rsrc->dst_ports[mbufs[i]->port];
+
+ if (flags & L2FWD_EVENT_UPDT_MAC)
+ l2fwd_mac_updating(
+ mbufs[i], mbufs[i]->port,
+ &rsrc->eth_addr[mbufs[i]->port]);
+
+ if (flags & L2FWD_EVENT_TX_DIRECT)
+ rte_event_eth_tx_adapter_txq_set(mbufs[i], 0);
+
+ if (timer_period > 0)
+ __atomic_fetch_add(
+ &rsrc->port_stats[mbufs[i]->port].tx, 1,
+ __ATOMIC_RELAXED);
+ }
+ }
+}
+
+static __rte_always_inline void
+l2fwd_event_loop_vector(struct l2fwd_resources *rsrc, const uint32_t flags)
+{
+ struct l2fwd_event_resources *evt_rsrc = rsrc->evt_rsrc;
+ const int port_id = l2fwd_get_free_event_port(evt_rsrc);
+ const uint8_t tx_q_id =
+ evt_rsrc->evq.event_q_id[evt_rsrc->evq.nb_queues - 1];
+ const uint64_t timer_period = rsrc->timer_period;
+ const uint8_t event_d_id = evt_rsrc->event_d_id;
+ const uint8_t deq_len = evt_rsrc->deq_depth;
+ struct rte_event ev[MAX_PKT_BURST];
+ uint16_t nb_rx, nb_tx;
+ uint8_t i;
+
+ if (port_id < 0)
+ return;
+
+ printf("%s(): entering eventdev main loop on lcore %u\n", __func__,
+ rte_lcore_id());
+
+ while (!rsrc->force_quit) {
+ nb_rx = rte_event_dequeue_burst(event_d_id, port_id, ev,
+ deq_len, 0);
+ if (nb_rx == 0)
+ continue;
+
+ for (i = 0; i < nb_rx; i++) {
+ if (flags & L2FWD_EVENT_TX_ENQ) {
+ ev[i].queue_id = tx_q_id;
+ ev[i].op = RTE_EVENT_OP_FORWARD;
+ }
+
+ l2fwd_event_vector_fwd(rsrc, ev[i].vec, timer_period,
+ flags);
+ }
+
+ if (flags & L2FWD_EVENT_TX_ENQ) {
+ nb_tx = rte_event_enqueue_burst(event_d_id, port_id, ev,
+ nb_rx);
+ while (nb_tx < nb_rx && !rsrc->force_quit)
+ nb_tx += rte_event_enqueue_burst(
+ event_d_id, port_id, ev + nb_tx,
+ nb_rx - nb_tx);
+ }
+
+ if (flags & L2FWD_EVENT_TX_DIRECT) {
+ nb_tx = rte_event_eth_tx_adapter_enqueue(
+ event_d_id, port_id, ev, nb_rx, 0);
+ while (nb_tx < nb_rx && !rsrc->force_quit)
+ nb_tx += rte_event_eth_tx_adapter_enqueue(
+ event_d_id, port_id, ev + nb_tx,
+ nb_rx - nb_tx, 0);
+ }
+ }
+}
+
+static void __rte_noinline
+l2fwd_event_main_loop_tx_d_vec(struct l2fwd_resources *rsrc)
+{
+ l2fwd_event_loop_vector(rsrc, L2FWD_EVENT_TX_DIRECT);
+}
+
+static void __rte_noinline
+l2fwd_event_main_loop_tx_d_brst_vec(struct l2fwd_resources *rsrc)
+{
+ l2fwd_event_loop_vector(rsrc, L2FWD_EVENT_TX_DIRECT);
+}
+
+static void __rte_noinline
+l2fwd_event_main_loop_tx_q_vec(struct l2fwd_resources *rsrc)
+{
+ l2fwd_event_loop_vector(rsrc, L2FWD_EVENT_TX_ENQ);
+}
+
+static void __rte_noinline
+l2fwd_event_main_loop_tx_q_brst_vec(struct l2fwd_resources *rsrc)
+{
+ l2fwd_event_loop_vector(rsrc, L2FWD_EVENT_TX_ENQ);
+}
+
+static void __rte_noinline
+l2fwd_event_main_loop_tx_d_mac_vec(struct l2fwd_resources *rsrc)
+{
+ l2fwd_event_loop_vector(rsrc,
+ L2FWD_EVENT_UPDT_MAC | L2FWD_EVENT_TX_DIRECT);
+}
+
+static void __rte_noinline
+l2fwd_event_main_loop_tx_d_brst_mac_vec(struct l2fwd_resources *rsrc)
+{
+ l2fwd_event_loop_vector(rsrc,
+ L2FWD_EVENT_UPDT_MAC | L2FWD_EVENT_TX_DIRECT);
+}
+
+static void __rte_noinline
+l2fwd_event_main_loop_tx_q_mac_vec(struct l2fwd_resources *rsrc)
+{
+ l2fwd_event_loop_vector(rsrc,
+ L2FWD_EVENT_UPDT_MAC | L2FWD_EVENT_TX_ENQ);
+}
+
+static void __rte_noinline
+l2fwd_event_main_loop_tx_q_brst_mac_vec(struct l2fwd_resources *rsrc)
+{
+ l2fwd_event_loop_vector(rsrc,
+ L2FWD_EVENT_UPDT_MAC | L2FWD_EVENT_TX_ENQ);
+}
+
void
l2fwd_event_resource_setup(struct l2fwd_resources *rsrc)
{
/* [MAC_UPDT][TX_MODE][BURST] */
- const event_loop_cb event_loop[2][2][2] = {
- [0][0][0] = l2fwd_event_main_loop_tx_d,
- [0][0][1] = l2fwd_event_main_loop_tx_d_brst,
- [0][1][0] = l2fwd_event_main_loop_tx_q,
- [0][1][1] = l2fwd_event_main_loop_tx_q_brst,
- [1][0][0] = l2fwd_event_main_loop_tx_d_mac,
- [1][0][1] = l2fwd_event_main_loop_tx_d_brst_mac,
- [1][1][0] = l2fwd_event_main_loop_tx_q_mac,
- [1][1][1] = l2fwd_event_main_loop_tx_q_brst_mac,
+ const event_loop_cb event_loop[2][2][2][2] = {
+ [0][0][0][0] = l2fwd_event_main_loop_tx_d,
+ [0][0][0][1] = l2fwd_event_main_loop_tx_d_brst,
+ [0][0][1][0] = l2fwd_event_main_loop_tx_q,
+ [0][0][1][1] = l2fwd_event_main_loop_tx_q_brst,
+ [0][1][0][0] = l2fwd_event_main_loop_tx_d_mac,
+ [0][1][0][1] = l2fwd_event_main_loop_tx_d_brst_mac,
+ [0][1][1][0] = l2fwd_event_main_loop_tx_q_mac,
+ [0][1][1][1] = l2fwd_event_main_loop_tx_q_brst_mac,
+ [1][0][0][0] = l2fwd_event_main_loop_tx_d_vec,
+ [1][0][0][1] = l2fwd_event_main_loop_tx_d_brst_vec,
+ [1][0][1][0] = l2fwd_event_main_loop_tx_q_vec,
+ [1][0][1][1] = l2fwd_event_main_loop_tx_q_brst_vec,
+ [1][1][0][0] = l2fwd_event_main_loop_tx_d_mac_vec,
+ [1][1][0][1] = l2fwd_event_main_loop_tx_d_brst_mac_vec,
+ [1][1][1][0] = l2fwd_event_main_loop_tx_q_mac_vec,
+ [1][1][1][1] = l2fwd_event_main_loop_tx_q_brst_mac_vec,
};
struct l2fwd_event_resources *evt_rsrc;
uint32_t event_queue_cfg;
@@ -394,8 +573,7 @@ l2fwd_event_resource_setup(struct l2fwd_resources *rsrc)
if (ret < 0)
rte_panic("Error in starting eventdev\n");
- evt_rsrc->ops.l2fwd_event_loop = event_loop
- [rsrc->mac_updating]
- [evt_rsrc->tx_mode_q]
- [evt_rsrc->has_burst];
+ evt_rsrc->ops.l2fwd_event_loop =
+ event_loop[rsrc->evt_vec.enabled][rsrc->mac_updating]
+ [evt_rsrc->tx_mode_q][evt_rsrc->has_burst];
}
diff --git a/examples/l2fwd-event/l2fwd_event_generic.c b/examples/l2fwd-event/l2fwd_event_generic.c
index c19b6f1411..203509dad0 100644
--- a/examples/l2fwd-event/l2fwd_event_generic.c
+++ b/examples/l2fwd-event/l2fwd_event_generic.c
@@ -245,6 +245,27 @@ l2fwd_rx_tx_adapter_setup_generic(struct l2fwd_resources *rsrc)
if ((rsrc->enabled_port_mask & (1 << port_id)) == 0)
continue;
eth_q_conf.ev.queue_id = evt_rsrc->evq.event_q_id[i];
+ if (rsrc->evt_vec.enabled) {
+ uint32_t cap;
+
+ if (rte_event_eth_rx_adapter_caps_get(event_d_id,
+ port_id, &cap))
+ rte_panic(
+ "Failed to get event rx adapter capability");
+
+ if (cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_EVENT_VECTOR) {
+ eth_q_conf.vector_sz = rsrc->evt_vec.size;
+ eth_q_conf.vector_timeout_ns =
+ rsrc->evt_vec.timeout_ns;
+ eth_q_conf.vector_mp = rsrc->evt_vec_pool;
+ eth_q_conf.rx_queue_flags |=
+ RTE_EVENT_ETH_RX_ADAPTER_QUEUE_EVENT_VECTOR;
+ } else {
+ rte_panic(
+ "Rx adapter doesn't support event vector");
+ }
+ }
+
ret = rte_event_eth_rx_adapter_queue_add(rx_adptr_id, port_id,
-1, ð_q_conf);
if (ret)
diff --git a/examples/l2fwd-event/l2fwd_event_internal_port.c b/examples/l2fwd-event/l2fwd_event_internal_port.c
index 16a52d67c9..5efd44f950 100644
--- a/examples/l2fwd-event/l2fwd_event_internal_port.c
+++ b/examples/l2fwd-event/l2fwd_event_internal_port.c
@@ -230,6 +230,28 @@ l2fwd_rx_tx_adapter_setup_internal_port(struct l2fwd_resources *rsrc)
RTE_ETH_FOREACH_DEV(port_id) {
if ((rsrc->enabled_port_mask & (1 << port_id)) == 0)
continue;
+
+ if (rsrc->evt_vec.enabled) {
+ uint32_t cap;
+
+ if (rte_event_eth_rx_adapter_caps_get(event_d_id,
+ port_id, &cap))
+ rte_panic(
+ "Failed to get event rx adapter capability");
+
+ if (cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_EVENT_VECTOR) {
+ eth_q_conf.vector_sz = rsrc->evt_vec.size;
+ eth_q_conf.vector_timeout_ns =
+ rsrc->evt_vec.timeout_ns;
+ eth_q_conf.vector_mp = rsrc->evt_vec_pool;
+ eth_q_conf.rx_queue_flags |=
+ RTE_EVENT_ETH_RX_ADAPTER_QUEUE_EVENT_VECTOR;
+ } else {
+ rte_panic(
+ "Rx adapter doesn't support event vector");
+ }
+ }
+
ret = rte_event_eth_rx_adapter_create(adapter_id, event_d_id,
&evt_rsrc->def_p_conf);
if (ret)
diff --git a/examples/l2fwd-event/main.c b/examples/l2fwd-event/main.c
index 1db89f2bd1..8d8267131d 100644
--- a/examples/l2fwd-event/main.c
+++ b/examples/l2fwd-event/main.c
@@ -25,6 +25,9 @@ l2fwd_event_usage(const char *prgname)
" --eventq-sched: Event queue schedule type, ordered, atomic or parallel.\n"
" Default: atomic\n"
" Valid only if --mode=eventdev\n"
+ " --enable-vector: Enable event vectorization.\n"
+ " --vector-size: Max vector size if event vectorization is enabled.\n"
+ " --vector-tmo-ns: Max timeout to form vector in nanoseconds if event vectorization is enabled\n"
" --config: Configure forwarding port pair mapping\n"
" Default: alternate port pairs\n\n",
prgname);
@@ -175,6 +178,9 @@ static const char short_options[] =
#define CMD_LINE_OPT_MODE "mode"
#define CMD_LINE_OPT_EVENTQ_SCHED "eventq-sched"
#define CMD_LINE_OPT_PORT_PAIR_CONF "config"
+#define CMD_LINE_OPT_ENABLE_VECTOR "enable-vector"
+#define CMD_LINE_OPT_VECTOR_SIZE "vector-size"
+#define CMD_LINE_OPT_VECTOR_TMO_NS "vector-tmo-ns"
enum {
/* long options mapped to a short option */
@@ -186,6 +192,9 @@ enum {
CMD_LINE_OPT_MODE_NUM,
CMD_LINE_OPT_EVENTQ_SCHED_NUM,
CMD_LINE_OPT_PORT_PAIR_CONF_NUM,
+ CMD_LINE_OPT_ENABLE_VECTOR_NUM,
+ CMD_LINE_OPT_VECTOR_SIZE_NUM,
+ CMD_LINE_OPT_VECTOR_TMO_NS_NUM
};
/* Parse the argument given in the command line of the application */
@@ -202,6 +211,12 @@ l2fwd_event_parse_args(int argc, char **argv, struct l2fwd_resources *rsrc)
CMD_LINE_OPT_EVENTQ_SCHED_NUM},
{ CMD_LINE_OPT_PORT_PAIR_CONF, required_argument, NULL,
CMD_LINE_OPT_PORT_PAIR_CONF_NUM},
+ {CMD_LINE_OPT_ENABLE_VECTOR, no_argument, NULL,
+ CMD_LINE_OPT_ENABLE_VECTOR_NUM},
+ {CMD_LINE_OPT_VECTOR_SIZE, required_argument, NULL,
+ CMD_LINE_OPT_VECTOR_SIZE_NUM},
+ {CMD_LINE_OPT_VECTOR_TMO_NS, required_argument, NULL,
+ CMD_LINE_OPT_VECTOR_TMO_NS_NUM},
{NULL, 0, 0, 0}
};
int opt, ret, timer_secs;
@@ -270,6 +285,16 @@ l2fwd_event_parse_args(int argc, char **argv, struct l2fwd_resources *rsrc)
return -1;
}
break;
+ case CMD_LINE_OPT_ENABLE_VECTOR_NUM:
+ printf("event vectorization is enabled\n");
+ rsrc->evt_vec.enabled = 1;
+ break;
+ case CMD_LINE_OPT_VECTOR_SIZE_NUM:
+ rsrc->evt_vec.size = strtol(optarg, NULL, 10);
+ break;
+ case CMD_LINE_OPT_VECTOR_TMO_NS_NUM:
+ rsrc->evt_vec.timeout_ns = strtoull(optarg, NULL, 10);
+ break;
/* long options */
case 0:
@@ -283,6 +308,18 @@ l2fwd_event_parse_args(int argc, char **argv, struct l2fwd_resources *rsrc)
rsrc->mac_updating = mac_updating;
+ if (rsrc->evt_vec.enabled && !rsrc->evt_vec.size) {
+ rsrc->evt_vec.size = VECTOR_SIZE_DEFAULT;
+ printf("vector size set to default (%" PRIu16 ")\n",
+ rsrc->evt_vec.size);
+ }
+
+ if (rsrc->evt_vec.enabled && !rsrc->evt_vec.timeout_ns) {
+ rsrc->evt_vec.timeout_ns = VECTOR_TMO_NS_DEFAULT;
+ printf("vector timeout set to default (%" PRIu64 " ns)\n",
+ rsrc->evt_vec.timeout_ns);
+ }
+
if (optind >= 0)
argv[optind-1] = prgname;
@@ -636,6 +673,17 @@ main(int argc, char **argv)
rte_panic("Cannot init mbuf pool\n");
/* >8 End of creation of mbuf pool. */
+ if (rsrc->evt_vec.enabled) {
+ unsigned int nb_vec, vec_size;
+
+ vec_size = rsrc->evt_vec.size;
+ nb_vec = (nb_mbufs + vec_size - 1) / vec_size;
+ rsrc->evt_vec_pool = rte_event_vector_pool_create(
+ "vector_pool", nb_vec, 0, vec_size, rte_socket_id());
+ if (rsrc->evt_vec_pool == NULL)
+ rte_panic("Cannot init event vector pool\n");
+ }
+
nb_ports_available = l2fwd_event_init_ports(rsrc);
if (!nb_ports_available)
rte_panic("All available ports are disabled. Please set portmask.\n");
--
2.25.1
^ permalink raw reply [flat|nested] 3+ messages in thread
* [dpdk-dev] [PATCH v2] examples/l2fwd-event: changes to use event vector
2021-09-04 7:46 [dpdk-dev] [PATCH] examples/l2fwd-event: changes to use event vector Shijith Thotton
@ 2021-09-27 6:21 ` Shijith Thotton
2021-10-20 16:15 ` Jerin Jacob
0 siblings, 1 reply; 3+ messages in thread
From: Shijith Thotton @ 2021-09-27 6:21 UTC (permalink / raw)
To: dev; +Cc: Shijith Thotton, jerinj, pbhagavatula, Sunil Kumar Kori
Added changes to receive packets as event vector. By default this is
disabled and can be enabled using the option --event-vector. Vector
size and timeout to form the vector can be configured using options
--event-vector-size and --event-vector-tmo.
Example:
dpdk-l2fwd-event -l 0-3 -n 4 -- -p 0x03 --mode=eventdev \
--eventq-sched=ordered --event-vector --event-vector-size 16
Signed-off-by: Shijith Thotton <sthotton@marvell.com>
---
v2:
* Added the prefix "event" to vector options.
doc/guides/sample_app_ug/l2_forward_event.rst | 13 +-
examples/l2fwd-event/l2fwd_common.h | 12 ++
examples/l2fwd-event/l2fwd_event.c | 204 ++++++++++++++++--
examples/l2fwd-event/l2fwd_event_generic.c | 21 ++
.../l2fwd-event/l2fwd_event_internal_port.c | 22 ++
examples/l2fwd-event/main.c | 48 +++++
6 files changed, 306 insertions(+), 14 deletions(-)
diff --git a/doc/guides/sample_app_ug/l2_forward_event.rst b/doc/guides/sample_app_ug/l2_forward_event.rst
index 540ca015db..904f6f1a4a 100644
--- a/doc/guides/sample_app_ug/l2_forward_event.rst
+++ b/doc/guides/sample_app_ug/l2_forward_event.rst
@@ -52,7 +52,12 @@ The application requires a number of command line options:
.. code-block:: console
- ./<build_dir>/examples/dpdk-l2fwd-event [EAL options] -- -p PORTMASK [-q NQ] --[no-]mac-updating --mode=MODE --eventq-sched=SCHED_MODE
+ ./<build_dir>/examples/dpdk-l2fwd-event [EAL options] -- -p PORTMASK
+ [-q NQ]
+ [--[no-]mac-updating]
+ [--mode=MODE]
+ [--eventq-sched=SCHED_MODE]
+ [--event-vector [--event-vector-size SIZE] [--event-vector-tmo NS]]
where,
@@ -68,6 +73,12 @@ where,
* --config: Configure forwarding port pair mapping. Alternate port pairs by default.
+* --event-vector: Enable event vectorization. Only valid if --mode=eventdev.
+
+* --event-vector-size: Max vector size if event vectorization is enabled.
+
+* --event-vector-tmo: Max timeout to form vector in nanoseconds if event vectorization is enabled.
+
Sample usage commands are given below to run the application into different mode:
Poll mode with 4 lcores, 16 ports and 8 RX queues per lcore and MAC address updating enabled,
diff --git a/examples/l2fwd-event/l2fwd_common.h b/examples/l2fwd-event/l2fwd_common.h
index 939221d45a..5e380ded16 100644
--- a/examples/l2fwd-event/l2fwd_common.h
+++ b/examples/l2fwd-event/l2fwd_common.h
@@ -56,6 +56,9 @@
#define DEFAULT_TIMER_PERIOD 10 /* default period is 10 seconds */
#define MAX_TIMER_PERIOD 86400 /* 1 day max */
+#define VECTOR_SIZE_DEFAULT MAX_PKT_BURST
+#define VECTOR_TMO_NS_DEFAULT 1E6 /* 1ms */
+
/* Per-port statistics struct */
struct l2fwd_port_statistics {
uint64_t dropped;
@@ -63,6 +66,13 @@ struct l2fwd_port_statistics {
uint64_t rx;
} __rte_cache_aligned;
+/* Event vector attributes */
+struct l2fwd_event_vector_params {
+ uint8_t enabled;
+ uint16_t size;
+ uint64_t timeout_ns;
+};
+
struct l2fwd_resources {
volatile uint8_t force_quit;
uint8_t event_mode;
@@ -75,9 +85,11 @@ struct l2fwd_resources {
uint32_t enabled_port_mask;
uint64_t timer_period;
struct rte_mempool *pktmbuf_pool;
+ struct rte_mempool *evt_vec_pool;
uint32_t dst_ports[RTE_MAX_ETHPORTS];
struct rte_ether_addr eth_addr[RTE_MAX_ETHPORTS];
struct l2fwd_port_statistics port_stats[RTE_MAX_ETHPORTS];
+ struct l2fwd_event_vector_params evt_vec;
void *evt_rsrc;
void *poll_rsrc;
} __rte_cache_aligned;
diff --git a/examples/l2fwd-event/l2fwd_event.c b/examples/l2fwd-event/l2fwd_event.c
index 7ba5311d66..09080fa666 100644
--- a/examples/l2fwd-event/l2fwd_event.c
+++ b/examples/l2fwd-event/l2fwd_event.c
@@ -346,19 +346,198 @@ l2fwd_event_main_loop_tx_q_brst_mac(struct l2fwd_resources *rsrc)
L2FWD_EVENT_TX_ENQ | L2FWD_EVENT_BURST);
}
+static __rte_always_inline void
+l2fwd_event_vector_fwd(struct l2fwd_resources *rsrc,
+ struct rte_event_vector *vec,
+ const uint64_t timer_period, const uint32_t flags)
+{
+ struct rte_mbuf **mbufs = vec->mbufs;
+ uint16_t i, j;
+
+ rte_prefetch0(rte_pktmbuf_mtod(mbufs[0], void *));
+
+ /* If vector attribute is valid, mbufs will be from same port/queue */
+ if (vec->attr_valid) {
+ vec->port = rsrc->dst_ports[mbufs[0]->port];
+ if (flags & L2FWD_EVENT_TX_DIRECT)
+ vec->queue = 0;
+
+ if (timer_period > 0)
+ __atomic_fetch_add(&rsrc->port_stats[mbufs[0]->port].rx,
+ vec->nb_elem, __ATOMIC_RELAXED);
+
+ for (i = 0, j = 1; i < vec->nb_elem; i++, j++) {
+ if (j < vec->nb_elem)
+ rte_prefetch0(
+ rte_pktmbuf_mtod(mbufs[j], void *));
+
+ if (flags & L2FWD_EVENT_UPDT_MAC)
+ l2fwd_mac_updating(
+ mbufs[i], vec->port,
+ &rsrc->eth_addr[vec->port]);
+ }
+
+ if (timer_period > 0)
+ __atomic_fetch_add(&rsrc->port_stats[vec->port].tx,
+ vec->nb_elem, __ATOMIC_RELAXED);
+ } else {
+ for (i = 0, j = 1; i < vec->nb_elem; i++, j++) {
+ if (timer_period > 0)
+ __atomic_fetch_add(
+ &rsrc->port_stats[mbufs[i]->port].rx, 1,
+ __ATOMIC_RELAXED);
+
+ if (j < vec->nb_elem)
+ rte_prefetch0(
+ rte_pktmbuf_mtod(mbufs[j], void *));
+
+ mbufs[i]->port = rsrc->dst_ports[mbufs[i]->port];
+
+ if (flags & L2FWD_EVENT_UPDT_MAC)
+ l2fwd_mac_updating(
+ mbufs[i], mbufs[i]->port,
+ &rsrc->eth_addr[mbufs[i]->port]);
+
+ if (flags & L2FWD_EVENT_TX_DIRECT)
+ rte_event_eth_tx_adapter_txq_set(mbufs[i], 0);
+
+ if (timer_period > 0)
+ __atomic_fetch_add(
+ &rsrc->port_stats[mbufs[i]->port].tx, 1,
+ __ATOMIC_RELAXED);
+ }
+ }
+}
+
+static __rte_always_inline void
+l2fwd_event_loop_vector(struct l2fwd_resources *rsrc, const uint32_t flags)
+{
+ struct l2fwd_event_resources *evt_rsrc = rsrc->evt_rsrc;
+ const int port_id = l2fwd_get_free_event_port(evt_rsrc);
+ const uint8_t tx_q_id =
+ evt_rsrc->evq.event_q_id[evt_rsrc->evq.nb_queues - 1];
+ const uint64_t timer_period = rsrc->timer_period;
+ const uint8_t event_d_id = evt_rsrc->event_d_id;
+ const uint8_t deq_len = evt_rsrc->deq_depth;
+ struct rte_event ev[MAX_PKT_BURST];
+ uint16_t nb_rx, nb_tx;
+ uint8_t i;
+
+ if (port_id < 0)
+ return;
+
+ printf("%s(): entering eventdev main loop on lcore %u\n", __func__,
+ rte_lcore_id());
+
+ while (!rsrc->force_quit) {
+ nb_rx = rte_event_dequeue_burst(event_d_id, port_id, ev,
+ deq_len, 0);
+ if (nb_rx == 0)
+ continue;
+
+ for (i = 0; i < nb_rx; i++) {
+ if (flags & L2FWD_EVENT_TX_ENQ) {
+ ev[i].queue_id = tx_q_id;
+ ev[i].op = RTE_EVENT_OP_FORWARD;
+ }
+
+ l2fwd_event_vector_fwd(rsrc, ev[i].vec, timer_period,
+ flags);
+ }
+
+ if (flags & L2FWD_EVENT_TX_ENQ) {
+ nb_tx = rte_event_enqueue_burst(event_d_id, port_id, ev,
+ nb_rx);
+ while (nb_tx < nb_rx && !rsrc->force_quit)
+ nb_tx += rte_event_enqueue_burst(
+ event_d_id, port_id, ev + nb_tx,
+ nb_rx - nb_tx);
+ }
+
+ if (flags & L2FWD_EVENT_TX_DIRECT) {
+ nb_tx = rte_event_eth_tx_adapter_enqueue(
+ event_d_id, port_id, ev, nb_rx, 0);
+ while (nb_tx < nb_rx && !rsrc->force_quit)
+ nb_tx += rte_event_eth_tx_adapter_enqueue(
+ event_d_id, port_id, ev + nb_tx,
+ nb_rx - nb_tx, 0);
+ }
+ }
+}
+
+static void __rte_noinline
+l2fwd_event_main_loop_tx_d_vec(struct l2fwd_resources *rsrc)
+{
+ l2fwd_event_loop_vector(rsrc, L2FWD_EVENT_TX_DIRECT);
+}
+
+static void __rte_noinline
+l2fwd_event_main_loop_tx_d_brst_vec(struct l2fwd_resources *rsrc)
+{
+ l2fwd_event_loop_vector(rsrc, L2FWD_EVENT_TX_DIRECT);
+}
+
+static void __rte_noinline
+l2fwd_event_main_loop_tx_q_vec(struct l2fwd_resources *rsrc)
+{
+ l2fwd_event_loop_vector(rsrc, L2FWD_EVENT_TX_ENQ);
+}
+
+static void __rte_noinline
+l2fwd_event_main_loop_tx_q_brst_vec(struct l2fwd_resources *rsrc)
+{
+ l2fwd_event_loop_vector(rsrc, L2FWD_EVENT_TX_ENQ);
+}
+
+static void __rte_noinline
+l2fwd_event_main_loop_tx_d_mac_vec(struct l2fwd_resources *rsrc)
+{
+ l2fwd_event_loop_vector(rsrc,
+ L2FWD_EVENT_UPDT_MAC | L2FWD_EVENT_TX_DIRECT);
+}
+
+static void __rte_noinline
+l2fwd_event_main_loop_tx_d_brst_mac_vec(struct l2fwd_resources *rsrc)
+{
+ l2fwd_event_loop_vector(rsrc,
+ L2FWD_EVENT_UPDT_MAC | L2FWD_EVENT_TX_DIRECT);
+}
+
+static void __rte_noinline
+l2fwd_event_main_loop_tx_q_mac_vec(struct l2fwd_resources *rsrc)
+{
+ l2fwd_event_loop_vector(rsrc,
+ L2FWD_EVENT_UPDT_MAC | L2FWD_EVENT_TX_ENQ);
+}
+
+static void __rte_noinline
+l2fwd_event_main_loop_tx_q_brst_mac_vec(struct l2fwd_resources *rsrc)
+{
+ l2fwd_event_loop_vector(rsrc,
+ L2FWD_EVENT_UPDT_MAC | L2FWD_EVENT_TX_ENQ);
+}
+
void
l2fwd_event_resource_setup(struct l2fwd_resources *rsrc)
{
/* [MAC_UPDT][TX_MODE][BURST] */
- const event_loop_cb event_loop[2][2][2] = {
- [0][0][0] = l2fwd_event_main_loop_tx_d,
- [0][0][1] = l2fwd_event_main_loop_tx_d_brst,
- [0][1][0] = l2fwd_event_main_loop_tx_q,
- [0][1][1] = l2fwd_event_main_loop_tx_q_brst,
- [1][0][0] = l2fwd_event_main_loop_tx_d_mac,
- [1][0][1] = l2fwd_event_main_loop_tx_d_brst_mac,
- [1][1][0] = l2fwd_event_main_loop_tx_q_mac,
- [1][1][1] = l2fwd_event_main_loop_tx_q_brst_mac,
+ const event_loop_cb event_loop[2][2][2][2] = {
+ [0][0][0][0] = l2fwd_event_main_loop_tx_d,
+ [0][0][0][1] = l2fwd_event_main_loop_tx_d_brst,
+ [0][0][1][0] = l2fwd_event_main_loop_tx_q,
+ [0][0][1][1] = l2fwd_event_main_loop_tx_q_brst,
+ [0][1][0][0] = l2fwd_event_main_loop_tx_d_mac,
+ [0][1][0][1] = l2fwd_event_main_loop_tx_d_brst_mac,
+ [0][1][1][0] = l2fwd_event_main_loop_tx_q_mac,
+ [0][1][1][1] = l2fwd_event_main_loop_tx_q_brst_mac,
+ [1][0][0][0] = l2fwd_event_main_loop_tx_d_vec,
+ [1][0][0][1] = l2fwd_event_main_loop_tx_d_brst_vec,
+ [1][0][1][0] = l2fwd_event_main_loop_tx_q_vec,
+ [1][0][1][1] = l2fwd_event_main_loop_tx_q_brst_vec,
+ [1][1][0][0] = l2fwd_event_main_loop_tx_d_mac_vec,
+ [1][1][0][1] = l2fwd_event_main_loop_tx_d_brst_mac_vec,
+ [1][1][1][0] = l2fwd_event_main_loop_tx_q_mac_vec,
+ [1][1][1][1] = l2fwd_event_main_loop_tx_q_brst_mac_vec,
};
struct l2fwd_event_resources *evt_rsrc;
uint32_t event_queue_cfg;
@@ -394,8 +573,7 @@ l2fwd_event_resource_setup(struct l2fwd_resources *rsrc)
if (ret < 0)
rte_panic("Error in starting eventdev\n");
- evt_rsrc->ops.l2fwd_event_loop = event_loop
- [rsrc->mac_updating]
- [evt_rsrc->tx_mode_q]
- [evt_rsrc->has_burst];
+ evt_rsrc->ops.l2fwd_event_loop =
+ event_loop[rsrc->evt_vec.enabled][rsrc->mac_updating]
+ [evt_rsrc->tx_mode_q][evt_rsrc->has_burst];
}
diff --git a/examples/l2fwd-event/l2fwd_event_generic.c b/examples/l2fwd-event/l2fwd_event_generic.c
index c19b6f1411..203509dad0 100644
--- a/examples/l2fwd-event/l2fwd_event_generic.c
+++ b/examples/l2fwd-event/l2fwd_event_generic.c
@@ -245,6 +245,27 @@ l2fwd_rx_tx_adapter_setup_generic(struct l2fwd_resources *rsrc)
if ((rsrc->enabled_port_mask & (1 << port_id)) == 0)
continue;
eth_q_conf.ev.queue_id = evt_rsrc->evq.event_q_id[i];
+ if (rsrc->evt_vec.enabled) {
+ uint32_t cap;
+
+ if (rte_event_eth_rx_adapter_caps_get(event_d_id,
+ port_id, &cap))
+ rte_panic(
+ "Failed to get event rx adapter capability");
+
+ if (cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_EVENT_VECTOR) {
+ eth_q_conf.vector_sz = rsrc->evt_vec.size;
+ eth_q_conf.vector_timeout_ns =
+ rsrc->evt_vec.timeout_ns;
+ eth_q_conf.vector_mp = rsrc->evt_vec_pool;
+ eth_q_conf.rx_queue_flags |=
+ RTE_EVENT_ETH_RX_ADAPTER_QUEUE_EVENT_VECTOR;
+ } else {
+ rte_panic(
+ "Rx adapter doesn't support event vector");
+ }
+ }
+
ret = rte_event_eth_rx_adapter_queue_add(rx_adptr_id, port_id,
-1, ð_q_conf);
if (ret)
diff --git a/examples/l2fwd-event/l2fwd_event_internal_port.c b/examples/l2fwd-event/l2fwd_event_internal_port.c
index 16a52d67c9..5efd44f950 100644
--- a/examples/l2fwd-event/l2fwd_event_internal_port.c
+++ b/examples/l2fwd-event/l2fwd_event_internal_port.c
@@ -230,6 +230,28 @@ l2fwd_rx_tx_adapter_setup_internal_port(struct l2fwd_resources *rsrc)
RTE_ETH_FOREACH_DEV(port_id) {
if ((rsrc->enabled_port_mask & (1 << port_id)) == 0)
continue;
+
+ if (rsrc->evt_vec.enabled) {
+ uint32_t cap;
+
+ if (rte_event_eth_rx_adapter_caps_get(event_d_id,
+ port_id, &cap))
+ rte_panic(
+ "Failed to get event rx adapter capability");
+
+ if (cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_EVENT_VECTOR) {
+ eth_q_conf.vector_sz = rsrc->evt_vec.size;
+ eth_q_conf.vector_timeout_ns =
+ rsrc->evt_vec.timeout_ns;
+ eth_q_conf.vector_mp = rsrc->evt_vec_pool;
+ eth_q_conf.rx_queue_flags |=
+ RTE_EVENT_ETH_RX_ADAPTER_QUEUE_EVENT_VECTOR;
+ } else {
+ rte_panic(
+ "Rx adapter doesn't support event vector");
+ }
+ }
+
ret = rte_event_eth_rx_adapter_create(adapter_id, event_d_id,
&evt_rsrc->def_p_conf);
if (ret)
diff --git a/examples/l2fwd-event/main.c b/examples/l2fwd-event/main.c
index 1db89f2bd1..f2adc66446 100644
--- a/examples/l2fwd-event/main.c
+++ b/examples/l2fwd-event/main.c
@@ -25,6 +25,9 @@ l2fwd_event_usage(const char *prgname)
" --eventq-sched: Event queue schedule type, ordered, atomic or parallel.\n"
" Default: atomic\n"
" Valid only if --mode=eventdev\n"
+ " --event-vector: Enable event vectorization.\n"
+ " --event-vector-size: Max vector size if event vectorization is enabled.\n"
+ " --event-vector-tmo: Max timeout to form vector in nanoseconds if event vectorization is enabled\n"
" --config: Configure forwarding port pair mapping\n"
" Default: alternate port pairs\n\n",
prgname);
@@ -175,6 +178,9 @@ static const char short_options[] =
#define CMD_LINE_OPT_MODE "mode"
#define CMD_LINE_OPT_EVENTQ_SCHED "eventq-sched"
#define CMD_LINE_OPT_PORT_PAIR_CONF "config"
+#define CMD_LINE_OPT_ENABLE_VECTOR "event-vector"
+#define CMD_LINE_OPT_VECTOR_SIZE "event-vector-size"
+#define CMD_LINE_OPT_VECTOR_TMO_NS "event-vector-tmo"
enum {
/* long options mapped to a short option */
@@ -186,6 +192,9 @@ enum {
CMD_LINE_OPT_MODE_NUM,
CMD_LINE_OPT_EVENTQ_SCHED_NUM,
CMD_LINE_OPT_PORT_PAIR_CONF_NUM,
+ CMD_LINE_OPT_ENABLE_VECTOR_NUM,
+ CMD_LINE_OPT_VECTOR_SIZE_NUM,
+ CMD_LINE_OPT_VECTOR_TMO_NS_NUM
};
/* Parse the argument given in the command line of the application */
@@ -202,6 +211,12 @@ l2fwd_event_parse_args(int argc, char **argv, struct l2fwd_resources *rsrc)
CMD_LINE_OPT_EVENTQ_SCHED_NUM},
{ CMD_LINE_OPT_PORT_PAIR_CONF, required_argument, NULL,
CMD_LINE_OPT_PORT_PAIR_CONF_NUM},
+ {CMD_LINE_OPT_ENABLE_VECTOR, no_argument, NULL,
+ CMD_LINE_OPT_ENABLE_VECTOR_NUM},
+ {CMD_LINE_OPT_VECTOR_SIZE, required_argument, NULL,
+ CMD_LINE_OPT_VECTOR_SIZE_NUM},
+ {CMD_LINE_OPT_VECTOR_TMO_NS, required_argument, NULL,
+ CMD_LINE_OPT_VECTOR_TMO_NS_NUM},
{NULL, 0, 0, 0}
};
int opt, ret, timer_secs;
@@ -270,6 +285,16 @@ l2fwd_event_parse_args(int argc, char **argv, struct l2fwd_resources *rsrc)
return -1;
}
break;
+ case CMD_LINE_OPT_ENABLE_VECTOR_NUM:
+ printf("event vectorization is enabled\n");
+ rsrc->evt_vec.enabled = 1;
+ break;
+ case CMD_LINE_OPT_VECTOR_SIZE_NUM:
+ rsrc->evt_vec.size = strtol(optarg, NULL, 10);
+ break;
+ case CMD_LINE_OPT_VECTOR_TMO_NS_NUM:
+ rsrc->evt_vec.timeout_ns = strtoull(optarg, NULL, 10);
+ break;
/* long options */
case 0:
@@ -283,6 +308,18 @@ l2fwd_event_parse_args(int argc, char **argv, struct l2fwd_resources *rsrc)
rsrc->mac_updating = mac_updating;
+ if (rsrc->evt_vec.enabled && !rsrc->evt_vec.size) {
+ rsrc->evt_vec.size = VECTOR_SIZE_DEFAULT;
+ printf("vector size set to default (%" PRIu16 ")\n",
+ rsrc->evt_vec.size);
+ }
+
+ if (rsrc->evt_vec.enabled && !rsrc->evt_vec.timeout_ns) {
+ rsrc->evt_vec.timeout_ns = VECTOR_TMO_NS_DEFAULT;
+ printf("vector timeout set to default (%" PRIu64 " ns)\n",
+ rsrc->evt_vec.timeout_ns);
+ }
+
if (optind >= 0)
argv[optind-1] = prgname;
@@ -636,6 +673,17 @@ main(int argc, char **argv)
rte_panic("Cannot init mbuf pool\n");
/* >8 End of creation of mbuf pool. */
+ if (rsrc->evt_vec.enabled) {
+ unsigned int nb_vec, vec_size;
+
+ vec_size = rsrc->evt_vec.size;
+ nb_vec = (nb_mbufs + vec_size - 1) / vec_size;
+ rsrc->evt_vec_pool = rte_event_vector_pool_create(
+ "vector_pool", nb_vec, 0, vec_size, rte_socket_id());
+ if (rsrc->evt_vec_pool == NULL)
+ rte_panic("Cannot init event vector pool\n");
+ }
+
nb_ports_available = l2fwd_event_init_ports(rsrc);
if (!nb_ports_available)
rte_panic("All available ports are disabled. Please set portmask.\n");
--
2.25.1
^ permalink raw reply [flat|nested] 3+ messages in thread
* Re: [dpdk-dev] [PATCH v2] examples/l2fwd-event: changes to use event vector
2021-09-27 6:21 ` [dpdk-dev] [PATCH v2] " Shijith Thotton
@ 2021-10-20 16:15 ` Jerin Jacob
0 siblings, 0 replies; 3+ messages in thread
From: Jerin Jacob @ 2021-10-20 16:15 UTC (permalink / raw)
To: Shijith Thotton; +Cc: dpdk-dev, Jerin Jacob, Pavan Nikhilesh, Sunil Kumar Kori
On Mon, Sep 27, 2021 at 11:51 AM Shijith Thotton <sthotton@marvell.com> wrote:
>
> Added changes to receive packets as event vector. By default this is
> disabled and can be enabled using the option --event-vector. Vector
> size and timeout to form the vector can be configured using options
> --event-vector-size and --event-vector-tmo.
>
> Example:
> dpdk-l2fwd-event -l 0-3 -n 4 -- -p 0x03 --mode=eventdev \
> --eventq-sched=ordered --event-vector --event-vector-size 16
>
> Signed-off-by: Shijith Thotton <sthotton@marvell.com>
Changed the subject to: examples/l2fwd: support event vector
Acked-by: Jerin Jacob <jerinj@marvell.com>
Applied to dpdk-next-eventdev/for-main. Thanks.
^ permalink raw reply [flat|nested] 3+ messages in thread
end of thread, other threads:[~2021-10-20 16:15 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-09-04 7:46 [dpdk-dev] [PATCH] examples/l2fwd-event: changes to use event vector Shijith Thotton
2021-09-27 6:21 ` [dpdk-dev] [PATCH v2] " Shijith Thotton
2021-10-20 16:15 ` Jerin Jacob
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).