* [dpdk-dev] [RFC 1/2] sched: new APIs for per-subport queue sizes
@ 2018-02-28 13:39 alangordondewar
2018-02-28 13:39 ` [dpdk-dev] [RFC 2/2] sched: support per-subport wred configurations alangordondewar
0 siblings, 1 reply; 2+ messages in thread
From: alangordondewar @ 2018-02-28 13:39 UTC (permalink / raw)
To: cristian.dumitrescu; +Cc: dev, Alan Dewar
From: Alan Dewar <alan.dewar@att.com>
Added new APIs to allow the maximum queue sizes for each traffic class
to be configured on a per-subport basis, rather than all subport's
inheriting their maximum queue sizes from their parent port.
Added new sched unit-test to exercise the new APIs.
Signed-off-by: Alan Dewar <alan.dewar@att.com>
---
lib/librte_sched/rte_sched.c | 243 +++++++++++++++++++++---------
lib/librte_sched/rte_sched.h | 48 ++++++
lib/librte_sched/rte_sched_version.map | 8 +
test/test/test_sched.c | 260 ++++++++++++++++++++++++++++++++-
4 files changed, 485 insertions(+), 74 deletions(-)
diff --git a/lib/librte_sched/rte_sched.c b/lib/librte_sched/rte_sched.c
index 634486c..9436ba5 100644
--- a/lib/librte_sched/rte_sched.c
+++ b/lib/librte_sched/rte_sched.c
@@ -58,6 +58,7 @@ struct rte_sched_subport {
uint64_t tc_time; /* time of next update */
uint32_t tc_credits_per_period[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
uint32_t tc_credits[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
+ uint16_t qsize[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
uint32_t tc_period;
/* TC oversubscription */
@@ -71,6 +72,11 @@ struct rte_sched_subport {
/* Statistics */
struct rte_sched_subport_stats stats;
+
+ /* Queue base calculation */
+ uint32_t qsize_add[RTE_SCHED_QUEUES_PER_PIPE];
+ uint32_t qsize_sum;
+ uint32_t qoffset;
};
struct rte_sched_pipe_profile {
@@ -215,10 +221,6 @@ struct rte_sched_port {
struct rte_mbuf **pkts_out;
uint32_t n_pkts_out;
- /* Queue base calculation */
- uint32_t qsize_add[RTE_SCHED_QUEUES_PER_PIPE];
- uint32_t qsize_sum;
-
/* Large data structures */
struct rte_sched_subport *subport;
struct rte_sched_pipe *pipe;
@@ -241,16 +243,12 @@ enum rte_sched_port_array {
e_RTE_SCHED_PORT_ARRAY_TOTAL,
};
-#ifdef RTE_SCHED_COLLECT_STATS
-
static inline uint32_t
rte_sched_port_queues_per_subport(struct rte_sched_port *port)
{
return RTE_SCHED_QUEUES_PER_PIPE * port->n_pipes_per_subport;
}
-#endif
-
static inline uint32_t
rte_sched_port_queues_per_port(struct rte_sched_port *port)
{
@@ -260,19 +258,27 @@ rte_sched_port_queues_per_port(struct rte_sched_port *port)
static inline struct rte_mbuf **
rte_sched_port_qbase(struct rte_sched_port *port, uint32_t qindex)
{
- uint32_t pindex = qindex >> 4;
uint32_t qpos = qindex & 0xF;
+ uint32_t subport_id = qindex / rte_sched_port_queues_per_subport(port);
+ struct rte_sched_subport *subport = port->subport + subport_id;
+ uint32_t subport_pipe_offset;
+
+ subport_pipe_offset = qindex % rte_sched_port_queues_per_subport(port);
+ subport_pipe_offset /= RTE_SCHED_QUEUES_PER_PIPE;
+ subport_pipe_offset *= RTE_SCHED_QUEUES_PER_PIPE;
- return (port->queue_array + pindex *
- port->qsize_sum + port->qsize_add[qpos]);
+ return (port->queue_array + subport->qoffset + subport_pipe_offset +
+ subport->qsize_add[qpos]);
}
static inline uint16_t
rte_sched_port_qsize(struct rte_sched_port *port, uint32_t qindex)
{
uint32_t tc = (qindex >> 2) & 0x3;
+ uint32_t subport_id = qindex / rte_sched_port_queues_per_subport(port);
+ struct rte_sched_subport *subport = port->subport + subport_id;
- return port->qsize[tc];
+ return subport->qsize[tc];
}
static int
@@ -360,7 +366,9 @@ rte_sched_port_check_params(struct rte_sched_port_params *params)
}
static uint32_t
-rte_sched_port_get_array_base(struct rte_sched_port_params *params, enum rte_sched_port_array array)
+rte_sched_port_get_array_base(struct rte_sched_port_params *params,
+ enum rte_sched_port_array array,
+ uint32_t size_queue_array)
{
uint32_t n_subports_per_port = params->n_subports_per_port;
uint32_t n_pipes_per_subport = params->n_pipes_per_subport;
@@ -375,16 +383,19 @@ rte_sched_port_get_array_base(struct rte_sched_port_params *params, enum rte_sch
uint32_t size_pipe_profiles
= RTE_SCHED_PIPE_PROFILES_PER_PORT * sizeof(struct rte_sched_pipe_profile);
uint32_t size_bmp_array = rte_bitmap_get_memory_footprint(n_queues_per_port);
- uint32_t size_per_pipe_queue_array, size_queue_array;
+ uint32_t size_per_pipe_queue_array;
uint32_t base, i;
- size_per_pipe_queue_array = 0;
- for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) {
- size_per_pipe_queue_array += RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS
- * params->qsize[i] * sizeof(struct rte_mbuf *);
+ if (size_queue_array == 0) {
+ size_per_pipe_queue_array = 0;
+ for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) {
+ size_per_pipe_queue_array +=
+ RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS
+ * params->qsize[i] * sizeof(struct rte_mbuf *);
+ }
+ size_queue_array = n_pipes_per_port * size_per_pipe_queue_array;
}
- size_queue_array = n_pipes_per_port * size_per_pipe_queue_array;
base = 0;
@@ -419,8 +430,9 @@ rte_sched_port_get_array_base(struct rte_sched_port_params *params, enum rte_sch
return base;
}
-uint32_t
-rte_sched_port_get_memory_footprint(struct rte_sched_port_params *params)
+static uint32_t
+rte_sched_port_get_memory_footprint_common(struct rte_sched_port_params *params,
+ uint32_t size_queue_array)
{
uint32_t size0, size1;
int status;
@@ -434,39 +446,93 @@ rte_sched_port_get_memory_footprint(struct rte_sched_port_params *params)
}
size0 = sizeof(struct rte_sched_port);
- size1 = rte_sched_port_get_array_base(params, e_RTE_SCHED_PORT_ARRAY_TOTAL);
+ size1 = rte_sched_port_get_array_base(params,
+ e_RTE_SCHED_PORT_ARRAY_TOTAL,
+ size_queue_array);
return size0 + size1;
}
+uint32_t
+rte_sched_port_get_memory_footprint(struct rte_sched_port_params *params)
+{
+ return rte_sched_port_get_memory_footprint_common(params, 0);
+}
+
+uint32_t
+rte_sched_port_get_memory_footprint_v2(struct rte_sched_port_params *params,
+ uint32_t size_queue_array)
+{
+ return rte_sched_port_get_memory_footprint_common(params,
+ size_queue_array);
+}
+
static void
-rte_sched_port_config_qsize(struct rte_sched_port *port)
+rte_sched_subport_config_qsize(struct rte_sched_port *port,
+ uint32_t subport_id,
+ uint16_t *qsize)
{
+ struct rte_sched_subport *subport = port->subport + subport_id;
+ uint32_t tc;
+
+ for (tc = 0; tc < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; tc++) {
+ if (qsize == NULL)
+ /* The subport inherits its qsizes from the port */
+ subport->qsize[tc] = port->qsize[tc];
+ else
+ /* The subport has explicity configured qsizes */
+ subport->qsize[tc] = qsize[tc];
+ }
+
/* TC 0 */
- port->qsize_add[0] = 0;
- port->qsize_add[1] = port->qsize_add[0] + port->qsize[0];
- port->qsize_add[2] = port->qsize_add[1] + port->qsize[0];
- port->qsize_add[3] = port->qsize_add[2] + port->qsize[0];
+ subport->qsize_add[0] = 0;
+ subport->qsize_add[1] = subport->qsize_add[0] + subport->qsize[0];
+ subport->qsize_add[2] = subport->qsize_add[1] + subport->qsize[0];
+ subport->qsize_add[3] = subport->qsize_add[2] + subport->qsize[0];
/* TC 1 */
- port->qsize_add[4] = port->qsize_add[3] + port->qsize[0];
- port->qsize_add[5] = port->qsize_add[4] + port->qsize[1];
- port->qsize_add[6] = port->qsize_add[5] + port->qsize[1];
- port->qsize_add[7] = port->qsize_add[6] + port->qsize[1];
+ subport->qsize_add[4] = subport->qsize_add[3] + subport->qsize[0];
+ subport->qsize_add[5] = subport->qsize_add[4] + subport->qsize[1];
+ subport->qsize_add[6] = subport->qsize_add[5] + subport->qsize[1];
+ subport->qsize_add[7] = subport->qsize_add[6] + subport->qsize[1];
/* TC 2 */
- port->qsize_add[8] = port->qsize_add[7] + port->qsize[1];
- port->qsize_add[9] = port->qsize_add[8] + port->qsize[2];
- port->qsize_add[10] = port->qsize_add[9] + port->qsize[2];
- port->qsize_add[11] = port->qsize_add[10] + port->qsize[2];
+ subport->qsize_add[8] = subport->qsize_add[7] + subport->qsize[1];
+ subport->qsize_add[9] = subport->qsize_add[8] + subport->qsize[2];
+ subport->qsize_add[10] = subport->qsize_add[9] + subport->qsize[2];
+ subport->qsize_add[11] = subport->qsize_add[10] + subport->qsize[2];
/* TC 3 */
- port->qsize_add[12] = port->qsize_add[11] + port->qsize[2];
- port->qsize_add[13] = port->qsize_add[12] + port->qsize[3];
- port->qsize_add[14] = port->qsize_add[13] + port->qsize[3];
- port->qsize_add[15] = port->qsize_add[14] + port->qsize[3];
+ subport->qsize_add[12] = subport->qsize_add[11] + subport->qsize[2];
+ subport->qsize_add[13] = subport->qsize_add[12] + subport->qsize[3];
+ subport->qsize_add[14] = subport->qsize_add[13] + subport->qsize[3];
+ subport->qsize_add[15] = subport->qsize_add[14] + subport->qsize[3];
+
+ subport->qsize_sum = subport->qsize_add[15] + subport->qsize[3];
- port->qsize_sum = port->qsize_add[15] + port->qsize[3];
+ if (subport_id != 0) {
+ struct rte_sched_subport *prev = port->subport +
+ (subport_id - 1);
+
+ subport->qoffset = prev->qoffset + prev->qsize_sum;
+ }
+}
+
+static char *
+rte_sched_build_queue_size_string(uint16_t *qsize, char *output_str)
+{
+ uint32_t tc;
+ int str_len;
+
+ str_len = sprintf(output_str, "[");
+ for (tc = 0; tc < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; tc++) {
+ str_len += sprintf(output_str + str_len, "%u",
+ qsize[tc]);
+ if (tc != RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE - 1)
+ str_len += sprintf(output_str + str_len, ", ");
+ }
+ sprintf(output_str + str_len, "]");
+ return output_str;
}
static void
@@ -590,16 +656,12 @@ rte_sched_port_config_pipe_profile_table(struct rte_sched_port *port, struct rte
}
}
-struct rte_sched_port *
-rte_sched_port_config(struct rte_sched_port_params *params)
+static struct rte_sched_port *
+rte_sched_port_config_common(struct rte_sched_port_params *params,
+ uint32_t mem_size)
{
struct rte_sched_port *port = NULL;
- uint32_t mem_size, bmp_mem_size, n_queues_per_port, i, cycles_per_byte;
-
- /* Check user parameters. Determine the amount of memory to allocate */
- mem_size = rte_sched_port_get_memory_footprint(params);
- if (mem_size == 0)
- return NULL;
+ uint32_t bmp_mem_size, n_queues_per_port, i, cycles_per_byte;
/* Allocate memory to store the data structures */
port = rte_zmalloc("qos_params", mem_size, RTE_CACHE_LINE_SIZE);
@@ -659,30 +721,28 @@ rte_sched_port_config(struct rte_sched_port_params *params)
port->pkts_out = NULL;
port->n_pkts_out = 0;
- /* Queue base calculation */
- rte_sched_port_config_qsize(port);
-
/* Large data structures */
port->subport = (struct rte_sched_subport *)
(port->memory + rte_sched_port_get_array_base(params,
- e_RTE_SCHED_PORT_ARRAY_SUBPORT));
+ e_RTE_SCHED_PORT_ARRAY_SUBPORT, 0));
port->pipe = (struct rte_sched_pipe *)
(port->memory + rte_sched_port_get_array_base(params,
- e_RTE_SCHED_PORT_ARRAY_PIPE));
+ e_RTE_SCHED_PORT_ARRAY_PIPE, 0));
port->queue = (struct rte_sched_queue *)
(port->memory + rte_sched_port_get_array_base(params,
- e_RTE_SCHED_PORT_ARRAY_QUEUE));
+ e_RTE_SCHED_PORT_ARRAY_QUEUE, 0));
port->queue_extra = (struct rte_sched_queue_extra *)
(port->memory + rte_sched_port_get_array_base(params,
- e_RTE_SCHED_PORT_ARRAY_QUEUE_EXTRA));
+ e_RTE_SCHED_PORT_ARRAY_QUEUE_EXTRA, 0));
port->pipe_profiles = (struct rte_sched_pipe_profile *)
(port->memory + rte_sched_port_get_array_base(params,
- e_RTE_SCHED_PORT_ARRAY_PIPE_PROFILES));
+ e_RTE_SCHED_PORT_ARRAY_PIPE_PROFILES, 0));
port->bmp_array = port->memory
- + rte_sched_port_get_array_base(params, e_RTE_SCHED_PORT_ARRAY_BMP_ARRAY);
+ + rte_sched_port_get_array_base(params,
+ e_RTE_SCHED_PORT_ARRAY_BMP_ARRAY, 0);
port->queue_array = (struct rte_mbuf **)
(port->memory + rte_sched_port_get_array_base(params,
- e_RTE_SCHED_PORT_ARRAY_QUEUE_ARRAY));
+ e_RTE_SCHED_PORT_ARRAY_QUEUE_ARRAY, 0));
/* Pipe profile table */
rte_sched_port_config_pipe_profile_table(port, params);
@@ -704,6 +764,35 @@ rte_sched_port_config(struct rte_sched_port_params *params)
return port;
}
+
+struct rte_sched_port *
+rte_sched_port_config(struct rte_sched_port_params *params)
+{
+ uint32_t mem_size;
+
+ /* Check user parameters. Determine the amount of memory to allocate */
+ mem_size = rte_sched_port_get_memory_footprint(params);
+ if (mem_size == 0)
+ return NULL;
+
+ return rte_sched_port_config_common(params, mem_size);
+}
+
+struct rte_sched_port *
+rte_sched_port_config_v2(struct rte_sched_port_params *params,
+ uint32_t queue_array_size)
+{
+ uint32_t mem_size;
+
+ /* Check user parameters. Determine the amount of memory to allocate */
+ mem_size = rte_sched_port_get_memory_footprint_common(params,
+ queue_array_size);
+ if (mem_size == 0)
+ return NULL;
+
+ return rte_sched_port_config_common(params, mem_size);
+}
+
void
rte_sched_port_free(struct rte_sched_port *port)
{
@@ -736,10 +825,13 @@ static void
rte_sched_port_log_subport_config(struct rte_sched_port *port, uint32_t i)
{
struct rte_sched_subport *s = port->subport + i;
+ char queue_size_str[(7 * RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE) + 3];
+ rte_sched_build_queue_size_string(s->qsize, queue_size_str);
RTE_LOG(DEBUG, SCHED, "Low level config for subport %u:\n"
" Token bucket: period = %u, credits per period = %u, size = %u\n"
" Traffic classes: period = %u, credits per period = [%u, %u, %u, %u]\n"
+ " Traffic class queue-sizes: %s\n"
" Traffic class 3 oversubscription: wm min = %u, wm max = %u\n",
i,
@@ -754,16 +846,18 @@ rte_sched_port_log_subport_config(struct rte_sched_port *port, uint32_t i)
s->tc_credits_per_period[1],
s->tc_credits_per_period[2],
s->tc_credits_per_period[3],
+ queue_size_str,
/* Traffic class 3 oversubscription */
s->tc_ov_wm_min,
s->tc_ov_wm_max);
}
-int
-rte_sched_subport_config(struct rte_sched_port *port,
- uint32_t subport_id,
- struct rte_sched_subport_params *params)
+static int
+rte_sched_subport_config_common(struct rte_sched_port *port,
+ uint32_t subport_id,
+ struct rte_sched_subport_params *params,
+ uint16_t *qsize)
{
struct rte_sched_subport *s;
uint32_t i;
@@ -808,15 +902,13 @@ rte_sched_subport_config(struct rte_sched_port *port,
/* Traffic Classes (TCs) */
s->tc_period = rte_sched_time_ms_to_bytes(params->tc_period, port->rate);
+ s->tc_time = port->time + s->tc_period;
for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) {
s->tc_credits_per_period[i]
= rte_sched_time_ms_to_bytes(params->tc_period,
params->tc_rate[i]);
- }
- s->tc_time = port->time + s->tc_period;
- for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
s->tc_credits[i] = s->tc_credits_per_period[i];
-
+ }
#ifdef RTE_SCHED_SUBPORT_TC_OV
/* TC oversubscription */
s->tc_ov_wm_min = port->mtu;
@@ -829,12 +921,31 @@ rte_sched_subport_config(struct rte_sched_port *port,
s->tc_ov_rate = 0;
#endif
+ rte_sched_subport_config_qsize(port, subport_id, qsize);
rte_sched_port_log_subport_config(port, subport_id);
return 0;
}
int
+rte_sched_subport_config(struct rte_sched_port *port,
+ uint32_t subport_id,
+ struct rte_sched_subport_params *params)
+{
+ return rte_sched_subport_config_common(port, subport_id, params, NULL);
+}
+
+int
+rte_sched_subport_config_v2(struct rte_sched_port *port,
+ uint32_t subport_id,
+ struct rte_sched_subport_params *params,
+ uint16_t *qsize)
+{
+ return rte_sched_subport_config_common(port, subport_id, params,
+ qsize);
+}
+
+int
rte_sched_pipe_config(struct rte_sched_port *port,
uint32_t subport_id,
uint32_t pipe_id,
diff --git a/lib/librte_sched/rte_sched.h b/lib/librte_sched/rte_sched.h
index 5d2a688..1e1d618 100644
--- a/lib/librte_sched/rte_sched.h
+++ b/lib/librte_sched/rte_sched.h
@@ -225,6 +225,20 @@ struct rte_sched_port *
rte_sched_port_config(struct rte_sched_port_params *params);
/**
+ * Hierarchical scheduler port configuration
+ *
+ * @param params
+ * Port scheduler configuration parameter structure
+ * @param size_queue_array
+ * Pre-calculated size of the port's queue-array
+ * @return
+ * Handle to port scheduler instance upon success or NULL otherwise.
+ */
+struct rte_sched_port *
+rte_sched_port_config_v2(struct rte_sched_port_params *params,
+ uint32_t size_queue_array);
+
+/**
* Hierarchical scheduler port free
*
* @param port
@@ -251,6 +265,26 @@ rte_sched_subport_config(struct rte_sched_port *port,
struct rte_sched_subport_params *params);
/**
+ * Hierarchical scheduler subport configuration
+ *
+ * @param port
+ * Handle to port scheduler instance
+ * @param subport_id
+ * Subport ID
+ * @param params
+ * Subport configuration parameters
+ * @param qsize
+ * Array of traffic-class maximum queue-lengths
+ * @return
+ * 0 upon success, error code otherwise
+ */
+int
+rte_sched_subport_config_v2(struct rte_sched_port *port,
+ uint32_t subport_id,
+ struct rte_sched_subport_params *params,
+ uint16_t *qsize);
+
+/**
* Hierarchical scheduler pipe configuration
*
* @param port
@@ -281,6 +315,20 @@ rte_sched_pipe_config(struct rte_sched_port *port,
uint32_t
rte_sched_port_get_memory_footprint(struct rte_sched_port_params *params);
+/**
+ * Hierarchical scheduler memory footprint size per port
+ *
+ * @param params
+ * Port scheduler configuration parameter structure
+ * @param size_queue_array
+ * The required size of the port's queue-array
+ * @return
+ * Memory footprint size in bytes upon success, 0 otherwise
+ */
+uint32_t
+rte_sched_port_get_memory_footprint_v2(struct rte_sched_port_params *params,
+ uint32_t size_queue_array);
+
/*
* Statistics
*
diff --git a/lib/librte_sched/rte_sched_version.map b/lib/librte_sched/rte_sched_version.map
index 3aa159a..ce92b82 100644
--- a/lib/librte_sched/rte_sched_version.map
+++ b/lib/librte_sched/rte_sched_version.map
@@ -29,3 +29,11 @@ DPDK_2.1 {
rte_sched_port_pkt_read_color;
} DPDK_2.0;
+
+DPDK_18.05 {
+ global;
+
+ rte_sched_port_config_v2;
+ rte_sched_subport_config_v2;
+ rte_sched_port_get_memory_footprint_v2;
+} DPDK_2.1;
diff --git a/test/test/test_sched.c b/test/test/test_sched.c
index 32e500b..2b22ebe 100644
--- a/test/test/test_sched.c
+++ b/test/test/test_sched.c
@@ -15,7 +15,7 @@
#include <rte_ip.h>
#include <rte_byteorder.h>
#include <rte_sched.h>
-
+#include <rte_malloc.h>
#define SUBPORT 0
#define PIPE 1
@@ -56,7 +56,7 @@ static struct rte_sched_port_params port_param = {
.n_pipe_profiles = 1,
};
-#define NB_MBUF 32
+#define NB_MBUF 8192
#define MBUF_DATA_SZ (2048 + RTE_PKTMBUF_HEADROOM)
#define MEMPOOL_CACHE_SZ 0
#define SOCKET 0
@@ -76,7 +76,8 @@ create_mempool(void)
}
static void
-prepare_pkt(struct rte_mbuf *mbuf)
+prepare_pkt(struct rte_mbuf *mbuf, uint32_t subport, uint32_t pipe, uint32_t tc,
+ uint32_t queue)
{
struct ether_hdr *eth_hdr;
struct vlan_hdr *vlan1, *vlan2;
@@ -89,13 +90,14 @@ prepare_pkt(struct rte_mbuf *mbuf)
eth_hdr = (struct ether_hdr *)((uintptr_t)ð_hdr->ether_type + 2 *sizeof(struct vlan_hdr));
ip_hdr = (struct ipv4_hdr *)((uintptr_t)eth_hdr + sizeof(eth_hdr->ether_type));
- vlan1->vlan_tci = rte_cpu_to_be_16(SUBPORT);
- vlan2->vlan_tci = rte_cpu_to_be_16(PIPE);
+ vlan1->vlan_tci = rte_cpu_to_be_16(subport);
+ vlan2->vlan_tci = rte_cpu_to_be_16(pipe);
eth_hdr->ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv4);
- ip_hdr->dst_addr = IPv4(0,0,TC,QUEUE);
+ ip_hdr->dst_addr = IPv4(0, 0, tc, queue);
- rte_sched_port_pkt_write(mbuf, SUBPORT, PIPE, TC, QUEUE, e_RTE_METER_YELLOW);
+ rte_sched_port_pkt_write(mbuf, subport, pipe, tc, queue,
+ e_RTE_METER_YELLOW);
/* 64 byte packet */
mbuf->pkt_len = 60;
@@ -138,7 +140,7 @@ test_sched(void)
for (i = 0; i < 10; i++) {
in_mbufs[i] = rte_pktmbuf_alloc(mp);
TEST_ASSERT_NOT_NULL(in_mbufs[i], "Packet allocation failed\n");
- prepare_pkt(in_mbufs[i]);
+ prepare_pkt(in_mbufs[i], SUBPORT, PIPE, TC, QUEUE);
}
@@ -185,3 +187,245 @@ test_sched(void)
}
REGISTER_TEST_COMMAND(sched_autotest, test_sched);
+
+#define NB_SUBPORTS 2
+
+static struct rte_sched_subport_params subport_param_v2[] = {
+ {
+ .tb_rate = 1250000000,
+ .tb_size = 1000000,
+
+ .tc_rate = {1250000000, 1250000000, 1250000000, 1250000000},
+ .tc_period = 10,
+ },
+ {
+ .tb_rate = 1250000000,
+ .tb_size = 1000000,
+
+ .tc_rate = {1250000000, 1250000000, 1250000000, 1250000000},
+ .tc_period = 10,
+ },
+};
+
+static struct rte_sched_pipe_params pipe_profile_v2[] = {
+ { /* Profile #0 */
+ .tb_rate = 1250000000,
+ .tb_size = 1000000,
+
+ .tc_rate = {1250000000, 1250000000, 1250000000, 1250000000},
+ .tc_period = 10,
+
+ .wrr_weights = {1, 1, 1, 1,
+ 1, 1, 1, 1,
+ 1, 1, 1, 1,
+ 1, 1, 1, 1},
+ },
+};
+
+static uint16_t subport_qsize[][RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE] = {
+ { 16, 32, 64, 128 },
+ { 256, 512, 1024, 2048 },
+};
+
+static struct rte_sched_port_params port_param_v2 = {
+ .socket = 0, /* computed */
+ .rate = 0, /* computed */
+ .mtu = 1522,
+ .frame_overhead = RTE_SCHED_FRAME_OVERHEAD_DEFAULT,
+ .n_subports_per_port = 2,
+ .n_pipes_per_subport = 128,
+ .qsize = {32, 32, 32, 32},
+ .pipe_profiles = pipe_profile_v2,
+ .n_pipe_profiles = 1,
+};
+
+static uint32_t subport_total_qsize(struct rte_sched_port_params *pp,
+ uint16_t *qsize)
+{
+ uint32_t queue_array_size = 0;
+ uint32_t tc;
+
+ for (tc = 0; tc < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; tc++)
+ queue_array_size += qsize[tc];
+
+ return (queue_array_size * RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS *
+ pp->n_pipes_per_subport * sizeof(struct rte_mbuf *));
+}
+
+static int fill_queue_to_drop(struct rte_mempool *mp,
+ struct rte_sched_port *port,
+ uint32_t in_subport, uint32_t in_pipe,
+ uint32_t in_tc, uint16_t qsize)
+{
+ struct rte_mbuf **in_mbufs;
+ struct rte_mbuf **out_mbufs;
+ uint32_t in_queue = 0;
+ uint32_t i;
+ int err;
+
+ in_mbufs = rte_malloc(NULL, ((qsize + 1) * sizeof(struct rte_mbuf *)),
+ RTE_CACHE_LINE_SIZE);
+ TEST_ASSERT_NOT_NULL(in_mbufs, "Buffer array allocation failed\n");
+
+ out_mbufs = rte_malloc(NULL, ((qsize + 1) * sizeof(struct rte_mbuf *)),
+ RTE_CACHE_LINE_SIZE);
+ TEST_ASSERT_NOT_NULL(out_mbufs, "Buffer array allocation failed\n");
+
+ /*
+ * Allocate qsize + 1 buffers so that we can completely fill the
+ * queue, then try to enqueue one more packet so that it will be tail
+ * dropped.
+ */
+ for (i = 0; i <= qsize; i++) {
+ in_mbufs[i] = rte_pktmbuf_alloc(mp);
+ TEST_ASSERT_NOT_NULL(in_mbufs[i], "Packet allocation failed\n");
+ prepare_pkt(in_mbufs[i], in_subport, in_pipe, in_tc, in_queue);
+ }
+
+ /*
+ * All these packets should be queued correctly.
+ */
+ err = rte_sched_port_enqueue(port, in_mbufs, qsize);
+ TEST_ASSERT_EQUAL(err, qsize, "Wrong enqueue, err=%d\n", err);
+
+ /*
+ * This packet should fail to be queued, it will be freed when dropped.
+ */
+ err = rte_sched_port_enqueue(port, &in_mbufs[qsize], 1);
+ TEST_ASSERT_EQUAL(err, 0, "Enqueue didn't fail, but should have\n");
+ in_mbufs[qsize] = NULL;
+
+ /*
+ * With small queues we should be able to dequeue a full queue's worth
+ * of packets with a single call to rte_sched_port_dequeue. With
+ * larger queues we will probably need to make multiple calls as we
+ * could run out of credit to dequeue all the packet in one attempt.
+ */
+ i = 0;
+ err = 1;
+ while (i < qsize && err != 0) {
+ err = rte_sched_port_dequeue(port, out_mbufs, qsize);
+ i += err;
+ }
+ TEST_ASSERT_EQUAL(i, qsize,
+ "Wrong dequeue, err=%d, i: %u, qsize: %u\n",
+ err, i, qsize);
+
+ /*
+ * Check that all the dequeued packets have to right numbers in them.
+ */
+ for (i = 0; i < qsize; i++) {
+ enum rte_meter_color color;
+ uint32_t out_subport, out_pipe, out_tc, out_queue;
+
+ color = rte_sched_port_pkt_read_color(out_mbufs[i]);
+ TEST_ASSERT_EQUAL(color, e_RTE_METER_YELLOW, "Wrong color\n");
+
+ rte_sched_port_pkt_read_tree_path(out_mbufs[i],
+ &out_subport, &out_pipe, &out_tc, &out_queue);
+
+ TEST_ASSERT_EQUAL(in_subport, out_subport, "Wrong subport\n");
+ TEST_ASSERT_EQUAL(in_pipe, out_pipe, "Wrong pipe\n");
+ TEST_ASSERT_EQUAL(in_tc, out_tc, "Wrong traffic_class\n");
+ TEST_ASSERT_EQUAL(in_queue, out_queue, "Wrong queue\n");
+ rte_pktmbuf_free(out_mbufs[i]);
+ }
+
+#ifdef RTE_SCHED_COLLECT_STATS
+ struct rte_sched_subport_stats subport_stats;
+ uint32_t tc_ov;
+
+ /*
+ * Did the subport stats see a packet dropped in this traffic-class?
+ */
+ rte_sched_subport_read_stats(port, in_subport, &subport_stats, &tc_ov);
+ TEST_ASSERT_EQUAL(subport_stats.n_pkts_tc_dropped[in_tc], 1,
+ "Wrong subport stats\n");
+#endif
+
+ rte_free(in_mbufs);
+ rte_free(out_mbufs);
+
+ return 0;
+}
+
+static int
+subport_fill_queues(struct rte_mempool *mp, struct rte_sched_port *port,
+ uint32_t subport)
+{
+ uint32_t pipe;
+ uint32_t tc;
+ int err;
+
+ for (pipe = 0; pipe < port_param_v2.n_pipes_per_subport; pipe++) {
+ for (tc = 0; tc < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; tc++) {
+ err = fill_queue_to_drop(mp, port, subport, pipe, tc,
+ subport_qsize[subport][tc]);
+ TEST_ASSERT_SUCCESS(err, "fill-queue-to-drop failed, "
+ "err=%d\n", err);
+ }
+ }
+ return 0;
+}
+
+/**
+ * test main entrance for library sched using the v2 APIs that
+ * allow queue-size and WRED configurations on a per-subport basis.
+ */
+static int
+test_sched_v2(void)
+{
+ struct rte_mempool *mp = NULL;
+ struct rte_sched_port *port = NULL;
+ uint32_t subport;
+ uint32_t pipe;
+ uint32_t queue_array_size;
+ int err;
+
+ rte_log_set_level(RTE_LOGTYPE_EAL, RTE_LOG_DEBUG);
+
+ mp = create_mempool();
+ TEST_ASSERT_NOT_NULL(mp, "Error creating mempool\n");
+
+ port_param_v2.socket = 0;
+ port_param_v2.rate = (uint64_t) 10000 * 1000 * 1000 / 8;
+
+ queue_array_size = 0;
+ for (subport = 0; subport < NB_SUBPORTS; subport++)
+ queue_array_size +=
+ subport_total_qsize(&port_param_v2,
+ &subport_qsize[subport][0]);
+
+ port = rte_sched_port_config_v2(&port_param_v2,
+ queue_array_size);
+ TEST_ASSERT_NOT_NULL(port, "Error config sched port\n");
+
+ for (subport = 0; subport < NB_SUBPORTS; subport++) {
+ err = rte_sched_subport_config_v2(port, subport,
+ &subport_param_v2[subport],
+ &subport_qsize[subport][0]);
+ TEST_ASSERT_SUCCESS(err,
+ "Error config sched subport %u, err=%d\n",
+ subport, err);
+ for (pipe = 0; pipe < port_param_v2.n_pipes_per_subport;
+ pipe++) {
+ err = rte_sched_pipe_config(port, subport, pipe, 0);
+ TEST_ASSERT_SUCCESS(err,
+ "Error config sched subport %u "
+ "pipe %u, err=%d\n",
+ subport, pipe, err);
+ }
+ }
+
+ for (subport = 0; subport < NB_SUBPORTS; subport++) {
+ err = subport_fill_queues(mp, port, subport);
+ TEST_ASSERT_SUCCESS(err, "subport-fill-queue failed, err=%d\n",
+ err);
+ }
+
+ rte_sched_port_free(port);
+
+ return 0;
+}
+
+REGISTER_TEST_COMMAND(sched_autotest_v2, test_sched_v2);
--
2.7.4
^ permalink raw reply [flat|nested] 2+ messages in thread
* [dpdk-dev] [RFC 2/2] sched: support per-subport wred configurations
2018-02-28 13:39 [dpdk-dev] [RFC 1/2] sched: new APIs for per-subport queue sizes alangordondewar
@ 2018-02-28 13:39 ` alangordondewar
0 siblings, 0 replies; 2+ messages in thread
From: alangordondewar @ 2018-02-28 13:39 UTC (permalink / raw)
To: cristian.dumitrescu; +Cc: dev, Alan Dewar
From: Alan Dewar <alan.dewar@att.com>
Move the WRED queue configuration parameters from rte_sched_port_params
into rte_sched_subport_params so that we can have different WRED
configuations on each subport.
Updated sched unit-test to exercise new functionality.
Signed-off-by: Alan Dewar <alan.dewar@att.com>
---
lib/librte_sched/rte_sched.c | 54 +++++-
lib/librte_sched/rte_sched.h | 6 +-
test/test/test_sched.c | 402 +++++++++++++++++++++++++++++++++----------
3 files changed, 363 insertions(+), 99 deletions(-)
diff --git a/lib/librte_sched/rte_sched.c b/lib/librte_sched/rte_sched.c
index 9436ba5..087d7fc 100644
--- a/lib/librte_sched/rte_sched.c
+++ b/lib/librte_sched/rte_sched.c
@@ -77,6 +77,11 @@ struct rte_sched_subport {
uint32_t qsize_add[RTE_SCHED_QUEUES_PER_PIPE];
uint32_t qsize_sum;
uint32_t qoffset;
+
+#ifdef RTE_SCHED_RED
+ struct rte_red_config red_config[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE]
+ [e_RTE_METER_COLORS];
+#endif
};
struct rte_sched_pipe_profile {
@@ -857,7 +862,9 @@ static int
rte_sched_subport_config_common(struct rte_sched_port *port,
uint32_t subport_id,
struct rte_sched_subport_params *params,
- uint16_t *qsize)
+ uint16_t *qsize,
+ struct rte_red_params red_params[]
+ [e_RTE_METER_COLORS])
{
struct rte_sched_subport *s;
uint32_t i;
@@ -909,6 +916,38 @@ rte_sched_subport_config_common(struct rte_sched_port *port,
params->tc_rate[i]);
s->tc_credits[i] = s->tc_credits_per_period[i];
}
+
+#ifdef RTE_SCHED_RED
+ for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) {
+ uint32_t j;
+
+ if (!red_params) {
+ /* Copy the red configuration from port */
+ for (j = 0; j < e_RTE_METER_COLORS; j++)
+ s->red_config[i][j] = port->red_config[i][j];
+ } else {
+ /* Subport has an individual red configuration */
+ for (j = 0; j < e_RTE_METER_COLORS; j++) {
+ /* if min/max are both zero, then RED is
+ * disabled
+ */
+ if ((red_params[i][j].min_th |
+ red_params[i][j].max_th) == 0) {
+ continue;
+ }
+
+ if (rte_red_config_init(&s->red_config[i][j],
+ red_params[i][j].wq_log2,
+ red_params[i][j].min_th,
+ red_params[i][j].max_th,
+ red_params[i][j].maxp_inv) != 0) {
+ return -6;
+ }
+ }
+ }
+ }
+#endif
+
#ifdef RTE_SCHED_SUBPORT_TC_OV
/* TC oversubscription */
s->tc_ov_wm_min = port->mtu;
@@ -932,17 +971,20 @@ rte_sched_subport_config(struct rte_sched_port *port,
uint32_t subport_id,
struct rte_sched_subport_params *params)
{
- return rte_sched_subport_config_common(port, subport_id, params, NULL);
+ return rte_sched_subport_config_common(port, subport_id, params, NULL,
+ NULL);
}
int
rte_sched_subport_config_v2(struct rte_sched_port *port,
uint32_t subport_id,
struct rte_sched_subport_params *params,
- uint16_t *qsize)
+ uint16_t *qsize,
+ struct rte_red_params red_params[]
+ [e_RTE_METER_COLORS])
{
return rte_sched_subport_config_common(port, subport_id, params,
- qsize);
+ qsize, red_params);
}
int
@@ -1236,6 +1278,8 @@ rte_sched_port_update_queue_stats_on_drop(struct rte_sched_port *port,
static inline int
rte_sched_port_red_drop(struct rte_sched_port *port, struct rte_mbuf *pkt, uint32_t qindex, uint16_t qlen)
{
+ struct rte_sched_subport *subport = port->subport +
+ (qindex / rte_sched_port_queues_per_subport(port));
struct rte_sched_queue_extra *qe;
struct rte_red_config *red_cfg;
struct rte_red *red;
@@ -1244,7 +1288,7 @@ rte_sched_port_red_drop(struct rte_sched_port *port, struct rte_mbuf *pkt, uint3
tc_index = (qindex >> 2) & 0x3;
color = rte_sched_port_pkt_read_color(pkt);
- red_cfg = &port->red_config[tc_index][color];
+ red_cfg = &subport->red_config[tc_index][color];
if ((red_cfg->min_th | red_cfg->max_th) == 0)
return 0;
diff --git a/lib/librte_sched/rte_sched.h b/lib/librte_sched/rte_sched.h
index 1e1d618..fe41ae4 100644
--- a/lib/librte_sched/rte_sched.h
+++ b/lib/librte_sched/rte_sched.h
@@ -275,6 +275,8 @@ rte_sched_subport_config(struct rte_sched_port *port,
* Subport configuration parameters
* @param qsize
* Array of traffic-class maximum queue-lengths
+ * @param red_params
+ * Subport WRED queue configuration parameters
* @return
* 0 upon success, error code otherwise
*/
@@ -282,7 +284,9 @@ int
rte_sched_subport_config_v2(struct rte_sched_port *port,
uint32_t subport_id,
struct rte_sched_subport_params *params,
- uint16_t *qsize);
+ uint16_t *qsize,
+ struct rte_red_params red_params[]
+ [e_RTE_METER_COLORS]);
/**
* Hierarchical scheduler pipe configuration
diff --git a/test/test/test_sched.c b/test/test/test_sched.c
index 2b22ebe..bc25d34 100644
--- a/test/test/test_sched.c
+++ b/test/test/test_sched.c
@@ -104,6 +104,32 @@ prepare_pkt(struct rte_mbuf *mbuf, uint32_t subport, uint32_t pipe, uint32_t tc,
mbuf->data_len = 60;
}
+static int
+pkt_check(struct rte_mbuf **mbufs, uint32_t nb_pkts, uint32_t in_subport,
+ uint32_t in_pipe, uint32_t in_tc, uint32_t in_queue)
+{
+ uint32_t i;
+
+ for (i = 0; i < nb_pkts; i++) {
+ enum rte_meter_color color;
+ uint32_t out_subport, out_pipe, out_tc, out_queue;
+
+ color = rte_sched_port_pkt_read_color(mbufs[i]);
+ TEST_ASSERT_EQUAL(color, e_RTE_METER_YELLOW, "Wrong color\n");
+
+ rte_sched_port_pkt_read_tree_path(mbufs[i], &out_subport,
+ &out_pipe, &out_tc,
+ &out_queue);
+
+ TEST_ASSERT_EQUAL(in_subport, out_subport, "Wrong subport\n");
+ TEST_ASSERT_EQUAL(in_pipe, out_pipe, "Wrong pipe\n");
+ TEST_ASSERT_EQUAL(in_tc, out_tc, "Wrong traffic_class\n");
+ TEST_ASSERT_EQUAL(in_queue, out_queue, "Wrong queue\n");
+ rte_pktmbuf_free(mbufs[i]);
+ }
+
+ return 0;
+}
/**
* test main entrance for library sched
@@ -143,41 +169,33 @@ test_sched(void)
prepare_pkt(in_mbufs[i], SUBPORT, PIPE, TC, QUEUE);
}
-
err = rte_sched_port_enqueue(port, in_mbufs, 10);
TEST_ASSERT_EQUAL(err, 10, "Wrong enqueue, err=%d\n", err);
err = rte_sched_port_dequeue(port, out_mbufs, 10);
TEST_ASSERT_EQUAL(err, 10, "Wrong dequeue, err=%d\n", err);
- for (i = 0; i < 10; i++) {
- enum rte_meter_color color;
- uint32_t subport, traffic_class, queue;
-
- color = rte_sched_port_pkt_read_color(out_mbufs[i]);
- TEST_ASSERT_EQUAL(color, e_RTE_METER_YELLOW, "Wrong color\n");
-
- rte_sched_port_pkt_read_tree_path(out_mbufs[i],
- &subport, &pipe, &traffic_class, &queue);
-
- TEST_ASSERT_EQUAL(subport, SUBPORT, "Wrong subport\n");
- TEST_ASSERT_EQUAL(pipe, PIPE, "Wrong pipe\n");
- TEST_ASSERT_EQUAL(traffic_class, TC, "Wrong traffic_class\n");
- TEST_ASSERT_EQUAL(queue, QUEUE, "Wrong queue\n");
-
- }
-
+ err = pkt_check(out_mbufs, err, SUBPORT, PIPE, TC, QUEUE);
+ TEST_ASSERT_SUCCESS(err, "Packet checking failed\n");
struct rte_sched_subport_stats subport_stats;
uint32_t tc_ov;
rte_sched_subport_read_stats(port, SUBPORT, &subport_stats, &tc_ov);
-#if 0
- TEST_ASSERT_EQUAL(subport_stats.n_pkts_tc[TC-1], 10, "Wrong subport stats\n");
+#ifdef RTE_SCHED_COLLECT_STATS
+ TEST_ASSERT_EQUAL(subport_stats.n_pkts_tc[TC], 10,
+ "Wrong subport stats\n");
#endif
struct rte_sched_queue_stats queue_stats;
uint16_t qlen;
rte_sched_queue_read_stats(port, QUEUE, &queue_stats, &qlen);
#if 0
+ /*
+ * This assert fails because the wrong queue_id is passed into
+ * rte_sched_queue_read_stats. To calculate the correct queue_id
+ * we really need to call rte_sched_port_qindex passing in port,
+ * subport, pipe, traffic-class and queue-number. Unfortunately
+ * rte_sched_port_qindex is a static function.
+ */
TEST_ASSERT_EQUAL(queue_stats.n_pkts, 10, "Wrong queue stats\n");
#endif
@@ -222,7 +240,7 @@ static struct rte_sched_pipe_params pipe_profile_v2[] = {
},
};
-static uint16_t subport_qsize[][RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE] = {
+static uint16_t config_subport_qsize[][RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE] = {
{ 16, 32, 64, 128 },
{ 256, 512, 1024, 2048 },
};
@@ -239,6 +257,64 @@ static struct rte_sched_port_params port_param_v2 = {
.n_pipe_profiles = 1,
};
+/*
+ * Note that currently all the packets are coloured yellow.
+ */
+struct rte_red_params subport_0_redparams[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE]
+ [e_RTE_METER_COLORS] = {
+ { /* TC-0 queue-size 16 */
+ /* min_th max_th maxp_inv wq_log2 */
+ { 0, 0, 0, 0 }, /* Green */
+ { 1, 15, 255, 12 }, /* Yellow */
+ { 1, 15, 1, 1 } /* Red */
+ },
+ { /* TC-1 queue-size 32 */
+ { 24, 31, 1, 1 }, /* Green */
+ { 16, 31, 10, 1 }, /* Yellow */
+ { 8, 31, 100, 1 } /* Red */
+ },
+ { /* TC-2 queue-size 64 */
+ { 32, 63, 1, 1 }, /* Green */
+ { 16, 31, 1, 1 }, /* Yellow */
+ { 8, 15, 1, 1 } /* Red */
+ },
+ { /* TC-3 queue-size 128 */
+ { 64, 127, 255, 12 }, /* Green */
+ { 32, 63, 255, 12 }, /* Yellow */
+ { 16, 31, 255, 12 } /* Red */
+ }
+};
+
+struct rte_red_params subport_1_redparams[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE]
+ [e_RTE_METER_COLORS] = {
+ { /* TC-0 queue-size 256 */
+ /* min_th max_th maxp_inv wq_log2 */
+ { 0, 0, 0, 0 }, /* Green */
+ { 128, 255, 100, 1 }, /* Yellow */
+ { 128, 255, 2, 10 } /* Red */
+ },
+ { /* TC-1 queue-size 512 */
+ { 256, 511, 2, 1 }, /* Green */
+ { 128, 511, 20, 1 }, /* Yellow */
+ { 64, 511, 200, 1 } /* Red */
+ },
+ { /* TC-2 queue-size 1024 */
+ { 512, 1023, 6, 4 }, /* Green */
+ { 256, 1023, 6, 4 }, /* Yellow */
+ { 128, 1023, 6, 4 } /* Red */
+ },
+ { /* TC-3 queue-size 2048 - RTE_RED_MAX_TH_MAX = 1023 */
+ { 1022, 1023, 128, 9 }, /* Green */
+ { 512, 1023, 64, 6 }, /* Yellow */
+ { 256, 1023, 32, 3 } /* Red */
+ }
+};
+
+struct rte_red_params *config_subport_redparams[] = {
+ &subport_0_redparams[0][0],
+ &subport_1_redparams[0][0]
+};
+
static uint32_t subport_total_qsize(struct rte_sched_port_params *pp,
uint16_t *qsize)
{
@@ -252,14 +328,81 @@ static uint32_t subport_total_qsize(struct rte_sched_port_params *pp,
pp->n_pipes_per_subport * sizeof(struct rte_mbuf *));
}
-static int fill_queue_to_drop(struct rte_mempool *mp,
- struct rte_sched_port *port,
- uint32_t in_subport, uint32_t in_pipe,
- uint32_t in_tc, uint16_t qsize)
+static int
+test_dequeue_pkts(struct rte_sched_port *port, struct rte_mbuf **mbufs,
+ uint16_t nb_pkts)
+{
+ uint16_t total_dequeued;
+ int err;
+
+ total_dequeued = 0;
+ err = 1;
+
+ /*
+ * With small queues we should be able to dequeue a full queue's worth
+ * of packets with a single call to rte_sched_port_dequeue. With
+ * larger queues we will probably need to make multiple calls as we
+ * could run out of credit to dequeue all the packet in one attempt.
+ */
+ while (total_dequeued < nb_pkts && err != 0) {
+ err = rte_sched_port_dequeue(port, mbufs, nb_pkts);
+ total_dequeued += err;
+ }
+ return total_dequeued;
+}
+
+static int
+test_sched_v2_setup(uint16_t qsize[][RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE],
+ struct rte_red_params **subport_redparams,
+ struct rte_sched_port **port)
+{
+ uint32_t queue_array_size;
+ uint32_t subport;
+ uint32_t pipe;
+ int err;
+
+ queue_array_size = 0;
+ for (subport = 0; subport < NB_SUBPORTS; subport++)
+ queue_array_size +=
+ subport_total_qsize(&port_param_v2, &qsize[subport][0]);
+
+ *port = rte_sched_port_config_v2(&port_param_v2,
+ queue_array_size);
+ TEST_ASSERT_NOT_NULL(port, "Error config sched port\n");
+
+ for (subport = 0; subport < NB_SUBPORTS; subport++) {
+ void *redparams = NULL;
+
+ if (subport_redparams)
+ redparams = subport_redparams[subport];
+
+ err = rte_sched_subport_config_v2(*port, subport,
+ &subport_param_v2[subport],
+ &qsize[subport][0],
+ redparams);
+ TEST_ASSERT_SUCCESS(err,
+ "Error config sched subport %u, err=%d\n",
+ subport, err);
+ for (pipe = 0; pipe < port_param_v2.n_pipes_per_subport;
+ pipe++) {
+ err = rte_sched_pipe_config(*port, subport, pipe, 0);
+ TEST_ASSERT_SUCCESS(err,
+ "Error config sched subport %u "
+ "pipe %u, err=%d\n",
+ subport, pipe, err);
+ }
+ }
+ return 0;
+}
+
+static int
+test_queue_size_drop(struct rte_mempool *mp, struct rte_sched_port *port,
+ uint32_t subport, uint32_t pipe, uint32_t tc,
+ uint16_t qsize)
{
struct rte_mbuf **in_mbufs;
struct rte_mbuf **out_mbufs;
- uint32_t in_queue = 0;
+ uint32_t queue = 0;
uint32_t i;
int err;
@@ -279,7 +422,7 @@ static int fill_queue_to_drop(struct rte_mempool *mp,
for (i = 0; i <= qsize; i++) {
in_mbufs[i] = rte_pktmbuf_alloc(mp);
TEST_ASSERT_NOT_NULL(in_mbufs[i], "Packet allocation failed\n");
- prepare_pkt(in_mbufs[i], in_subport, in_pipe, in_tc, in_queue);
+ prepare_pkt(in_mbufs[i], subport, pipe, tc, queue);
}
/*
@@ -296,53 +439,117 @@ static int fill_queue_to_drop(struct rte_mempool *mp,
in_mbufs[qsize] = NULL;
/*
- * With small queues we should be able to dequeue a full queue's worth
- * of packets with a single call to rte_sched_port_dequeue. With
- * larger queues we will probably need to make multiple calls as we
- * could run out of credit to dequeue all the packet in one attempt.
+ * Dequeue all the packets off the queue.
*/
- i = 0;
- err = 1;
- while (i < qsize && err != 0) {
- err = rte_sched_port_dequeue(port, out_mbufs, qsize);
- i += err;
- }
- TEST_ASSERT_EQUAL(i, qsize,
- "Wrong dequeue, err=%d, i: %u, qsize: %u\n",
- err, i, qsize);
+ i = test_dequeue_pkts(port, out_mbufs, qsize);
+ TEST_ASSERT_EQUAL(i, qsize, "Failed to dequeue all pkts\n");
/*
- * Check that all the dequeued packets have to right numbers in them.
+ * Check that all the dequeued packets have the right numbers in them.
*/
- for (i = 0; i < qsize; i++) {
- enum rte_meter_color color;
- uint32_t out_subport, out_pipe, out_tc, out_queue;
+ err = pkt_check(out_mbufs, qsize, subport, pipe, tc, queue);
+ TEST_ASSERT_SUCCESS(err, "Packet checking failed\n");
- color = rte_sched_port_pkt_read_color(out_mbufs[i]);
- TEST_ASSERT_EQUAL(color, e_RTE_METER_YELLOW, "Wrong color\n");
+#ifdef RTE_SCHED_COLLECT_STATS
+ struct rte_sched_subport_stats subport_stats;
+ uint32_t tc_ov;
- rte_sched_port_pkt_read_tree_path(out_mbufs[i],
- &out_subport, &out_pipe, &out_tc, &out_queue);
+ /*
+ * Did the subport stats see a packet dropped in this traffic-class?
+ */
+ rte_sched_subport_read_stats(port, subport, &subport_stats, &tc_ov);
+ TEST_ASSERT_EQUAL(subport_stats.n_pkts_tc_dropped[tc], 1,
+ "Wrong subport stats\n");
+#endif
- TEST_ASSERT_EQUAL(in_subport, out_subport, "Wrong subport\n");
- TEST_ASSERT_EQUAL(in_pipe, out_pipe, "Wrong pipe\n");
- TEST_ASSERT_EQUAL(in_tc, out_tc, "Wrong traffic_class\n");
- TEST_ASSERT_EQUAL(in_queue, out_queue, "Wrong queue\n");
- rte_pktmbuf_free(out_mbufs[i]);
+ rte_free(in_mbufs);
+ rte_free(out_mbufs);
+
+ return 0;
+}
+
+static int
+test_queue_size(struct rte_mempool *mp, struct rte_sched_port *port,
+ uint32_t subport)
+{
+ uint32_t pipe;
+ uint32_t tc;
+ int err;
+
+ for (pipe = 0; pipe < port_param_v2.n_pipes_per_subport; pipe++) {
+ for (tc = 0; tc < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; tc++) {
+ err = test_queue_size_drop(mp, port, subport, pipe, tc,
+ config_subport_qsize[subport][tc]);
+ TEST_ASSERT_SUCCESS(err, "test_queue_size_drop "
+ "failed\n");
+ }
+ }
+ return 0;
+}
+
+static int test_red(struct rte_mempool *mp, struct rte_sched_port *port,
+ uint32_t subport, uint32_t pipe, uint32_t tc,
+ uint16_t qsize)
+{
+ struct rte_mbuf **in_mbufs;
+ struct rte_mbuf **out_mbufs;
+ uint32_t queue = 0;
+ uint32_t i;
+ int err;
+ int queued;
+
+ in_mbufs = rte_malloc(NULL, ((qsize + 1) * sizeof(struct rte_mbuf *)),
+ RTE_CACHE_LINE_SIZE);
+ TEST_ASSERT_NOT_NULL(in_mbufs, "Buffer array allocation failed\n");
+
+ out_mbufs = rte_malloc(NULL, ((qsize + 1) * sizeof(struct rte_mbuf *)),
+ RTE_CACHE_LINE_SIZE);
+ TEST_ASSERT_NOT_NULL(out_mbufs, "Buffer array allocation failed\n");
+
+ /*
+ * Allocate qsize buffers so that we can attemp to completely fill the
+ * queue, then check the subport stats to see if any packets were
+ * red-dropped.
+ */
+ for (i = 0; i < qsize; i++) {
+ in_mbufs[i] = rte_pktmbuf_alloc(mp);
+ TEST_ASSERT_NOT_NULL(in_mbufs[i], "Packet allocation failed\n");
+ prepare_pkt(in_mbufs[i], subport, pipe, tc, queue);
}
+ /*
+ * Some of these packets might not get queued correctly due to
+ * red-drops.
+ */
+ queued = rte_sched_port_enqueue(port, in_mbufs, qsize);
+
#ifdef RTE_SCHED_COLLECT_STATS
struct rte_sched_subport_stats subport_stats;
uint32_t tc_ov;
+ uint32_t red_drops;
/*
- * Did the subport stats see a packet dropped in this traffic-class?
+ * Did the subport stats see any packets red-dropped in this
+ * traffic-class?
*/
- rte_sched_subport_read_stats(port, in_subport, &subport_stats, &tc_ov);
- TEST_ASSERT_EQUAL(subport_stats.n_pkts_tc_dropped[in_tc], 1,
- "Wrong subport stats\n");
+ rte_sched_subport_read_stats(port, subport, &subport_stats, &tc_ov);
+ red_drops = subport_stats.n_pkts_red_dropped[tc];
+ TEST_ASSERT_EQUAL((qsize - red_drops), (uint32_t)queued,
+ "Red-drop count doesn't agree queued count\n");
#endif
+ /*
+ * Dequeue all the packets off the queue.
+ */
+ i = test_dequeue_pkts(port, out_mbufs, queued);
+ TEST_ASSERT_EQUAL(i, (uint32_t)queued, "Failed to dequeue all pkts\n");
+
+ /*
+ * Check that all the dequeued packets have the right numbers in them.
+ */
+ err = pkt_check(out_mbufs, queued, subport, pipe, tc, queue);
+ TEST_ASSERT_SUCCESS(err, "Packet checking failed\n");
+
rte_free(in_mbufs);
rte_free(out_mbufs);
@@ -350,8 +557,8 @@ static int fill_queue_to_drop(struct rte_mempool *mp,
}
static int
-subport_fill_queues(struct rte_mempool *mp, struct rte_sched_port *port,
- uint32_t subport)
+test_red_queues(struct rte_mempool *mp, struct rte_sched_port *port,
+ uint32_t subport)
{
uint32_t pipe;
uint32_t tc;
@@ -359,27 +566,24 @@ subport_fill_queues(struct rte_mempool *mp, struct rte_sched_port *port,
for (pipe = 0; pipe < port_param_v2.n_pipes_per_subport; pipe++) {
for (tc = 0; tc < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; tc++) {
- err = fill_queue_to_drop(mp, port, subport, pipe, tc,
- subport_qsize[subport][tc]);
- TEST_ASSERT_SUCCESS(err, "fill-queue-to-drop failed, "
- "err=%d\n", err);
+ err = test_red(mp, port, subport, pipe, tc,
+ config_subport_qsize[subport][tc]);
+ TEST_ASSERT_SUCCESS(err, "test_red failed\n");
}
}
return 0;
}
/**
- * test main entrance for library sched using the v2 APIs that
- * allow queue-size and WRED configurations on a per-subport basis.
+ * test main entrance for library sched using the v2 APIs that allow
+ * queue-size on a per-subport basis.
*/
static int
-test_sched_v2(void)
+test_sched_v2_qsize(void)
{
struct rte_mempool *mp = NULL;
struct rte_sched_port *port = NULL;
uint32_t subport;
- uint32_t pipe;
- uint32_t queue_array_size;
int err;
rte_log_set_level(RTE_LOGTYPE_EAL, RTE_LOG_DEBUG);
@@ -390,35 +594,45 @@ test_sched_v2(void)
port_param_v2.socket = 0;
port_param_v2.rate = (uint64_t) 10000 * 1000 * 1000 / 8;
- queue_array_size = 0;
- for (subport = 0; subport < NB_SUBPORTS; subport++)
- queue_array_size +=
- subport_total_qsize(&port_param_v2,
- &subport_qsize[subport][0]);
-
- port = rte_sched_port_config_v2(&port_param_v2,
- queue_array_size);
- TEST_ASSERT_NOT_NULL(port, "Error config sched port\n");
+ err = test_sched_v2_setup(config_subport_qsize, NULL, &port);
+ TEST_ASSERT_SUCCESS(err, "test_sched_v2_setup failed\n");
for (subport = 0; subport < NB_SUBPORTS; subport++) {
- err = rte_sched_subport_config_v2(port, subport,
- &subport_param_v2[subport],
- &subport_qsize[subport][0]);
- TEST_ASSERT_SUCCESS(err,
- "Error config sched subport %u, err=%d\n",
- subport, err);
- for (pipe = 0; pipe < port_param_v2.n_pipes_per_subport;
- pipe++) {
- err = rte_sched_pipe_config(port, subport, pipe, 0);
- TEST_ASSERT_SUCCESS(err,
- "Error config sched subport %u "
- "pipe %u, err=%d\n",
- subport, pipe, err);
- }
+ err = test_queue_size(mp, port, subport);
+ TEST_ASSERT_SUCCESS(err, "test_queue_size failed\n");
}
+ rte_sched_port_free(port);
+
+ return 0;
+}
+
+/**
+ * test main entrance for library sched using the v2 APIs that allow WRED
+ * configurations on a per-subport basis.
+ */
+static int
+test_sched_v2_red(void)
+{
+ struct rte_mempool *mp = NULL;
+ struct rte_sched_port *port = NULL;
+ uint32_t subport;
+ int err;
+
+ rte_log_set_level(RTE_LOGTYPE_EAL, RTE_LOG_DEBUG);
+
+ mp = create_mempool();
+ TEST_ASSERT_NOT_NULL(mp, "Error creating mempool\n");
+
+ port_param_v2.socket = 0;
+ port_param_v2.rate = (uint64_t) 10000 * 1000 * 1000 / 8;
+
+ err = test_sched_v2_setup(config_subport_qsize,
+ config_subport_redparams, &port);
+ TEST_ASSERT_SUCCESS(err, "Test setup failed\n");
+
for (subport = 0; subport < NB_SUBPORTS; subport++) {
- err = subport_fill_queues(mp, port, subport);
+ err = test_red_queues(mp, port, subport);
TEST_ASSERT_SUCCESS(err, "subport-fill-queue failed, err=%d\n",
err);
}
@@ -428,4 +642,6 @@ test_sched_v2(void)
return 0;
}
-REGISTER_TEST_COMMAND(sched_autotest_v2, test_sched_v2);
+
+REGISTER_TEST_COMMAND(sched_autotest_v2_qsize, test_sched_v2_qsize);
+REGISTER_TEST_COMMAND(sched_autotest_v2_red, test_sched_v2_red);
--
2.7.4
^ permalink raw reply [flat|nested] 2+ messages in thread
end of thread, other threads:[~2018-02-28 13:40 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2018-02-28 13:39 [dpdk-dev] [RFC 1/2] sched: new APIs for per-subport queue sizes alangordondewar
2018-02-28 13:39 ` [dpdk-dev] [RFC 2/2] sched: support per-subport wred configurations alangordondewar
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).