* [PATCH] event/dlb2: add ldb port specific COS support
@ 2022-04-10 22:56 Timothy McDaniel
2022-06-14 10:42 ` Jerin Jacob
0 siblings, 1 reply; 3+ messages in thread
From: Timothy McDaniel @ 2022-04-10 22:56 UTC (permalink / raw)
To: jerinj; +Cc: dev
DLB supports 4 class of service domains, to aid in managing the
device bandwidth across ldb ports. This commit allows specifying
which ldb ports will participate in the COS scheme, which class
they are a part of, and the specific bandwidth percentage
associated with each class. The cumulative bandwidth associated
with the 4 classes must not exceed 100%. This feature is enabled
on the command line, and will be documented in the DLB2 programmers
guide.
Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
Depends-on: patch-109560 ("[v2] event/dlb2: add CQ weight support")
---
drivers/event/dlb2/dlb2.c | 249 ++++++++++++++-------
drivers/event/dlb2/dlb2_iface.c | 3 +
drivers/event/dlb2/dlb2_iface.h | 3 +
drivers/event/dlb2/dlb2_priv.h | 20 +-
drivers/event/dlb2/pf/base/dlb2_resource.c | 65 ++++++
drivers/event/dlb2/pf/dlb2_pf.c | 21 +-
6 files changed, 278 insertions(+), 83 deletions(-)
diff --git a/drivers/event/dlb2/dlb2.c b/drivers/event/dlb2/dlb2.c
index 9bac92c7b5..895dcb3550 100644
--- a/drivers/event/dlb2/dlb2.c
+++ b/drivers/event/dlb2/dlb2.c
@@ -116,6 +116,28 @@ dlb2_init_cq_weight(struct dlb2_eventdev *dlb2, int *cq_weight)
dlb2->ev_ports[q].cq_weight = cq_weight[q];
}
+/* override defaults with value(s) provided on command line */
+static void
+dlb2_init_port_cos(struct dlb2_eventdev *dlb2, int *port_cos)
+{
+ int q;
+
+ for (q = 0; q < DLB2_MAX_NUM_PORTS_ALL; q++) {
+ dlb2->ev_ports[q].cos_id = port_cos[q];
+ dlb2->cos_ports[port_cos[q]]++;
+ }
+}
+
+static void
+dlb2_init_cos_bw(struct dlb2_eventdev *dlb2,
+ struct dlb2_cos_bw *cos_bw)
+{
+ int q;
+ for (q = 0; q < DLB2_COS_NUM_VALS; q++)
+ dlb2->cos_bw[q] = cos_bw->val[q];
+
+}
+
static int
dlb2_hw_query_resources(struct dlb2_eventdev *dlb2)
{
@@ -330,36 +352,6 @@ set_dev_id(const char *key __rte_unused,
return 0;
}
-static int
-set_cos(const char *key __rte_unused,
- const char *value,
- void *opaque)
-{
- enum dlb2_cos *cos_id = opaque;
- int x = 0;
- int ret;
-
- if (value == NULL || opaque == NULL) {
- DLB2_LOG_ERR("NULL pointer\n");
- return -EINVAL;
- }
-
- ret = dlb2_string_to_int(&x, value);
- if (ret < 0)
- return ret;
-
- if (x != DLB2_COS_DEFAULT && (x < DLB2_COS_0 || x > DLB2_COS_3)) {
- DLB2_LOG_ERR(
- "COS %d out of range, must be DLB2_COS_DEFAULT or 0-3\n",
- x);
- return -EINVAL;
- }
-
- *cos_id = x;
-
- return 0;
-}
-
static int
set_poll_interval(const char *key __rte_unused,
const char *value,
@@ -603,6 +595,80 @@ set_cq_weight(const char *key __rte_unused,
return 0;
}
+static int
+set_port_cos(const char *key __rte_unused,
+ const char *value,
+ void *opaque)
+{
+ struct dlb2_port_cos *port_cos = opaque;
+ int first, last, cos_id, i;
+
+ if (value == NULL || opaque == NULL) {
+ DLB2_LOG_ERR("NULL pointer\n");
+ return -EINVAL;
+ }
+
+ /* command line override may take one of the following 3 forms:
+ * port_cos=all:<cos_id> ... all ports
+ * port_cos=port-port:<cos_id> ... a range of ports
+ * port_cos=port:<cos_id> ... just one port
+ */
+ if (sscanf(value, "all:%d", &cos_id) == 1) {
+ first = 0;
+ last = DLB2_MAX_NUM_LDB_PORTS - 1;
+ } else if (sscanf(value, "%d-%d:%d", &first, &last, &cos_id) == 3) {
+ /* we have everything we need */
+ } else if (sscanf(value, "%d:%d", &first, &cos_id) == 2) {
+ last = first;
+ } else {
+ DLB2_LOG_ERR("Error parsing ldb port port_cos devarg. Should be all:val, port-port:val, or port:val\n");
+ return -EINVAL;
+ }
+
+ if (first > last || first < 0 ||
+ last >= DLB2_MAX_NUM_LDB_PORTS) {
+ DLB2_LOG_ERR("Error parsing ldb port cos_id arg, invalid port value\n");
+ return -EINVAL;
+ }
+
+ if (cos_id < DLB2_COS_0 || cos_id > DLB2_COS_3) {
+ DLB2_LOG_ERR("Error parsing ldb port cos_id devarg, must be between 0 and 4\n");
+ return -EINVAL;
+ }
+
+ for (i = first; i <= last; i++)
+ port_cos->cos_id[i] = cos_id; /* indexed by port */
+
+ return 0;
+}
+
+static int
+set_cos_bw(const char *key __rte_unused,
+ const char *value,
+ void *opaque)
+{
+ struct dlb2_cos_bw *cos_bw = opaque;
+
+ if (opaque == NULL) {
+ DLB2_LOG_ERR("NULL pointer\n");
+ return -EINVAL;
+ }
+
+ /* format must be %d,%d,%d,%d */
+
+ if (sscanf(value, "%d,%d,%d,%d", &cos_bw->val[0], &cos_bw->val[1],
+ &cos_bw->val[2], &cos_bw->val[3]) != 4) {
+ DLB2_LOG_ERR("Error parsing cos bandwidth devarg. Should be bw0,bw1,bw2,bw3 where all values combined are <= 100\n");
+ return -EINVAL;
+ }
+ if (cos_bw->val[0] + cos_bw->val[1] + cos_bw->val[2] + cos_bw->val[4] > 100) {
+ DLB2_LOG_ERR("Error parsing cos bandwidth devarg. Should be bw0,bw1,bw2,bw3 where all values combined are <= 100\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
static void
dlb2_eventdev_info_get(struct rte_eventdev *dev,
struct rte_event_dev_info *dev_info)
@@ -647,11 +713,13 @@ dlb2_eventdev_info_get(struct rte_eventdev *dev,
}
static int
-dlb2_hw_create_sched_domain(struct dlb2_hw_dev *handle,
+dlb2_hw_create_sched_domain(struct dlb2_eventdev *dlb2,
+ struct dlb2_hw_dev *handle,
const struct dlb2_hw_rsrcs *resources_asked,
uint8_t device_version)
{
int ret = 0;
+ uint32_t cos_ports = 0;
struct dlb2_create_sched_domain_args *cfg;
if (resources_asked == NULL) {
@@ -677,38 +745,23 @@ dlb2_hw_create_sched_domain(struct dlb2_hw_dev *handle,
/* LDB ports */
- cfg->cos_strict = 0; /* Best effort */
- cfg->num_cos_ldb_ports[0] = 0;
- cfg->num_cos_ldb_ports[1] = 0;
- cfg->num_cos_ldb_ports[2] = 0;
- cfg->num_cos_ldb_ports[3] = 0;
-
- switch (handle->cos_id) {
- case DLB2_COS_0:
- cfg->num_ldb_ports = 0; /* no don't care ports */
- cfg->num_cos_ldb_ports[0] =
- resources_asked->num_ldb_ports;
- break;
- case DLB2_COS_1:
- cfg->num_ldb_ports = 0; /* no don't care ports */
- cfg->num_cos_ldb_ports[1] = resources_asked->num_ldb_ports;
- break;
- case DLB2_COS_2:
- cfg->num_ldb_ports = 0; /* no don't care ports */
- cfg->num_cos_ldb_ports[2] = resources_asked->num_ldb_ports;
- break;
- case DLB2_COS_3:
- cfg->num_ldb_ports = 0; /* no don't care ports */
- cfg->num_cos_ldb_ports[3] =
- resources_asked->num_ldb_ports;
- break;
- case DLB2_COS_DEFAULT:
- /* all ldb ports are don't care ports from a cos perspective */
- cfg->num_ldb_ports =
- resources_asked->num_ldb_ports;
- break;
+ /* tally of ports with non default COS */
+ cos_ports = dlb2->cos_ports[1] + dlb2->cos_ports[2] +
+ dlb2->cos_ports[3];
+
+ if (cos_ports > resources_asked->num_ldb_ports) {
+ DLB2_LOG_ERR("dlb2: num_ldb_ports < nonzero cos_ports\n");
+ ret = EINVAL;
+ goto error_exit;
}
+ cfg->cos_strict = 0; /* Best effort */
+ cfg->num_cos_ldb_ports[0] = resources_asked->num_ldb_ports - cos_ports;
+ cfg->num_cos_ldb_ports[1] = dlb2->cos_ports[1];
+ cfg->num_cos_ldb_ports[2] = dlb2->cos_ports[2];
+ cfg->num_cos_ldb_ports[3] = dlb2->cos_ports[3];
+
+
if (device_version == DLB2_HW_V2)
cfg->num_ldb_credits = resources_asked->num_ldb_credits;
@@ -886,7 +939,8 @@ dlb2_eventdev_configure(const struct rte_eventdev *dev)
rsrcs->num_dir_credits = dlb2->num_dir_credits_override;
}
- if (dlb2_hw_create_sched_domain(handle, rsrcs, dlb2->version) < 0) {
+ if (dlb2_hw_create_sched_domain(dlb2, handle, rsrcs,
+ dlb2->version) < 0) {
DLB2_LOG_ERR("dlb2_hw_create_sched_domain failed\n");
return -ENODEV;
}
@@ -1449,12 +1503,8 @@ dlb2_hw_create_ldb_port(struct dlb2_eventdev *dlb2,
cfg.cq_history_list_size = DLB2_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
- if (handle->cos_id == DLB2_COS_DEFAULT)
- cfg.cos_id = 0;
- else
- cfg.cos_id = handle->cos_id;
-
- cfg.cos_strict = 0;
+ cfg.cos_id = ev_port->cos_id;
+ cfg.cos_strict = 0;/* best effots */
/* User controls the LDB high watermark via enqueue depth. The DIR high
* watermark is equal, unless the directed credit pool is too small.
@@ -4668,7 +4718,6 @@ dlb2_primary_eventdev_probe(struct rte_eventdev *dev,
dlb2->max_num_events_override = dlb2_args->max_num_events;
dlb2->num_dir_credits_override = dlb2_args->num_dir_credits_override;
- dlb2->qm_instance.cos_id = dlb2_args->cos_id;
dlb2->poll_interval = dlb2_args->poll_interval;
dlb2->sw_credit_quanta = dlb2_args->sw_credit_quanta;
dlb2->hw_credit_quanta = dlb2_args->hw_credit_quanta;
@@ -4700,6 +4749,27 @@ dlb2_primary_eventdev_probe(struct rte_eventdev *dev,
dlb2_iface_hardware_init(&dlb2->qm_instance);
+ /* configure class of service */
+ {
+ struct dlb2_set_cos_bw_args set_cos_bw_args = {0};
+ int id;
+ int ret = 0;
+
+ for (id = 0; id < DLB2_COS_NUM_VALS; id++) {
+ set_cos_bw_args.cos_id = id;
+ set_cos_bw_args.cos_id = dlb2->cos_bw[id];
+ ret = dlb2_iface_set_cos_bw(&dlb2->qm_instance,
+ &set_cos_bw_args);
+ if (ret != 0)
+ break;
+ }
+ if (ret) {
+ DLB2_LOG_ERR("dlb2: failed to configure class of service, err=%d\n",
+ err);
+ return err;
+ }
+ }
+
err = dlb2_iface_get_cq_poll_mode(&dlb2->qm_instance, &dlb2->poll_mode);
if (err < 0) {
DLB2_LOG_ERR("dlb2: failed to get the poll mode, err=%d\n",
@@ -4730,6 +4800,12 @@ dlb2_primary_eventdev_probe(struct rte_eventdev *dev,
dlb2_init_cq_weight(dlb2,
dlb2_args->cq_weight.limit);
+ dlb2_init_port_cos(dlb2,
+ dlb2_args->port_cos.cos_id);
+
+ dlb2_init_cos_bw(dlb2,
+ &dlb2_args->cos_bw);
+
return 0;
}
@@ -4785,6 +4861,8 @@ dlb2_parse_params(const char *params,
DLB2_VECTOR_OPTS_ENAB_ARG,
DLB2_MAX_CQ_DEPTH,
DLB2_CQ_WEIGHT,
+ DLB2_PORT_COS,
+ DLB2_COS_BW,
NULL };
if (params != NULL && params[0] != '\0') {
@@ -4857,16 +4935,6 @@ dlb2_parse_params(const char *params,
return ret;
}
- ret = rte_kvargs_process(kvlist, DLB2_COS_ARG,
- set_cos,
- &dlb2_args->cos_id);
- if (ret != 0) {
- DLB2_LOG_ERR("%s: Error parsing cos parameter",
- name);
- rte_kvargs_free(kvlist);
- return ret;
- }
-
ret = rte_kvargs_process(kvlist, DLB2_POLL_INTERVAL_ARG,
set_poll_interval,
&dlb2_args->poll_interval);
@@ -4942,6 +5010,29 @@ dlb2_parse_params(const char *params,
return ret;
}
+ ret = rte_kvargs_process(kvlist,
+ DLB2_PORT_COS,
+ set_port_cos,
+ &dlb2_args->port_cos);
+ if (ret != 0) {
+ DLB2_LOG_ERR("%s: Error parsing port cos",
+ name);
+ rte_kvargs_free(kvlist);
+ return ret;
+ }
+
+ ret = rte_kvargs_process(kvlist,
+ DLB2_COS_BW,
+ set_cos_bw,
+ &dlb2_args->cos_bw);
+ if (ret != 0) {
+ DLB2_LOG_ERR("%s: Error parsing cos_bw",
+ name);
+ rte_kvargs_free(kvlist);
+ return ret;
+ }
+
+
rte_kvargs_free(kvlist);
}
}
diff --git a/drivers/event/dlb2/dlb2_iface.c b/drivers/event/dlb2/dlb2_iface.c
index 7913525e0f..2d4c5f617b 100644
--- a/drivers/event/dlb2/dlb2_iface.c
+++ b/drivers/event/dlb2/dlb2_iface.c
@@ -76,3 +76,6 @@ int (*dlb2_iface_get_dir_queue_depth)(struct dlb2_hw_dev *handle,
int (*dlb2_iface_enable_cq_weight)(struct dlb2_hw_dev *handle,
struct dlb2_enable_cq_weight_args *args);
+int (*dlb2_iface_set_cos_bw)(struct dlb2_hw_dev *handle,
+ struct dlb2_set_cos_bw_args *args);
+
diff --git a/drivers/event/dlb2/dlb2_iface.h b/drivers/event/dlb2/dlb2_iface.h
index 3aace0ba19..02ed99a9a7 100644
--- a/drivers/event/dlb2/dlb2_iface.h
+++ b/drivers/event/dlb2/dlb2_iface.h
@@ -75,4 +75,7 @@ extern int (*dlb2_iface_get_dir_queue_depth)(struct dlb2_hw_dev *handle,
extern int (*dlb2_iface_enable_cq_weight)(struct dlb2_hw_dev *handle,
struct dlb2_enable_cq_weight_args *args);
+extern int (*dlb2_iface_set_cos_bw)(struct dlb2_hw_dev *handle,
+ struct dlb2_set_cos_bw_args *args);
+
#endif /* _DLB2_IFACE_H_ */
diff --git a/drivers/event/dlb2/dlb2_priv.h b/drivers/event/dlb2/dlb2_priv.h
index 6084fc4183..20bf299563 100644
--- a/drivers/event/dlb2/dlb2_priv.h
+++ b/drivers/event/dlb2/dlb2_priv.h
@@ -45,6 +45,8 @@
#define DLB2_VECTOR_OPTS_ENAB_ARG "vector_opts_enable"
#define DLB2_MAX_CQ_DEPTH "max_cq_depth"
#define DLB2_CQ_WEIGHT "cq_weight"
+#define DLB2_PORT_COS "port_cos"
+#define DLB2_COS_BW "cos_bw"
/* Begin HW related defines and structs */
@@ -415,7 +417,8 @@ enum dlb2_cos {
DLB2_COS_0 = 0,
DLB2_COS_1,
DLB2_COS_2,
- DLB2_COS_3
+ DLB2_COS_3,
+ DLB2_COS_NUM_VALS
};
struct dlb2_hw_dev {
@@ -423,7 +426,6 @@ struct dlb2_hw_dev {
struct dlb2_hw_resource_info info;
void *pf_dev; /* opaque pointer to PF PMD dev (struct dlb2_dev) */
uint32_t domain_id;
- enum dlb2_cos cos_id;
rte_spinlock_t resource_lock; /* for MP support */
} __rte_cache_aligned;
@@ -527,6 +529,7 @@ struct dlb2_eventdev_port {
bool enq_configured;
uint8_t implicit_release; /* release events before dequeuing */
uint32_t cq_weight; /* DLB2.5 and above ldb ports only */
+ int cos_id; /*ldb port class of service */
} __rte_cache_aligned;
struct dlb2_queue {
@@ -621,6 +624,8 @@ struct dlb2_eventdev {
uint32_t credit_pool __rte_cache_aligned;
};
};
+ uint32_t cos_ports[DLB2_COS_NUM_VALS]; /* total ldb ports in each class */
+ uint32_t cos_bw[DLB2_COS_NUM_VALS]; /* bandwidth per cos domain */
};
/* used for collecting and passing around the dev args */
@@ -632,13 +637,20 @@ struct dlb2_cq_weight {
int limit[DLB2_MAX_NUM_LDB_PORTS];
};
+struct dlb2_port_cos {
+ int cos_id[DLB2_MAX_NUM_LDB_PORTS];
+};
+
+struct dlb2_cos_bw {
+ int val[DLB2_COS_NUM_VALS];
+};
+
struct dlb2_devargs {
int socket_id;
int max_num_events;
int num_dir_credits_override;
int dev_id;
struct dlb2_qid_depth_thresholds qid_depth_thresholds;
- enum dlb2_cos cos_id;
int poll_interval;
int sw_credit_quanta;
int hw_credit_quanta;
@@ -646,6 +658,8 @@ struct dlb2_devargs {
bool vector_opts_enabled;
int max_cq_depth;
struct dlb2_cq_weight cq_weight;
+ struct dlb2_port_cos port_cos;
+ struct dlb2_cos_bw cos_bw;
};
/* End Eventdev related defines and structs */
diff --git a/drivers/event/dlb2/pf/base/dlb2_resource.c b/drivers/event/dlb2/pf/base/dlb2_resource.c
index 54222d732c..cb295bab2f 100644
--- a/drivers/event/dlb2/pf/base/dlb2_resource.c
+++ b/drivers/event/dlb2/pf/base/dlb2_resource.c
@@ -6467,3 +6467,68 @@ int dlb2_hw_enable_cq_weight(struct dlb2_hw *hw,
return 0;
}
+static void dlb2_log_set_cos_bandwidth(struct dlb2_hw *hw, u32 cos_id, u8 bw)
+{
+ DLB2_HW_DBG(hw, "DLB2 set port CoS bandwidth:\n");
+ DLB2_HW_DBG(hw, "\tCoS ID: %u\n", cos_id);
+ DLB2_HW_DBG(hw, "\tBandwidth: %u\n", bw);
+}
+
+#define DLB2_MAX_BW_PCT 100
+
+/**
+ * dlb2_hw_set_cos_bandwidth() - set a bandwidth allocation percentage for a
+ * port class-of-service.
+ * @hw: dlb2_hw handle for a particular device.
+ * @cos_id: class-of-service ID.
+ * @bandwidth: class-of-service bandwidth.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - Invalid cos ID, bandwidth is greater than 100, or bandwidth would
+ * cause the total bandwidth across all classes of service to exceed
+ * 100%.
+ */
+int dlb2_hw_set_cos_bandwidth(struct dlb2_hw *hw, u32 cos_id, u8 bandwidth)
+{
+ unsigned int i;
+ u32 reg;
+ u8 total;
+
+ if (cos_id >= DLB2_NUM_COS_DOMAINS)
+ return -EINVAL;
+
+ if (bandwidth > DLB2_MAX_BW_PCT)
+ return -EINVAL;
+
+ total = 0;
+
+ for (i = 0; i < DLB2_NUM_COS_DOMAINS; i++)
+ total += (i == cos_id) ? bandwidth : hw->cos_reservation[i];
+
+ if (total > DLB2_MAX_BW_PCT)
+ return -EINVAL;
+
+ reg = DLB2_CSR_RD(hw, DLB2_LSP_CFG_SHDW_RANGE_COS(hw->ver, cos_id));
+
+ /*
+ * Normalize the bandwidth to a value in the range 0-255. Integer
+ * division may leave unreserved scheduling slots; these will be
+ * divided among the 4 classes of service.
+ */
+ DLB2_BITS_SET(reg, (bandwidth * 256) / 100, DLB2_LSP_CFG_SHDW_RANGE_COS_BW_RANGE);
+ DLB2_CSR_WR(hw, DLB2_LSP_CFG_SHDW_RANGE_COS(hw->ver, cos_id), reg);
+
+ reg = 0;
+ DLB2_BIT_SET(reg, DLB2_LSP_CFG_SHDW_CTRL_TRANSFER);
+ /* Atomically transfer the newly configured service weight */
+ DLB2_CSR_WR(hw, DLB2_LSP_CFG_SHDW_CTRL(hw->ver), reg);
+
+ dlb2_log_set_cos_bandwidth(hw, cos_id, bandwidth);
+
+ hw->cos_reservation[cos_id] = bandwidth;
+
+ return 0;
+}
diff --git a/drivers/event/dlb2/pf/dlb2_pf.c b/drivers/event/dlb2/pf/dlb2_pf.c
index 10918ff281..244c1591f8 100644
--- a/drivers/event/dlb2/pf/dlb2_pf.c
+++ b/drivers/event/dlb2/pf/dlb2_pf.c
@@ -645,6 +645,25 @@ dlb2_pf_enable_cq_weight(struct dlb2_hw_dev *handle,
return ret;
}
+static int
+dlb2_pf_set_cos_bandwidth(struct dlb2_hw_dev *handle,
+ struct dlb2_set_cos_bw_args *args)
+{
+ struct dlb2_dev *dlb2_dev = (struct dlb2_dev *)handle->pf_dev;
+ int ret = 0;
+
+ DLB2_INFO(dev->dlb2_device, "Entering %s()\n", __func__);
+
+ ret = dlb2_hw_set_cos_bandwidth(&dlb2_dev->hw,
+ args->cos_id,
+ args->bandwidth);
+
+ DLB2_INFO(dev->dlb2_device, "Exiting %s() with ret=%d\n",
+ __func__, ret);
+
+ return ret;
+}
+
static void
dlb2_pf_iface_fn_ptrs_init(void)
{
@@ -670,6 +689,7 @@ dlb2_pf_iface_fn_ptrs_init(void)
dlb2_iface_set_sn_allocation = dlb2_pf_set_sn_allocation;
dlb2_iface_get_sn_occupancy = dlb2_pf_get_sn_occupancy;
dlb2_iface_enable_cq_weight = dlb2_pf_enable_cq_weight;
+ dlb2_iface_set_cos_bw = dlb2_pf_set_cos_bandwidth;
}
/* PCI DEV HOOKS */
@@ -683,7 +703,6 @@ dlb2_eventdev_pci_init(struct rte_eventdev *eventdev)
.max_num_events = DLB2_MAX_NUM_LDB_CREDITS,
.num_dir_credits_override = -1,
.qid_depth_thresholds = { {0} },
- .cos_id = DLB2_COS_DEFAULT,
.poll_interval = DLB2_POLL_INTERVAL_DEFAULT,
.sw_credit_quanta = DLB2_SW_CREDIT_QUANTA_DEFAULT,
.hw_credit_quanta = DLB2_SW_CREDIT_BATCH_SZ,
--
2.23.0
^ permalink raw reply [flat|nested] 3+ messages in thread
* Re: [PATCH] event/dlb2: add ldb port specific COS support
2022-04-10 22:56 [PATCH] event/dlb2: add ldb port specific COS support Timothy McDaniel
@ 2022-06-14 10:42 ` Jerin Jacob
2022-06-14 10:45 ` Jerin Jacob
0 siblings, 1 reply; 3+ messages in thread
From: Jerin Jacob @ 2022-06-14 10:42 UTC (permalink / raw)
To: Timothy McDaniel; +Cc: Jerin Jacob, dpdk-dev
On Mon, Apr 11, 2022 at 4:26 AM Timothy McDaniel
<timothy.mcdaniel@intel.com> wrote:
>
> DLB supports 4 class of service domains, to aid in managing the
> device bandwidth across ldb ports. This commit allows specifying
> which ldb ports will participate in the COS scheme, which class
> they are a part of, and the specific bandwidth percentage
> associated with each class. The cumulative bandwidth associated
> with the 4 classes must not exceed 100%. This feature is enabled
> on the command line, and will be documented in the DLB2 programmers
> guide.
>
> Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
> ---
> Depends-on: patch-109560 ("[v2] event/dlb2: add CQ weight support")
Please rebase with dpdk-next-eventdev tree.
[for-main]dell[dpdk-next-eventdev] $ git pw series apply 22464
Failed to apply patch:
Applying: event/dlb2: add CQ weight support
.git/rebase-apply/patch:198: new blank line at EOF.
+
.git/rebase-apply/patch:551: new blank line at EOF.
+
warning: 2 lines add whitespace errors.
Using index info to reconstruct a base tree...
M drivers/event/dlb2/dlb2.c
M drivers/event/dlb2/dlb2_priv.h
M drivers/event/dlb2/pf/base/dlb2_resource.c
Falling back to patching base and 3-way merge...
Auto-merging drivers/event/dlb2/pf/base/dlb2_resource.c
Auto-merging drivers/event/dlb2/dlb2_priv.h
CONFLICT (content): Merge conflict in drivers/event/dlb2/dlb2_priv.h
Auto-merging drivers/event/dlb2/dlb2.c
CONFLICT (content): Merge conflict in drivers/event/dlb2/dlb2.c
error: Failed to merge in the changes.
hint: Use 'git am --show-current-patch=diff' to see the failed patch
Patch failed at 0001 event/dlb2: add CQ weight support
When you have resolved this problem, run "git am --continue".
If you prefer to skip this patch, run "git am --skip" instead.
To restore the original branch and stop patching, run "git am --abort"
> ---
> drivers/event/dlb2/dlb2.c | 249 ++++++++++++++-------
> drivers/event/dlb2/dlb2_iface.c | 3 +
> drivers/event/dlb2/dlb2_iface.h | 3 +
> drivers/event/dlb2/dlb2_priv.h | 20 +-
> drivers/event/dlb2/pf/base/dlb2_resource.c | 65 ++++++
> drivers/event/dlb2/pf/dlb2_pf.c | 21 +-
> 6 files changed, 278 insertions(+), 83 deletions(-)
>
> diff --git a/drivers/event/dlb2/dlb2.c b/drivers/event/dlb2/dlb2.c
> index 9bac92c7b5..895dcb3550 100644
> --- a/drivers/event/dlb2/dlb2.c
> +++ b/drivers/event/dlb2/dlb2.c
> @@ -116,6 +116,28 @@ dlb2_init_cq_weight(struct dlb2_eventdev *dlb2, int *cq_weight)
> dlb2->ev_ports[q].cq_weight = cq_weight[q];
> }
>
> +/* override defaults with value(s) provided on command line */
> +static void
> +dlb2_init_port_cos(struct dlb2_eventdev *dlb2, int *port_cos)
> +{
> + int q;
> +
> + for (q = 0; q < DLB2_MAX_NUM_PORTS_ALL; q++) {
> + dlb2->ev_ports[q].cos_id = port_cos[q];
> + dlb2->cos_ports[port_cos[q]]++;
> + }
> +}
> +
> +static void
> +dlb2_init_cos_bw(struct dlb2_eventdev *dlb2,
> + struct dlb2_cos_bw *cos_bw)
> +{
> + int q;
> + for (q = 0; q < DLB2_COS_NUM_VALS; q++)
> + dlb2->cos_bw[q] = cos_bw->val[q];
> +
> +}
> +
> static int
> dlb2_hw_query_resources(struct dlb2_eventdev *dlb2)
> {
> @@ -330,36 +352,6 @@ set_dev_id(const char *key __rte_unused,
> return 0;
> }
>
> -static int
> -set_cos(const char *key __rte_unused,
> - const char *value,
> - void *opaque)
> -{
> - enum dlb2_cos *cos_id = opaque;
> - int x = 0;
> - int ret;
> -
> - if (value == NULL || opaque == NULL) {
> - DLB2_LOG_ERR("NULL pointer\n");
> - return -EINVAL;
> - }
> -
> - ret = dlb2_string_to_int(&x, value);
> - if (ret < 0)
> - return ret;
> -
> - if (x != DLB2_COS_DEFAULT && (x < DLB2_COS_0 || x > DLB2_COS_3)) {
> - DLB2_LOG_ERR(
> - "COS %d out of range, must be DLB2_COS_DEFAULT or 0-3\n",
> - x);
> - return -EINVAL;
> - }
> -
> - *cos_id = x;
> -
> - return 0;
> -}
> -
> static int
> set_poll_interval(const char *key __rte_unused,
> const char *value,
> @@ -603,6 +595,80 @@ set_cq_weight(const char *key __rte_unused,
> return 0;
> }
>
> +static int
> +set_port_cos(const char *key __rte_unused,
> + const char *value,
> + void *opaque)
> +{
> + struct dlb2_port_cos *port_cos = opaque;
> + int first, last, cos_id, i;
> +
> + if (value == NULL || opaque == NULL) {
> + DLB2_LOG_ERR("NULL pointer\n");
> + return -EINVAL;
> + }
> +
> + /* command line override may take one of the following 3 forms:
> + * port_cos=all:<cos_id> ... all ports
> + * port_cos=port-port:<cos_id> ... a range of ports
> + * port_cos=port:<cos_id> ... just one port
> + */
> + if (sscanf(value, "all:%d", &cos_id) == 1) {
> + first = 0;
> + last = DLB2_MAX_NUM_LDB_PORTS - 1;
> + } else if (sscanf(value, "%d-%d:%d", &first, &last, &cos_id) == 3) {
> + /* we have everything we need */
> + } else if (sscanf(value, "%d:%d", &first, &cos_id) == 2) {
> + last = first;
> + } else {
> + DLB2_LOG_ERR("Error parsing ldb port port_cos devarg. Should be all:val, port-port:val, or port:val\n");
> + return -EINVAL;
> + }
> +
> + if (first > last || first < 0 ||
> + last >= DLB2_MAX_NUM_LDB_PORTS) {
> + DLB2_LOG_ERR("Error parsing ldb port cos_id arg, invalid port value\n");
> + return -EINVAL;
> + }
> +
> + if (cos_id < DLB2_COS_0 || cos_id > DLB2_COS_3) {
> + DLB2_LOG_ERR("Error parsing ldb port cos_id devarg, must be between 0 and 4\n");
> + return -EINVAL;
> + }
> +
> + for (i = first; i <= last; i++)
> + port_cos->cos_id[i] = cos_id; /* indexed by port */
> +
> + return 0;
> +}
> +
> +static int
> +set_cos_bw(const char *key __rte_unused,
> + const char *value,
> + void *opaque)
> +{
> + struct dlb2_cos_bw *cos_bw = opaque;
> +
> + if (opaque == NULL) {
> + DLB2_LOG_ERR("NULL pointer\n");
> + return -EINVAL;
> + }
> +
> + /* format must be %d,%d,%d,%d */
> +
> + if (sscanf(value, "%d,%d,%d,%d", &cos_bw->val[0], &cos_bw->val[1],
> + &cos_bw->val[2], &cos_bw->val[3]) != 4) {
> + DLB2_LOG_ERR("Error parsing cos bandwidth devarg. Should be bw0,bw1,bw2,bw3 where all values combined are <= 100\n");
> + return -EINVAL;
> + }
> + if (cos_bw->val[0] + cos_bw->val[1] + cos_bw->val[2] + cos_bw->val[4] > 100) {
> + DLB2_LOG_ERR("Error parsing cos bandwidth devarg. Should be bw0,bw1,bw2,bw3 where all values combined are <= 100\n");
> + return -EINVAL;
> + }
> +
> + return 0;
> +}
> +
> static void
> dlb2_eventdev_info_get(struct rte_eventdev *dev,
> struct rte_event_dev_info *dev_info)
> @@ -647,11 +713,13 @@ dlb2_eventdev_info_get(struct rte_eventdev *dev,
> }
>
> static int
> -dlb2_hw_create_sched_domain(struct dlb2_hw_dev *handle,
> +dlb2_hw_create_sched_domain(struct dlb2_eventdev *dlb2,
> + struct dlb2_hw_dev *handle,
> const struct dlb2_hw_rsrcs *resources_asked,
> uint8_t device_version)
> {
> int ret = 0;
> + uint32_t cos_ports = 0;
> struct dlb2_create_sched_domain_args *cfg;
>
> if (resources_asked == NULL) {
> @@ -677,38 +745,23 @@ dlb2_hw_create_sched_domain(struct dlb2_hw_dev *handle,
>
> /* LDB ports */
>
> - cfg->cos_strict = 0; /* Best effort */
> - cfg->num_cos_ldb_ports[0] = 0;
> - cfg->num_cos_ldb_ports[1] = 0;
> - cfg->num_cos_ldb_ports[2] = 0;
> - cfg->num_cos_ldb_ports[3] = 0;
> -
> - switch (handle->cos_id) {
> - case DLB2_COS_0:
> - cfg->num_ldb_ports = 0; /* no don't care ports */
> - cfg->num_cos_ldb_ports[0] =
> - resources_asked->num_ldb_ports;
> - break;
> - case DLB2_COS_1:
> - cfg->num_ldb_ports = 0; /* no don't care ports */
> - cfg->num_cos_ldb_ports[1] = resources_asked->num_ldb_ports;
> - break;
> - case DLB2_COS_2:
> - cfg->num_ldb_ports = 0; /* no don't care ports */
> - cfg->num_cos_ldb_ports[2] = resources_asked->num_ldb_ports;
> - break;
> - case DLB2_COS_3:
> - cfg->num_ldb_ports = 0; /* no don't care ports */
> - cfg->num_cos_ldb_ports[3] =
> - resources_asked->num_ldb_ports;
> - break;
> - case DLB2_COS_DEFAULT:
> - /* all ldb ports are don't care ports from a cos perspective */
> - cfg->num_ldb_ports =
> - resources_asked->num_ldb_ports;
> - break;
> + /* tally of ports with non default COS */
> + cos_ports = dlb2->cos_ports[1] + dlb2->cos_ports[2] +
> + dlb2->cos_ports[3];
> +
> + if (cos_ports > resources_asked->num_ldb_ports) {
> + DLB2_LOG_ERR("dlb2: num_ldb_ports < nonzero cos_ports\n");
> + ret = EINVAL;
> + goto error_exit;
> }
>
> + cfg->cos_strict = 0; /* Best effort */
> + cfg->num_cos_ldb_ports[0] = resources_asked->num_ldb_ports - cos_ports;
> + cfg->num_cos_ldb_ports[1] = dlb2->cos_ports[1];
> + cfg->num_cos_ldb_ports[2] = dlb2->cos_ports[2];
> + cfg->num_cos_ldb_ports[3] = dlb2->cos_ports[3];
> +
> +
> if (device_version == DLB2_HW_V2)
> cfg->num_ldb_credits = resources_asked->num_ldb_credits;
>
> @@ -886,7 +939,8 @@ dlb2_eventdev_configure(const struct rte_eventdev *dev)
> rsrcs->num_dir_credits = dlb2->num_dir_credits_override;
> }
>
> - if (dlb2_hw_create_sched_domain(handle, rsrcs, dlb2->version) < 0) {
> + if (dlb2_hw_create_sched_domain(dlb2, handle, rsrcs,
> + dlb2->version) < 0) {
> DLB2_LOG_ERR("dlb2_hw_create_sched_domain failed\n");
> return -ENODEV;
> }
> @@ -1449,12 +1503,8 @@ dlb2_hw_create_ldb_port(struct dlb2_eventdev *dlb2,
>
> cfg.cq_history_list_size = DLB2_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
>
> - if (handle->cos_id == DLB2_COS_DEFAULT)
> - cfg.cos_id = 0;
> - else
> - cfg.cos_id = handle->cos_id;
> -
> - cfg.cos_strict = 0;
> + cfg.cos_id = ev_port->cos_id;
> + cfg.cos_strict = 0;/* best effots */
>
> /* User controls the LDB high watermark via enqueue depth. The DIR high
> * watermark is equal, unless the directed credit pool is too small.
> @@ -4668,7 +4718,6 @@ dlb2_primary_eventdev_probe(struct rte_eventdev *dev,
>
> dlb2->max_num_events_override = dlb2_args->max_num_events;
> dlb2->num_dir_credits_override = dlb2_args->num_dir_credits_override;
> - dlb2->qm_instance.cos_id = dlb2_args->cos_id;
> dlb2->poll_interval = dlb2_args->poll_interval;
> dlb2->sw_credit_quanta = dlb2_args->sw_credit_quanta;
> dlb2->hw_credit_quanta = dlb2_args->hw_credit_quanta;
> @@ -4700,6 +4749,27 @@ dlb2_primary_eventdev_probe(struct rte_eventdev *dev,
>
> dlb2_iface_hardware_init(&dlb2->qm_instance);
>
> + /* configure class of service */
> + {
> + struct dlb2_set_cos_bw_args set_cos_bw_args = {0};
> + int id;
> + int ret = 0;
> +
> + for (id = 0; id < DLB2_COS_NUM_VALS; id++) {
> + set_cos_bw_args.cos_id = id;
> + set_cos_bw_args.cos_id = dlb2->cos_bw[id];
> + ret = dlb2_iface_set_cos_bw(&dlb2->qm_instance,
> + &set_cos_bw_args);
> + if (ret != 0)
> + break;
> + }
> + if (ret) {
> + DLB2_LOG_ERR("dlb2: failed to configure class of service, err=%d\n",
> + err);
> + return err;
> + }
> + }
> +
> err = dlb2_iface_get_cq_poll_mode(&dlb2->qm_instance, &dlb2->poll_mode);
> if (err < 0) {
> DLB2_LOG_ERR("dlb2: failed to get the poll mode, err=%d\n",
> @@ -4730,6 +4800,12 @@ dlb2_primary_eventdev_probe(struct rte_eventdev *dev,
> dlb2_init_cq_weight(dlb2,
> dlb2_args->cq_weight.limit);
>
> + dlb2_init_port_cos(dlb2,
> + dlb2_args->port_cos.cos_id);
> +
> + dlb2_init_cos_bw(dlb2,
> + &dlb2_args->cos_bw);
> +
> return 0;
> }
>
> @@ -4785,6 +4861,8 @@ dlb2_parse_params(const char *params,
> DLB2_VECTOR_OPTS_ENAB_ARG,
> DLB2_MAX_CQ_DEPTH,
> DLB2_CQ_WEIGHT,
> + DLB2_PORT_COS,
> + DLB2_COS_BW,
> NULL };
>
> if (params != NULL && params[0] != '\0') {
> @@ -4857,16 +4935,6 @@ dlb2_parse_params(const char *params,
> return ret;
> }
>
> - ret = rte_kvargs_process(kvlist, DLB2_COS_ARG,
> - set_cos,
> - &dlb2_args->cos_id);
> - if (ret != 0) {
> - DLB2_LOG_ERR("%s: Error parsing cos parameter",
> - name);
> - rte_kvargs_free(kvlist);
> - return ret;
> - }
> -
> ret = rte_kvargs_process(kvlist, DLB2_POLL_INTERVAL_ARG,
> set_poll_interval,
> &dlb2_args->poll_interval);
> @@ -4942,6 +5010,29 @@ dlb2_parse_params(const char *params,
> return ret;
> }
>
> + ret = rte_kvargs_process(kvlist,
> + DLB2_PORT_COS,
> + set_port_cos,
> + &dlb2_args->port_cos);
> + if (ret != 0) {
> + DLB2_LOG_ERR("%s: Error parsing port cos",
> + name);
> + rte_kvargs_free(kvlist);
> + return ret;
> + }
> +
> + ret = rte_kvargs_process(kvlist,
> + DLB2_COS_BW,
> + set_cos_bw,
> + &dlb2_args->cos_bw);
> + if (ret != 0) {
> + DLB2_LOG_ERR("%s: Error parsing cos_bw",
> + name);
> + rte_kvargs_free(kvlist);
> + return ret;
> + }
> +
> +
> rte_kvargs_free(kvlist);
> }
> }
> diff --git a/drivers/event/dlb2/dlb2_iface.c b/drivers/event/dlb2/dlb2_iface.c
> index 7913525e0f..2d4c5f617b 100644
> --- a/drivers/event/dlb2/dlb2_iface.c
> +++ b/drivers/event/dlb2/dlb2_iface.c
> @@ -76,3 +76,6 @@ int (*dlb2_iface_get_dir_queue_depth)(struct dlb2_hw_dev *handle,
> int (*dlb2_iface_enable_cq_weight)(struct dlb2_hw_dev *handle,
> struct dlb2_enable_cq_weight_args *args);
>
> +int (*dlb2_iface_set_cos_bw)(struct dlb2_hw_dev *handle,
> + struct dlb2_set_cos_bw_args *args);
> +
> diff --git a/drivers/event/dlb2/dlb2_iface.h b/drivers/event/dlb2/dlb2_iface.h
> index 3aace0ba19..02ed99a9a7 100644
> --- a/drivers/event/dlb2/dlb2_iface.h
> +++ b/drivers/event/dlb2/dlb2_iface.h
> @@ -75,4 +75,7 @@ extern int (*dlb2_iface_get_dir_queue_depth)(struct dlb2_hw_dev *handle,
> extern int (*dlb2_iface_enable_cq_weight)(struct dlb2_hw_dev *handle,
> struct dlb2_enable_cq_weight_args *args);
>
> +extern int (*dlb2_iface_set_cos_bw)(struct dlb2_hw_dev *handle,
> + struct dlb2_set_cos_bw_args *args);
> +
> #endif /* _DLB2_IFACE_H_ */
> diff --git a/drivers/event/dlb2/dlb2_priv.h b/drivers/event/dlb2/dlb2_priv.h
> index 6084fc4183..20bf299563 100644
> --- a/drivers/event/dlb2/dlb2_priv.h
> +++ b/drivers/event/dlb2/dlb2_priv.h
> @@ -45,6 +45,8 @@
> #define DLB2_VECTOR_OPTS_ENAB_ARG "vector_opts_enable"
> #define DLB2_MAX_CQ_DEPTH "max_cq_depth"
> #define DLB2_CQ_WEIGHT "cq_weight"
> +#define DLB2_PORT_COS "port_cos"
> +#define DLB2_COS_BW "cos_bw"
>
> /* Begin HW related defines and structs */
>
> @@ -415,7 +417,8 @@ enum dlb2_cos {
> DLB2_COS_0 = 0,
> DLB2_COS_1,
> DLB2_COS_2,
> - DLB2_COS_3
> + DLB2_COS_3,
> + DLB2_COS_NUM_VALS
> };
>
> struct dlb2_hw_dev {
> @@ -423,7 +426,6 @@ struct dlb2_hw_dev {
> struct dlb2_hw_resource_info info;
> void *pf_dev; /* opaque pointer to PF PMD dev (struct dlb2_dev) */
> uint32_t domain_id;
> - enum dlb2_cos cos_id;
> rte_spinlock_t resource_lock; /* for MP support */
> } __rte_cache_aligned;
>
> @@ -527,6 +529,7 @@ struct dlb2_eventdev_port {
> bool enq_configured;
> uint8_t implicit_release; /* release events before dequeuing */
> uint32_t cq_weight; /* DLB2.5 and above ldb ports only */
> + int cos_id; /*ldb port class of service */
> } __rte_cache_aligned;
>
> struct dlb2_queue {
> @@ -621,6 +624,8 @@ struct dlb2_eventdev {
> uint32_t credit_pool __rte_cache_aligned;
> };
> };
> + uint32_t cos_ports[DLB2_COS_NUM_VALS]; /* total ldb ports in each class */
> + uint32_t cos_bw[DLB2_COS_NUM_VALS]; /* bandwidth per cos domain */
> };
>
> /* used for collecting and passing around the dev args */
> @@ -632,13 +637,20 @@ struct dlb2_cq_weight {
> int limit[DLB2_MAX_NUM_LDB_PORTS];
> };
>
> +struct dlb2_port_cos {
> + int cos_id[DLB2_MAX_NUM_LDB_PORTS];
> +};
> +
> +struct dlb2_cos_bw {
> + int val[DLB2_COS_NUM_VALS];
> +};
> +
> struct dlb2_devargs {
> int socket_id;
> int max_num_events;
> int num_dir_credits_override;
> int dev_id;
> struct dlb2_qid_depth_thresholds qid_depth_thresholds;
> - enum dlb2_cos cos_id;
> int poll_interval;
> int sw_credit_quanta;
> int hw_credit_quanta;
> @@ -646,6 +658,8 @@ struct dlb2_devargs {
> bool vector_opts_enabled;
> int max_cq_depth;
> struct dlb2_cq_weight cq_weight;
> + struct dlb2_port_cos port_cos;
> + struct dlb2_cos_bw cos_bw;
> };
>
> /* End Eventdev related defines and structs */
> diff --git a/drivers/event/dlb2/pf/base/dlb2_resource.c b/drivers/event/dlb2/pf/base/dlb2_resource.c
> index 54222d732c..cb295bab2f 100644
> --- a/drivers/event/dlb2/pf/base/dlb2_resource.c
> +++ b/drivers/event/dlb2/pf/base/dlb2_resource.c
> @@ -6467,3 +6467,68 @@ int dlb2_hw_enable_cq_weight(struct dlb2_hw *hw,
> return 0;
> }
>
> +static void dlb2_log_set_cos_bandwidth(struct dlb2_hw *hw, u32 cos_id, u8 bw)
> +{
> + DLB2_HW_DBG(hw, "DLB2 set port CoS bandwidth:\n");
> + DLB2_HW_DBG(hw, "\tCoS ID: %u\n", cos_id);
> + DLB2_HW_DBG(hw, "\tBandwidth: %u\n", bw);
> +}
> +
> +#define DLB2_MAX_BW_PCT 100
> +
> +/**
> + * dlb2_hw_set_cos_bandwidth() - set a bandwidth allocation percentage for a
> + * port class-of-service.
> + * @hw: dlb2_hw handle for a particular device.
> + * @cos_id: class-of-service ID.
> + * @bandwidth: class-of-service bandwidth.
> + *
> + * Return:
> + * Returns 0 upon success, < 0 otherwise.
> + *
> + * Errors:
> + * EINVAL - Invalid cos ID, bandwidth is greater than 100, or bandwidth would
> + * cause the total bandwidth across all classes of service to exceed
> + * 100%.
> + */
> +int dlb2_hw_set_cos_bandwidth(struct dlb2_hw *hw, u32 cos_id, u8 bandwidth)
> +{
> + unsigned int i;
> + u32 reg;
> + u8 total;
> +
> + if (cos_id >= DLB2_NUM_COS_DOMAINS)
> + return -EINVAL;
> +
> + if (bandwidth > DLB2_MAX_BW_PCT)
> + return -EINVAL;
> +
> + total = 0;
> +
> + for (i = 0; i < DLB2_NUM_COS_DOMAINS; i++)
> + total += (i == cos_id) ? bandwidth : hw->cos_reservation[i];
> +
> + if (total > DLB2_MAX_BW_PCT)
> + return -EINVAL;
> +
> + reg = DLB2_CSR_RD(hw, DLB2_LSP_CFG_SHDW_RANGE_COS(hw->ver, cos_id));
> +
> + /*
> + * Normalize the bandwidth to a value in the range 0-255. Integer
> + * division may leave unreserved scheduling slots; these will be
> + * divided among the 4 classes of service.
> + */
> + DLB2_BITS_SET(reg, (bandwidth * 256) / 100, DLB2_LSP_CFG_SHDW_RANGE_COS_BW_RANGE);
> + DLB2_CSR_WR(hw, DLB2_LSP_CFG_SHDW_RANGE_COS(hw->ver, cos_id), reg);
> +
> + reg = 0;
> + DLB2_BIT_SET(reg, DLB2_LSP_CFG_SHDW_CTRL_TRANSFER);
> + /* Atomically transfer the newly configured service weight */
> + DLB2_CSR_WR(hw, DLB2_LSP_CFG_SHDW_CTRL(hw->ver), reg);
> +
> + dlb2_log_set_cos_bandwidth(hw, cos_id, bandwidth);
> +
> + hw->cos_reservation[cos_id] = bandwidth;
> +
> + return 0;
> +}
> diff --git a/drivers/event/dlb2/pf/dlb2_pf.c b/drivers/event/dlb2/pf/dlb2_pf.c
> index 10918ff281..244c1591f8 100644
> --- a/drivers/event/dlb2/pf/dlb2_pf.c
> +++ b/drivers/event/dlb2/pf/dlb2_pf.c
> @@ -645,6 +645,25 @@ dlb2_pf_enable_cq_weight(struct dlb2_hw_dev *handle,
> return ret;
> }
>
> +static int
> +dlb2_pf_set_cos_bandwidth(struct dlb2_hw_dev *handle,
> + struct dlb2_set_cos_bw_args *args)
> +{
> + struct dlb2_dev *dlb2_dev = (struct dlb2_dev *)handle->pf_dev;
> + int ret = 0;
> +
> + DLB2_INFO(dev->dlb2_device, "Entering %s()\n", __func__);
> +
> + ret = dlb2_hw_set_cos_bandwidth(&dlb2_dev->hw,
> + args->cos_id,
> + args->bandwidth);
> +
> + DLB2_INFO(dev->dlb2_device, "Exiting %s() with ret=%d\n",
> + __func__, ret);
> +
> + return ret;
> +}
> +
> static void
> dlb2_pf_iface_fn_ptrs_init(void)
> {
> @@ -670,6 +689,7 @@ dlb2_pf_iface_fn_ptrs_init(void)
> dlb2_iface_set_sn_allocation = dlb2_pf_set_sn_allocation;
> dlb2_iface_get_sn_occupancy = dlb2_pf_get_sn_occupancy;
> dlb2_iface_enable_cq_weight = dlb2_pf_enable_cq_weight;
> + dlb2_iface_set_cos_bw = dlb2_pf_set_cos_bandwidth;
> }
>
> /* PCI DEV HOOKS */
> @@ -683,7 +703,6 @@ dlb2_eventdev_pci_init(struct rte_eventdev *eventdev)
> .max_num_events = DLB2_MAX_NUM_LDB_CREDITS,
> .num_dir_credits_override = -1,
> .qid_depth_thresholds = { {0} },
> - .cos_id = DLB2_COS_DEFAULT,
> .poll_interval = DLB2_POLL_INTERVAL_DEFAULT,
> .sw_credit_quanta = DLB2_SW_CREDIT_QUANTA_DEFAULT,
> .hw_credit_quanta = DLB2_SW_CREDIT_BATCH_SZ,
> --
> 2.23.0
>
^ permalink raw reply [flat|nested] 3+ messages in thread
* Re: [PATCH] event/dlb2: add ldb port specific COS support
2022-06-14 10:42 ` Jerin Jacob
@ 2022-06-14 10:45 ` Jerin Jacob
0 siblings, 0 replies; 3+ messages in thread
From: Jerin Jacob @ 2022-06-14 10:45 UTC (permalink / raw)
To: Timothy McDaniel; +Cc: Jerin Jacob, dpdk-dev
On Tue, Jun 14, 2022 at 4:12 PM Jerin Jacob <jerinjacobk@gmail.com> wrote:
>
> On Mon, Apr 11, 2022 at 4:26 AM Timothy McDaniel
> <timothy.mcdaniel@intel.com> wrote:
> >
> > DLB supports 4 class of service domains, to aid in managing the
> > device bandwidth across ldb ports. This commit allows specifying
> > which ldb ports will participate in the COS scheme, which class
> > they are a part of, and the specific bandwidth percentage
> > associated with each class. The cumulative bandwidth associated
> > with the 4 classes must not exceed 100%. This feature is enabled
> > on the command line, and will be documented in the DLB2 programmers
> > guide.
> >
> > Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
> > ---
> > Depends-on: patch-109560 ("[v2] event/dlb2: add CQ weight support")
>
>
>
> Please rebase with dpdk-next-eventdev tree.
>
> [for-main]dell[dpdk-next-eventdev] $ git pw series apply 22464
> Failed to apply patch:
> Applying: event/dlb2: add CQ weight support
> .git/rebase-apply/patch:198: new blank line at EOF.
> +
> .git/rebase-apply/patch:551: new blank line at EOF.
> +
> warning: 2 lines add whitespace errors.
> Using index info to reconstruct a base tree...
> M drivers/event/dlb2/dlb2.c
> M drivers/event/dlb2/dlb2_priv.h
> M drivers/event/dlb2/pf/base/dlb2_resource.c
> Falling back to patching base and 3-way merge...
> Auto-merging drivers/event/dlb2/pf/base/dlb2_resource.c
> Auto-merging drivers/event/dlb2/dlb2_priv.h
> CONFLICT (content): Merge conflict in drivers/event/dlb2/dlb2_priv.h
> Auto-merging drivers/event/dlb2/dlb2.c
> CONFLICT (content): Merge conflict in drivers/event/dlb2/dlb2.c
> error: Failed to merge in the changes.
> hint: Use 'git am --show-current-patch=diff' to see the failed patch
> Patch failed at 0001 event/dlb2: add CQ weight support
> When you have resolved this problem, run "git am --continue".
> If you prefer to skip this patch, run "git am --skip" instead.
> To restore the original branch and stop patching, run "git am --abort"
Please rebase these two patches and send the next version as SERIES as
these are interconnected.
https://patches.dpdk.org/project/dpdk/patch/20220410225602.1524724-1-timothy.mcdaniel@intel.com/
https://patches.dpdk.org/project/dpdk/patch/20220410224755.1524117-1-timothy.mcdaniel@intel.com/
>
> > ---
> > drivers/event/dlb2/dlb2.c | 249 ++++++++++++++-------
> > drivers/event/dlb2/dlb2_iface.c | 3 +
> > drivers/event/dlb2/dlb2_iface.h | 3 +
> > drivers/event/dlb2/dlb2_priv.h | 20 +-
> > drivers/event/dlb2/pf/base/dlb2_resource.c | 65 ++++++
> > drivers/event/dlb2/pf/dlb2_pf.c | 21 +-
> > 6 files changed, 278 insertions(+), 83 deletions(-)
> >
> > diff --git a/drivers/event/dlb2/dlb2.c b/drivers/event/dlb2/dlb2.c
> > index 9bac92c7b5..895dcb3550 100644
> > --- a/drivers/event/dlb2/dlb2.c
> > +++ b/drivers/event/dlb2/dlb2.c
> > @@ -116,6 +116,28 @@ dlb2_init_cq_weight(struct dlb2_eventdev *dlb2, int *cq_weight)
> > dlb2->ev_ports[q].cq_weight = cq_weight[q];
> > }
> >
> > +/* override defaults with value(s) provided on command line */
> > +static void
> > +dlb2_init_port_cos(struct dlb2_eventdev *dlb2, int *port_cos)
> > +{
> > + int q;
> > +
> > + for (q = 0; q < DLB2_MAX_NUM_PORTS_ALL; q++) {
> > + dlb2->ev_ports[q].cos_id = port_cos[q];
> > + dlb2->cos_ports[port_cos[q]]++;
> > + }
> > +}
> > +
> > +static void
> > +dlb2_init_cos_bw(struct dlb2_eventdev *dlb2,
> > + struct dlb2_cos_bw *cos_bw)
> > +{
> > + int q;
> > + for (q = 0; q < DLB2_COS_NUM_VALS; q++)
> > + dlb2->cos_bw[q] = cos_bw->val[q];
> > +
> > +}
> > +
> > static int
> > dlb2_hw_query_resources(struct dlb2_eventdev *dlb2)
> > {
> > @@ -330,36 +352,6 @@ set_dev_id(const char *key __rte_unused,
> > return 0;
> > }
> >
> > -static int
> > -set_cos(const char *key __rte_unused,
> > - const char *value,
> > - void *opaque)
> > -{
> > - enum dlb2_cos *cos_id = opaque;
> > - int x = 0;
> > - int ret;
> > -
> > - if (value == NULL || opaque == NULL) {
> > - DLB2_LOG_ERR("NULL pointer\n");
> > - return -EINVAL;
> > - }
> > -
> > - ret = dlb2_string_to_int(&x, value);
> > - if (ret < 0)
> > - return ret;
> > -
> > - if (x != DLB2_COS_DEFAULT && (x < DLB2_COS_0 || x > DLB2_COS_3)) {
> > - DLB2_LOG_ERR(
> > - "COS %d out of range, must be DLB2_COS_DEFAULT or 0-3\n",
> > - x);
> > - return -EINVAL;
> > - }
> > -
> > - *cos_id = x;
> > -
> > - return 0;
> > -}
> > -
> > static int
> > set_poll_interval(const char *key __rte_unused,
> > const char *value,
> > @@ -603,6 +595,80 @@ set_cq_weight(const char *key __rte_unused,
> > return 0;
> > }
> >
> > +static int
> > +set_port_cos(const char *key __rte_unused,
> > + const char *value,
> > + void *opaque)
> > +{
> > + struct dlb2_port_cos *port_cos = opaque;
> > + int first, last, cos_id, i;
> > +
> > + if (value == NULL || opaque == NULL) {
> > + DLB2_LOG_ERR("NULL pointer\n");
> > + return -EINVAL;
> > + }
> > +
> > + /* command line override may take one of the following 3 forms:
> > + * port_cos=all:<cos_id> ... all ports
> > + * port_cos=port-port:<cos_id> ... a range of ports
> > + * port_cos=port:<cos_id> ... just one port
> > + */
> > + if (sscanf(value, "all:%d", &cos_id) == 1) {
> > + first = 0;
> > + last = DLB2_MAX_NUM_LDB_PORTS - 1;
> > + } else if (sscanf(value, "%d-%d:%d", &first, &last, &cos_id) == 3) {
> > + /* we have everything we need */
> > + } else if (sscanf(value, "%d:%d", &first, &cos_id) == 2) {
> > + last = first;
> > + } else {
> > + DLB2_LOG_ERR("Error parsing ldb port port_cos devarg. Should be all:val, port-port:val, or port:val\n");
> > + return -EINVAL;
> > + }
> > +
> > + if (first > last || first < 0 ||
> > + last >= DLB2_MAX_NUM_LDB_PORTS) {
> > + DLB2_LOG_ERR("Error parsing ldb port cos_id arg, invalid port value\n");
> > + return -EINVAL;
> > + }
> > +
> > + if (cos_id < DLB2_COS_0 || cos_id > DLB2_COS_3) {
> > + DLB2_LOG_ERR("Error parsing ldb port cos_id devarg, must be between 0 and 4\n");
> > + return -EINVAL;
> > + }
> > +
> > + for (i = first; i <= last; i++)
> > + port_cos->cos_id[i] = cos_id; /* indexed by port */
> > +
> > + return 0;
> > +}
> > +
> > +static int
> > +set_cos_bw(const char *key __rte_unused,
> > + const char *value,
> > + void *opaque)
> > +{
> > + struct dlb2_cos_bw *cos_bw = opaque;
> > +
> > + if (opaque == NULL) {
> > + DLB2_LOG_ERR("NULL pointer\n");
> > + return -EINVAL;
> > + }
> > +
> > + /* format must be %d,%d,%d,%d */
> > +
> > + if (sscanf(value, "%d,%d,%d,%d", &cos_bw->val[0], &cos_bw->val[1],
> > + &cos_bw->val[2], &cos_bw->val[3]) != 4) {
> > + DLB2_LOG_ERR("Error parsing cos bandwidth devarg. Should be bw0,bw1,bw2,bw3 where all values combined are <= 100\n");
> > + return -EINVAL;
> > + }
> > + if (cos_bw->val[0] + cos_bw->val[1] + cos_bw->val[2] + cos_bw->val[4] > 100) {
> > + DLB2_LOG_ERR("Error parsing cos bandwidth devarg. Should be bw0,bw1,bw2,bw3 where all values combined are <= 100\n");
> > + return -EINVAL;
> > + }
> > +
> > + return 0;
> > +}
> > +
> > static void
> > dlb2_eventdev_info_get(struct rte_eventdev *dev,
> > struct rte_event_dev_info *dev_info)
> > @@ -647,11 +713,13 @@ dlb2_eventdev_info_get(struct rte_eventdev *dev,
> > }
> >
> > static int
> > -dlb2_hw_create_sched_domain(struct dlb2_hw_dev *handle,
> > +dlb2_hw_create_sched_domain(struct dlb2_eventdev *dlb2,
> > + struct dlb2_hw_dev *handle,
> > const struct dlb2_hw_rsrcs *resources_asked,
> > uint8_t device_version)
> > {
> > int ret = 0;
> > + uint32_t cos_ports = 0;
> > struct dlb2_create_sched_domain_args *cfg;
> >
> > if (resources_asked == NULL) {
> > @@ -677,38 +745,23 @@ dlb2_hw_create_sched_domain(struct dlb2_hw_dev *handle,
> >
> > /* LDB ports */
> >
> > - cfg->cos_strict = 0; /* Best effort */
> > - cfg->num_cos_ldb_ports[0] = 0;
> > - cfg->num_cos_ldb_ports[1] = 0;
> > - cfg->num_cos_ldb_ports[2] = 0;
> > - cfg->num_cos_ldb_ports[3] = 0;
> > -
> > - switch (handle->cos_id) {
> > - case DLB2_COS_0:
> > - cfg->num_ldb_ports = 0; /* no don't care ports */
> > - cfg->num_cos_ldb_ports[0] =
> > - resources_asked->num_ldb_ports;
> > - break;
> > - case DLB2_COS_1:
> > - cfg->num_ldb_ports = 0; /* no don't care ports */
> > - cfg->num_cos_ldb_ports[1] = resources_asked->num_ldb_ports;
> > - break;
> > - case DLB2_COS_2:
> > - cfg->num_ldb_ports = 0; /* no don't care ports */
> > - cfg->num_cos_ldb_ports[2] = resources_asked->num_ldb_ports;
> > - break;
> > - case DLB2_COS_3:
> > - cfg->num_ldb_ports = 0; /* no don't care ports */
> > - cfg->num_cos_ldb_ports[3] =
> > - resources_asked->num_ldb_ports;
> > - break;
> > - case DLB2_COS_DEFAULT:
> > - /* all ldb ports are don't care ports from a cos perspective */
> > - cfg->num_ldb_ports =
> > - resources_asked->num_ldb_ports;
> > - break;
> > + /* tally of ports with non default COS */
> > + cos_ports = dlb2->cos_ports[1] + dlb2->cos_ports[2] +
> > + dlb2->cos_ports[3];
> > +
> > + if (cos_ports > resources_asked->num_ldb_ports) {
> > + DLB2_LOG_ERR("dlb2: num_ldb_ports < nonzero cos_ports\n");
> > + ret = EINVAL;
> > + goto error_exit;
> > }
> >
> > + cfg->cos_strict = 0; /* Best effort */
> > + cfg->num_cos_ldb_ports[0] = resources_asked->num_ldb_ports - cos_ports;
> > + cfg->num_cos_ldb_ports[1] = dlb2->cos_ports[1];
> > + cfg->num_cos_ldb_ports[2] = dlb2->cos_ports[2];
> > + cfg->num_cos_ldb_ports[3] = dlb2->cos_ports[3];
> > +
> > +
> > if (device_version == DLB2_HW_V2)
> > cfg->num_ldb_credits = resources_asked->num_ldb_credits;
> >
> > @@ -886,7 +939,8 @@ dlb2_eventdev_configure(const struct rte_eventdev *dev)
> > rsrcs->num_dir_credits = dlb2->num_dir_credits_override;
> > }
> >
> > - if (dlb2_hw_create_sched_domain(handle, rsrcs, dlb2->version) < 0) {
> > + if (dlb2_hw_create_sched_domain(dlb2, handle, rsrcs,
> > + dlb2->version) < 0) {
> > DLB2_LOG_ERR("dlb2_hw_create_sched_domain failed\n");
> > return -ENODEV;
> > }
> > @@ -1449,12 +1503,8 @@ dlb2_hw_create_ldb_port(struct dlb2_eventdev *dlb2,
> >
> > cfg.cq_history_list_size = DLB2_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
> >
> > - if (handle->cos_id == DLB2_COS_DEFAULT)
> > - cfg.cos_id = 0;
> > - else
> > - cfg.cos_id = handle->cos_id;
> > -
> > - cfg.cos_strict = 0;
> > + cfg.cos_id = ev_port->cos_id;
> > + cfg.cos_strict = 0;/* best effots */
> >
> > /* User controls the LDB high watermark via enqueue depth. The DIR high
> > * watermark is equal, unless the directed credit pool is too small.
> > @@ -4668,7 +4718,6 @@ dlb2_primary_eventdev_probe(struct rte_eventdev *dev,
> >
> > dlb2->max_num_events_override = dlb2_args->max_num_events;
> > dlb2->num_dir_credits_override = dlb2_args->num_dir_credits_override;
> > - dlb2->qm_instance.cos_id = dlb2_args->cos_id;
> > dlb2->poll_interval = dlb2_args->poll_interval;
> > dlb2->sw_credit_quanta = dlb2_args->sw_credit_quanta;
> > dlb2->hw_credit_quanta = dlb2_args->hw_credit_quanta;
> > @@ -4700,6 +4749,27 @@ dlb2_primary_eventdev_probe(struct rte_eventdev *dev,
> >
> > dlb2_iface_hardware_init(&dlb2->qm_instance);
> >
> > + /* configure class of service */
> > + {
> > + struct dlb2_set_cos_bw_args set_cos_bw_args = {0};
> > + int id;
> > + int ret = 0;
> > +
> > + for (id = 0; id < DLB2_COS_NUM_VALS; id++) {
> > + set_cos_bw_args.cos_id = id;
> > + set_cos_bw_args.cos_id = dlb2->cos_bw[id];
> > + ret = dlb2_iface_set_cos_bw(&dlb2->qm_instance,
> > + &set_cos_bw_args);
> > + if (ret != 0)
> > + break;
> > + }
> > + if (ret) {
> > + DLB2_LOG_ERR("dlb2: failed to configure class of service, err=%d\n",
> > + err);
> > + return err;
> > + }
> > + }
> > +
> > err = dlb2_iface_get_cq_poll_mode(&dlb2->qm_instance, &dlb2->poll_mode);
> > if (err < 0) {
> > DLB2_LOG_ERR("dlb2: failed to get the poll mode, err=%d\n",
> > @@ -4730,6 +4800,12 @@ dlb2_primary_eventdev_probe(struct rte_eventdev *dev,
> > dlb2_init_cq_weight(dlb2,
> > dlb2_args->cq_weight.limit);
> >
> > + dlb2_init_port_cos(dlb2,
> > + dlb2_args->port_cos.cos_id);
> > +
> > + dlb2_init_cos_bw(dlb2,
> > + &dlb2_args->cos_bw);
> > +
> > return 0;
> > }
> >
> > @@ -4785,6 +4861,8 @@ dlb2_parse_params(const char *params,
> > DLB2_VECTOR_OPTS_ENAB_ARG,
> > DLB2_MAX_CQ_DEPTH,
> > DLB2_CQ_WEIGHT,
> > + DLB2_PORT_COS,
> > + DLB2_COS_BW,
> > NULL };
> >
> > if (params != NULL && params[0] != '\0') {
> > @@ -4857,16 +4935,6 @@ dlb2_parse_params(const char *params,
> > return ret;
> > }
> >
> > - ret = rte_kvargs_process(kvlist, DLB2_COS_ARG,
> > - set_cos,
> > - &dlb2_args->cos_id);
> > - if (ret != 0) {
> > - DLB2_LOG_ERR("%s: Error parsing cos parameter",
> > - name);
> > - rte_kvargs_free(kvlist);
> > - return ret;
> > - }
> > -
> > ret = rte_kvargs_process(kvlist, DLB2_POLL_INTERVAL_ARG,
> > set_poll_interval,
> > &dlb2_args->poll_interval);
> > @@ -4942,6 +5010,29 @@ dlb2_parse_params(const char *params,
> > return ret;
> > }
> >
> > + ret = rte_kvargs_process(kvlist,
> > + DLB2_PORT_COS,
> > + set_port_cos,
> > + &dlb2_args->port_cos);
> > + if (ret != 0) {
> > + DLB2_LOG_ERR("%s: Error parsing port cos",
> > + name);
> > + rte_kvargs_free(kvlist);
> > + return ret;
> > + }
> > +
> > + ret = rte_kvargs_process(kvlist,
> > + DLB2_COS_BW,
> > + set_cos_bw,
> > + &dlb2_args->cos_bw);
> > + if (ret != 0) {
> > + DLB2_LOG_ERR("%s: Error parsing cos_bw",
> > + name);
> > + rte_kvargs_free(kvlist);
> > + return ret;
> > + }
> > +
> > +
> > rte_kvargs_free(kvlist);
> > }
> > }
> > diff --git a/drivers/event/dlb2/dlb2_iface.c b/drivers/event/dlb2/dlb2_iface.c
> > index 7913525e0f..2d4c5f617b 100644
> > --- a/drivers/event/dlb2/dlb2_iface.c
> > +++ b/drivers/event/dlb2/dlb2_iface.c
> > @@ -76,3 +76,6 @@ int (*dlb2_iface_get_dir_queue_depth)(struct dlb2_hw_dev *handle,
> > int (*dlb2_iface_enable_cq_weight)(struct dlb2_hw_dev *handle,
> > struct dlb2_enable_cq_weight_args *args);
> >
> > +int (*dlb2_iface_set_cos_bw)(struct dlb2_hw_dev *handle,
> > + struct dlb2_set_cos_bw_args *args);
> > +
> > diff --git a/drivers/event/dlb2/dlb2_iface.h b/drivers/event/dlb2/dlb2_iface.h
> > index 3aace0ba19..02ed99a9a7 100644
> > --- a/drivers/event/dlb2/dlb2_iface.h
> > +++ b/drivers/event/dlb2/dlb2_iface.h
> > @@ -75,4 +75,7 @@ extern int (*dlb2_iface_get_dir_queue_depth)(struct dlb2_hw_dev *handle,
> > extern int (*dlb2_iface_enable_cq_weight)(struct dlb2_hw_dev *handle,
> > struct dlb2_enable_cq_weight_args *args);
> >
> > +extern int (*dlb2_iface_set_cos_bw)(struct dlb2_hw_dev *handle,
> > + struct dlb2_set_cos_bw_args *args);
> > +
> > #endif /* _DLB2_IFACE_H_ */
> > diff --git a/drivers/event/dlb2/dlb2_priv.h b/drivers/event/dlb2/dlb2_priv.h
> > index 6084fc4183..20bf299563 100644
> > --- a/drivers/event/dlb2/dlb2_priv.h
> > +++ b/drivers/event/dlb2/dlb2_priv.h
> > @@ -45,6 +45,8 @@
> > #define DLB2_VECTOR_OPTS_ENAB_ARG "vector_opts_enable"
> > #define DLB2_MAX_CQ_DEPTH "max_cq_depth"
> > #define DLB2_CQ_WEIGHT "cq_weight"
> > +#define DLB2_PORT_COS "port_cos"
> > +#define DLB2_COS_BW "cos_bw"
> >
> > /* Begin HW related defines and structs */
> >
> > @@ -415,7 +417,8 @@ enum dlb2_cos {
> > DLB2_COS_0 = 0,
> > DLB2_COS_1,
> > DLB2_COS_2,
> > - DLB2_COS_3
> > + DLB2_COS_3,
> > + DLB2_COS_NUM_VALS
> > };
> >
> > struct dlb2_hw_dev {
> > @@ -423,7 +426,6 @@ struct dlb2_hw_dev {
> > struct dlb2_hw_resource_info info;
> > void *pf_dev; /* opaque pointer to PF PMD dev (struct dlb2_dev) */
> > uint32_t domain_id;
> > - enum dlb2_cos cos_id;
> > rte_spinlock_t resource_lock; /* for MP support */
> > } __rte_cache_aligned;
> >
> > @@ -527,6 +529,7 @@ struct dlb2_eventdev_port {
> > bool enq_configured;
> > uint8_t implicit_release; /* release events before dequeuing */
> > uint32_t cq_weight; /* DLB2.5 and above ldb ports only */
> > + int cos_id; /*ldb port class of service */
> > } __rte_cache_aligned;
> >
> > struct dlb2_queue {
> > @@ -621,6 +624,8 @@ struct dlb2_eventdev {
> > uint32_t credit_pool __rte_cache_aligned;
> > };
> > };
> > + uint32_t cos_ports[DLB2_COS_NUM_VALS]; /* total ldb ports in each class */
> > + uint32_t cos_bw[DLB2_COS_NUM_VALS]; /* bandwidth per cos domain */
> > };
> >
> > /* used for collecting and passing around the dev args */
> > @@ -632,13 +637,20 @@ struct dlb2_cq_weight {
> > int limit[DLB2_MAX_NUM_LDB_PORTS];
> > };
> >
> > +struct dlb2_port_cos {
> > + int cos_id[DLB2_MAX_NUM_LDB_PORTS];
> > +};
> > +
> > +struct dlb2_cos_bw {
> > + int val[DLB2_COS_NUM_VALS];
> > +};
> > +
> > struct dlb2_devargs {
> > int socket_id;
> > int max_num_events;
> > int num_dir_credits_override;
> > int dev_id;
> > struct dlb2_qid_depth_thresholds qid_depth_thresholds;
> > - enum dlb2_cos cos_id;
> > int poll_interval;
> > int sw_credit_quanta;
> > int hw_credit_quanta;
> > @@ -646,6 +658,8 @@ struct dlb2_devargs {
> > bool vector_opts_enabled;
> > int max_cq_depth;
> > struct dlb2_cq_weight cq_weight;
> > + struct dlb2_port_cos port_cos;
> > + struct dlb2_cos_bw cos_bw;
> > };
> >
> > /* End Eventdev related defines and structs */
> > diff --git a/drivers/event/dlb2/pf/base/dlb2_resource.c b/drivers/event/dlb2/pf/base/dlb2_resource.c
> > index 54222d732c..cb295bab2f 100644
> > --- a/drivers/event/dlb2/pf/base/dlb2_resource.c
> > +++ b/drivers/event/dlb2/pf/base/dlb2_resource.c
> > @@ -6467,3 +6467,68 @@ int dlb2_hw_enable_cq_weight(struct dlb2_hw *hw,
> > return 0;
> > }
> >
> > +static void dlb2_log_set_cos_bandwidth(struct dlb2_hw *hw, u32 cos_id, u8 bw)
> > +{
> > + DLB2_HW_DBG(hw, "DLB2 set port CoS bandwidth:\n");
> > + DLB2_HW_DBG(hw, "\tCoS ID: %u\n", cos_id);
> > + DLB2_HW_DBG(hw, "\tBandwidth: %u\n", bw);
> > +}
> > +
> > +#define DLB2_MAX_BW_PCT 100
> > +
> > +/**
> > + * dlb2_hw_set_cos_bandwidth() - set a bandwidth allocation percentage for a
> > + * port class-of-service.
> > + * @hw: dlb2_hw handle for a particular device.
> > + * @cos_id: class-of-service ID.
> > + * @bandwidth: class-of-service bandwidth.
> > + *
> > + * Return:
> > + * Returns 0 upon success, < 0 otherwise.
> > + *
> > + * Errors:
> > + * EINVAL - Invalid cos ID, bandwidth is greater than 100, or bandwidth would
> > + * cause the total bandwidth across all classes of service to exceed
> > + * 100%.
> > + */
> > +int dlb2_hw_set_cos_bandwidth(struct dlb2_hw *hw, u32 cos_id, u8 bandwidth)
> > +{
> > + unsigned int i;
> > + u32 reg;
> > + u8 total;
> > +
> > + if (cos_id >= DLB2_NUM_COS_DOMAINS)
> > + return -EINVAL;
> > +
> > + if (bandwidth > DLB2_MAX_BW_PCT)
> > + return -EINVAL;
> > +
> > + total = 0;
> > +
> > + for (i = 0; i < DLB2_NUM_COS_DOMAINS; i++)
> > + total += (i == cos_id) ? bandwidth : hw->cos_reservation[i];
> > +
> > + if (total > DLB2_MAX_BW_PCT)
> > + return -EINVAL;
> > +
> > + reg = DLB2_CSR_RD(hw, DLB2_LSP_CFG_SHDW_RANGE_COS(hw->ver, cos_id));
> > +
> > + /*
> > + * Normalize the bandwidth to a value in the range 0-255. Integer
> > + * division may leave unreserved scheduling slots; these will be
> > + * divided among the 4 classes of service.
> > + */
> > + DLB2_BITS_SET(reg, (bandwidth * 256) / 100, DLB2_LSP_CFG_SHDW_RANGE_COS_BW_RANGE);
> > + DLB2_CSR_WR(hw, DLB2_LSP_CFG_SHDW_RANGE_COS(hw->ver, cos_id), reg);
> > +
> > + reg = 0;
> > + DLB2_BIT_SET(reg, DLB2_LSP_CFG_SHDW_CTRL_TRANSFER);
> > + /* Atomically transfer the newly configured service weight */
> > + DLB2_CSR_WR(hw, DLB2_LSP_CFG_SHDW_CTRL(hw->ver), reg);
> > +
> > + dlb2_log_set_cos_bandwidth(hw, cos_id, bandwidth);
> > +
> > + hw->cos_reservation[cos_id] = bandwidth;
> > +
> > + return 0;
> > +}
> > diff --git a/drivers/event/dlb2/pf/dlb2_pf.c b/drivers/event/dlb2/pf/dlb2_pf.c
> > index 10918ff281..244c1591f8 100644
> > --- a/drivers/event/dlb2/pf/dlb2_pf.c
> > +++ b/drivers/event/dlb2/pf/dlb2_pf.c
> > @@ -645,6 +645,25 @@ dlb2_pf_enable_cq_weight(struct dlb2_hw_dev *handle,
> > return ret;
> > }
> >
> > +static int
> > +dlb2_pf_set_cos_bandwidth(struct dlb2_hw_dev *handle,
> > + struct dlb2_set_cos_bw_args *args)
> > +{
> > + struct dlb2_dev *dlb2_dev = (struct dlb2_dev *)handle->pf_dev;
> > + int ret = 0;
> > +
> > + DLB2_INFO(dev->dlb2_device, "Entering %s()\n", __func__);
> > +
> > + ret = dlb2_hw_set_cos_bandwidth(&dlb2_dev->hw,
> > + args->cos_id,
> > + args->bandwidth);
> > +
> > + DLB2_INFO(dev->dlb2_device, "Exiting %s() with ret=%d\n",
> > + __func__, ret);
> > +
> > + return ret;
> > +}
> > +
> > static void
> > dlb2_pf_iface_fn_ptrs_init(void)
> > {
> > @@ -670,6 +689,7 @@ dlb2_pf_iface_fn_ptrs_init(void)
> > dlb2_iface_set_sn_allocation = dlb2_pf_set_sn_allocation;
> > dlb2_iface_get_sn_occupancy = dlb2_pf_get_sn_occupancy;
> > dlb2_iface_enable_cq_weight = dlb2_pf_enable_cq_weight;
> > + dlb2_iface_set_cos_bw = dlb2_pf_set_cos_bandwidth;
> > }
> >
> > /* PCI DEV HOOKS */
> > @@ -683,7 +703,6 @@ dlb2_eventdev_pci_init(struct rte_eventdev *eventdev)
> > .max_num_events = DLB2_MAX_NUM_LDB_CREDITS,
> > .num_dir_credits_override = -1,
> > .qid_depth_thresholds = { {0} },
> > - .cos_id = DLB2_COS_DEFAULT,
> > .poll_interval = DLB2_POLL_INTERVAL_DEFAULT,
> > .sw_credit_quanta = DLB2_SW_CREDIT_QUANTA_DEFAULT,
> > .hw_credit_quanta = DLB2_SW_CREDIT_BATCH_SZ,
> > --
> > 2.23.0
> >
^ permalink raw reply [flat|nested] 3+ messages in thread
end of thread, other threads:[~2022-06-14 10:45 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2022-04-10 22:56 [PATCH] event/dlb2: add ldb port specific COS support Timothy McDaniel
2022-06-14 10:42 ` Jerin Jacob
2022-06-14 10:45 ` Jerin Jacob
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).