From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id EEBBE2C4F for ; Sun, 16 Sep 2018 15:43:07 +0200 (CEST) Received: from Internal Mail-Server by MTLPINE1 (envelope-from dekelp@mellanox.com) with ESMTPS (AES256-SHA encrypted); 16 Sep 2018 15:47:27 +0200 Received: from mtl-vdi-280.wap.labs.mlnx. (mtl-vdi-280.wap.labs.mlnx [10.128.130.87]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id w8GDh7wB026324; Sun, 16 Sep 2018 16:43:07 +0300 From: Dekel Peled To: dev@dpdk.org, shahafs@mellanox.com, yskoh@mellanox.com Cc: orika@mellanox.com Date: Sun, 16 Sep 2018 16:42:08 +0300 Message-Id: <1537105328-9367-1-git-send-email-dekelp@mellanox.com> X-Mailer: git-send-email 1.7.1 Subject: [dpdk-dev] [PATCH] net/mlx5: support metadata as flow rule criteria X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 16 Sep 2018 13:43:08 -0000 As described in series starting at [1], it adds option to set metadata value as match pattern when creating a new flow rule. This patch adds metadata support in mlx5 driver, in several parts: - Add the setting of metadata value in matcher when creating a new flow rule. - Add the passing of metadata value from mbuf to wqe when indicated by ol_flag, in different burst functions. - Allow flow rule with attribute egress in specific cases. This patch must be built together with the files in series [1]. [1] "ethdev: support metadata as flow rule criteria" Signed-off-by: Dekel Peled --- drivers/net/mlx5/mlx5_flow.c | 60 +++++++++++++---- drivers/net/mlx5/mlx5_flow.h | 22 +++++-- drivers/net/mlx5/mlx5_flow_dv.c | 120 ++++++++++++++++++++++++++++++---- drivers/net/mlx5/mlx5_flow_verbs.c | 21 ++++-- drivers/net/mlx5/mlx5_prm.h | 2 +- drivers/net/mlx5/mlx5_rxtx.c | 34 ++++++++-- drivers/net/mlx5/mlx5_rxtx_vec.c | 28 ++++++-- drivers/net/mlx5/mlx5_rxtx_vec.h | 1 + drivers/net/mlx5/mlx5_rxtx_vec_neon.h | 4 +- drivers/net/mlx5/mlx5_rxtx_vec_sse.h | 4 +- drivers/net/mlx5/mlx5_txq.c | 6 ++ 11 files changed, 255 insertions(+), 47 deletions(-) diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index 4234be6..7932e0f 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -402,7 +402,7 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, * @return * 0 on success, a negative errno value otherwise and rte_errno is set. */ -static int +int mlx5_flow_item_acceptable(const struct rte_flow_item *item, const uint8_t *mask, const uint8_t *nic_mask, @@ -602,7 +602,8 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, * 0 on success, a negative errno value otherwise and rte_ernno is set. */ int mlx5_flow_validate_action_flag(uint64_t action_flags, - struct rte_flow_error *error) + const struct rte_flow_attr *attr, + struct rte_flow_error *error) { if (action_flags & MLX5_ACTION_DROP) @@ -624,6 +625,11 @@ int mlx5_flow_validate_action_flag(uint64_t action_flags, NULL, "can't have 2 flag" " actions in same flow"); + if (attr->egress) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, + NULL, + "flag action not supported for egress"); return 0; } @@ -642,8 +648,9 @@ int mlx5_flow_validate_action_flag(uint64_t action_flags, * 0 on success, a negative errno value otherwise and rte_ernno is set. */ int mlx5_flow_validate_action_mark(uint64_t action_flags, - const struct rte_flow_action *action, - struct rte_flow_error *error) + const struct rte_flow_action *action, + const struct rte_flow_attr *attr, + struct rte_flow_error *error) { const struct rte_flow_action_mark *mark = action->conf; @@ -677,6 +684,11 @@ int mlx5_flow_validate_action_mark(uint64_t action_flags, NULL, "can't have 2 flag actions in same" " flow"); + if (attr->egress) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, + NULL, + "mark action not supported for egress"); return 0; } @@ -693,7 +705,8 @@ int mlx5_flow_validate_action_mark(uint64_t action_flags, * 0 on success, a negative errno value otherwise and rte_ernno is set. */ int mlx5_flow_validate_action_drop(uint64_t action_flags, - struct rte_flow_error *error) + const struct rte_flow_attr *attr, + struct rte_flow_error *error) { if (action_flags & MLX5_ACTION_FLAG) return rte_flow_error_set(error, @@ -715,6 +728,11 @@ int mlx5_flow_validate_action_drop(uint64_t action_flags, NULL, "can't have 2 fate actions in" " same flow"); + if (attr->egress) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, + NULL, + "drop action not supported for egress"); return 0; } @@ -735,9 +753,10 @@ int mlx5_flow_validate_action_drop(uint64_t action_flags, * 0 on success, a negative errno value otherwise and rte_ernno is set. */ int mlx5_flow_validate_action_queue(uint64_t action_flags, - struct rte_eth_dev *dev, - const struct rte_flow_action *action, - struct rte_flow_error *error) + struct rte_eth_dev *dev, + const struct rte_flow_action *action, + const struct rte_flow_attr *attr, + struct rte_flow_error *error) { struct priv *priv = dev->data->dev_private; const struct rte_flow_action_queue *queue = action->conf; @@ -760,6 +779,11 @@ int mlx5_flow_validate_action_queue(uint64_t action_flags, RTE_FLOW_ERROR_TYPE_ACTION_CONF, &queue->index, "queue is not configured"); + if (attr->egress) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, + NULL, + "queue action not supported for egress"); return 0; } @@ -780,9 +804,10 @@ int mlx5_flow_validate_action_queue(uint64_t action_flags, * 0 on success, a negative errno value otherwise and rte_ernno is set. */ int mlx5_flow_validate_action_rss(uint64_t action_flags, - struct rte_eth_dev *dev, - const struct rte_flow_action *action, - struct rte_flow_error *error) + struct rte_eth_dev *dev, + const struct rte_flow_action *action, + const struct rte_flow_attr *attr, + struct rte_flow_error *error) { struct priv *priv = dev->data->dev_private; const struct rte_flow_action_rss *rss = action->conf; @@ -839,6 +864,11 @@ int mlx5_flow_validate_action_rss(uint64_t action_flags, &rss->queue[i], "queue is not configured"); } + if (attr->egress) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, + NULL, + "rss action not supported for egress"); return 0; } @@ -855,7 +885,8 @@ int mlx5_flow_validate_action_rss(uint64_t action_flags, * 0 on success, a negative errno value otherwise and rte_ernno is set. */ int mlx5_flow_validate_action_count(struct rte_eth_dev *dev, - struct rte_flow_error *error) + const struct rte_flow_attr *attr, + struct rte_flow_error *error) { struct priv *priv = dev->data->dev_private; @@ -864,6 +895,11 @@ int mlx5_flow_validate_action_count(struct rte_eth_dev *dev, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "flow counters are not supported."); + if (attr->egress) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, + NULL, + "count action not supported for egress"); return 0; } diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index f1a72d4..e0446c5 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -43,6 +43,9 @@ #define MLX5_FLOW_LAYER_GRE (1u << 14) #define MLX5_FLOW_LAYER_MPLS (1u << 15) +/* General pattern items bits. */ +#define MLX5_FLOW_ITEM_METADATA (1u << 16) + /* Outer Masks. */ #define MLX5_FLOW_LAYER_OUTER_L3 \ (MLX5_FLOW_LAYER_OUTER_L3_IPV4 | MLX5_FLOW_LAYER_OUTER_L3_IPV6) @@ -237,21 +240,27 @@ struct mlx5_flow_driver_ops { /* mlx5_flow.c */ int mlx5_flow_validate_action_flag(uint64_t action_flags, - struct rte_flow_error *error); + const struct rte_flow_attr *attr, + struct rte_flow_error *error); int mlx5_flow_validate_action_mark(uint64_t action_flags, - const struct rte_flow_action *action, - struct rte_flow_error *error); + const struct rte_flow_action *action, + const struct rte_flow_attr *attr, + struct rte_flow_error *error); int mlx5_flow_validate_action_drop(uint64_t action_flags, - struct rte_flow_error *error); + const struct rte_flow_attr *attr, + struct rte_flow_error *error); int mlx5_flow_validate_action_queue(uint64_t action_flags, struct rte_eth_dev *dev, const struct rte_flow_action *action, + const struct rte_flow_attr *attr, struct rte_flow_error *error); int mlx5_flow_validate_action_rss(uint64_t action_flags, struct rte_eth_dev *dev, const struct rte_flow_action *action, + const struct rte_flow_attr *attr, struct rte_flow_error *error); int mlx5_flow_validate_action_count(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, struct rte_flow_error *error); int mlx5_flow_validate_attributes(struct rte_eth_dev *dev, const struct rte_flow_attr *attributes, @@ -294,6 +303,11 @@ int mlx5_flow_validate_item_mpls(uint64_t item_flags, void mlx5_flow_init_driver_ops(struct rte_eth_dev *dev); uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, uint32_t subpriority); +int mlx5_flow_item_acceptable(const struct rte_flow_item *item, + const uint8_t *mask, + const uint8_t *nic_mask, + unsigned int size, + struct rte_flow_error *error); /* mlx5_flow_dv.c */ void mlx5_flow_dv_get_driver_ops(struct mlx5_flow_driver_ops *flow_ops); diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 4090c5f..3b52181 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -37,6 +37,47 @@ #ifdef HAVE_IBV_FLOW_DV_SUPPORT /** + * Validate META item. + * + * @param[in] item + * Item specification. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +mlx5_flow_validate_item_meta(const struct rte_flow_item *item, + const struct rte_flow_attr *attributes, + struct rte_flow_error *error) +{ + const struct rte_flow_item_meta *mask = item->mask; + + const struct rte_flow_item_meta nic_mask = { + .data = RTE_BE32(UINT32_MAX) + }; + + int ret; + + if (!mask) + mask = &rte_flow_item_meta_mask; + ret = mlx5_flow_item_acceptable + (item, (const uint8_t *)mask, + (const uint8_t *)&nic_mask, + sizeof(struct rte_flow_item_meta), error); + if (ret < 0) + return ret; + + if (attributes->ingress) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, + NULL, + "pattern not supported for ingress"); + return 0; +} + +/** * Verify the @p attributes will be correctly understood by the NIC and store * them in the @p flow if everything is correct. * @@ -68,21 +109,17 @@ static int flow_dv_validate_attributes(struct rte_eth_dev *dev, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, NULL, "priority out of range"); - if (attributes->egress) - return rte_flow_error_set(error, ENOTSUP, - RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, - NULL, - "egress is not supported"); if (attributes->transfer) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, NULL, "transfer is not supported"); - if (!attributes->ingress) + if (!(attributes->egress ^ attributes->ingress)) return rte_flow_error_set(error, ENOTSUP, - RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, + RTE_FLOW_ERROR_TYPE_ATTR, NULL, - "ingress attribute is mandatory"); + "must specify exactly one of " + "ingress or egress"); return 0; } @@ -219,6 +256,12 @@ static int flow_dv_validate_attributes(struct rte_eth_dev *dev, return ret; item_flags |= MLX5_FLOW_LAYER_MPLS; break; + case RTE_FLOW_ITEM_TYPE_META: + ret = mlx5_flow_validate_item_meta(items, attr, error); + if (ret < 0) + return ret; + item_flags |= MLX5_FLOW_ITEM_METADATA; + break; default: return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, @@ -233,6 +276,7 @@ static int flow_dv_validate_attributes(struct rte_eth_dev *dev, break; case RTE_FLOW_ACTION_TYPE_FLAG: ret = mlx5_flow_validate_action_flag(action_flags, + attr, error); if (ret < 0) return ret; @@ -241,6 +285,7 @@ static int flow_dv_validate_attributes(struct rte_eth_dev *dev, case RTE_FLOW_ACTION_TYPE_MARK: ret = mlx5_flow_validate_action_mark(action_flags, actions, + attr, error); if (ret < 0) return ret; @@ -248,27 +293,36 @@ static int flow_dv_validate_attributes(struct rte_eth_dev *dev, break; case RTE_FLOW_ACTION_TYPE_DROP: ret = mlx5_flow_validate_action_drop(action_flags, + attr, error); if (ret < 0) return ret; action_flags |= MLX5_ACTION_DROP; break; case RTE_FLOW_ACTION_TYPE_QUEUE: - ret = mlx5_flow_validate_action_queue(action_flags, dev, - actions, error); + ret = mlx5_flow_validate_action_queue(action_flags, + dev, + actions, + attr, + error); if (ret < 0) return ret; action_flags |= MLX5_ACTION_QUEUE; break; case RTE_FLOW_ACTION_TYPE_RSS: - ret = mlx5_flow_validate_action_rss(action_flags, dev, - actions, error); + ret = mlx5_flow_validate_action_rss(action_flags, + dev, + actions, + attr, + error); if (ret < 0) return ret; action_flags |= MLX5_ACTION_RSS; break; case RTE_FLOW_ACTION_TYPE_COUNT: - ret = mlx5_flow_validate_action_count(dev, error); + ret = mlx5_flow_validate_action_count(dev, + attr, + error); if (ret < 0) return ret; action_flags |= MLX5_ACTION_COUNT; @@ -865,6 +919,43 @@ static int flow_dv_validate_attributes(struct rte_eth_dev *dev, } /** + * Add META item to matcher + * + * @param[in, out] matcher + * Flow matcher. + * @param[in, out] key + * Flow matcher value. + * @param[in] item + * Flow pattern to translate. + * @param[in] inner + * Item is inner pattern. + */ +static void +flow_dv_translate_item_meta(void *matcher, void *key, + const struct rte_flow_item *item) +{ + const struct rte_flow_item_meta *metam; + const struct rte_flow_item_meta *metav; + + void *misc2_m = + MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2); + void *misc2_v = + MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2); + + metam = (const void *)item->mask; + if (!metam) + metam = &rte_flow_item_meta_mask; + + metav = (const void *)item->spec; + if (metav) { + MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, + RTE_BE32((uint32_t)rte_be_to_cpu_64(metam->data))); + MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, + RTE_BE32((uint32_t)rte_be_to_cpu_64(metav->data))); + } +} + +/** * * Translate flow item. * @@ -946,6 +1037,9 @@ static int flow_dv_validate_attributes(struct rte_eth_dev *dev, case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: flow_dv_translate_item_vxlan(tmatcher->key, key, item, inner); break; + case RTE_FLOW_ITEM_TYPE_META: + flow_dv_translate_item_meta(tmatcher->key, key, item); + break; case RTE_FLOW_ITEM_TYPE_ICMP: case RTE_FLOW_ITEM_TYPE_MPLS: case RTE_FLOW_ITEM_TYPE_SCTP: diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c index 48e816d..3c42016 100644 --- a/drivers/net/mlx5/mlx5_flow_verbs.c +++ b/drivers/net/mlx5/mlx5_flow_verbs.c @@ -1272,6 +1272,7 @@ struct ibv_spec_header { break; case RTE_FLOW_ACTION_TYPE_FLAG: ret = mlx5_flow_validate_action_flag(action_flags, + attr, error); if (ret < 0) return ret; @@ -1280,6 +1281,7 @@ struct ibv_spec_header { case RTE_FLOW_ACTION_TYPE_MARK: ret = mlx5_flow_validate_action_mark(action_flags, actions, + attr, error); if (ret < 0) return ret; @@ -1287,27 +1289,36 @@ struct ibv_spec_header { break; case RTE_FLOW_ACTION_TYPE_DROP: ret = mlx5_flow_validate_action_drop(action_flags, + attr, error); if (ret < 0) return ret; action_flags |= MLX5_ACTION_DROP; break; case RTE_FLOW_ACTION_TYPE_QUEUE: - ret = mlx5_flow_validate_action_queue(action_flags, dev, - actions, error); + ret = mlx5_flow_validate_action_queue(action_flags, + dev, + actions, + attr, + error); if (ret < 0) return ret; action_flags |= MLX5_ACTION_QUEUE; break; case RTE_FLOW_ACTION_TYPE_RSS: - ret = mlx5_flow_validate_action_rss(action_flags, dev, - actions, error); + ret = mlx5_flow_validate_action_rss(action_flags, + dev, + actions, + attr, + error); if (ret < 0) return ret; action_flags |= MLX5_ACTION_RSS; break; case RTE_FLOW_ACTION_TYPE_COUNT: - ret = mlx5_flow_validate_action_count(dev, error); + ret = mlx5_flow_validate_action_count(dev, + attr, + error); if (ret < 0) return ret; action_flags |= MLX5_ACTION_COUNT; diff --git a/drivers/net/mlx5/mlx5_prm.h b/drivers/net/mlx5/mlx5_prm.h index 117cec7..2f33aef 100644 --- a/drivers/net/mlx5/mlx5_prm.h +++ b/drivers/net/mlx5/mlx5_prm.h @@ -159,7 +159,7 @@ struct mlx5_wqe_eth_seg_small { uint8_t cs_flags; uint8_t rsvd1; uint16_t mss; - uint32_t rsvd2; + uint32_t flow_table_metadata; uint16_t inline_hdr_sz; uint8_t inline_hdr[2]; } __rte_aligned(MLX5_WQE_DWORD_SIZE); diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c index 2d14f8a..080de57 100644 --- a/drivers/net/mlx5/mlx5_rxtx.c +++ b/drivers/net/mlx5/mlx5_rxtx.c @@ -523,6 +523,7 @@ uint8_t tso = txq->tso_en && (buf->ol_flags & PKT_TX_TCP_SEG); uint32_t swp_offsets = 0; uint8_t swp_types = 0; + uint32_t metadata = 0; uint16_t tso_segsz = 0; #ifdef MLX5_PMD_SOFT_COUNTERS uint32_t total_length = 0; @@ -566,6 +567,9 @@ cs_flags = txq_ol_cksum_to_cs(buf); txq_mbuf_to_swp(txq, buf, (uint8_t *)&swp_offsets, &swp_types); raw = ((uint8_t *)(uintptr_t)wqe) + 2 * MLX5_WQE_DWORD_SIZE; + /* Copy metadata from mbuf if valid */ + if (buf->ol_flags & PKT_TX_METADATA) + metadata = buf->hash.fdir.hi; /* Replace the Ethernet type by the VLAN if necessary. */ if (buf->ol_flags & PKT_TX_VLAN_PKT) { uint32_t vlan = rte_cpu_to_be_32(0x81000000 | @@ -781,7 +785,7 @@ swp_offsets, cs_flags | (swp_types << 8) | (rte_cpu_to_be_16(tso_segsz) << 16), - 0, + rte_cpu_to_be_32(metadata), (ehdr << 16) | rte_cpu_to_be_16(tso_header_sz), }; } else { @@ -795,7 +799,7 @@ wqe->eseg = (rte_v128u32_t){ swp_offsets, cs_flags | (swp_types << 8), - 0, + rte_cpu_to_be_32(metadata), (ehdr << 16) | rte_cpu_to_be_16(pkt_inline_sz), }; } @@ -861,7 +865,7 @@ mpw->wqe->eseg.inline_hdr_sz = 0; mpw->wqe->eseg.rsvd0 = 0; mpw->wqe->eseg.rsvd1 = 0; - mpw->wqe->eseg.rsvd2 = 0; + mpw->wqe->eseg.flow_table_metadata = 0; mpw->wqe->ctrl[0] = rte_cpu_to_be_32((MLX5_OPC_MOD_MPW << 24) | (txq->wqe_ci << 8) | MLX5_OPCODE_TSO); @@ -971,6 +975,8 @@ if ((mpw.state == MLX5_MPW_STATE_OPENED) && ((mpw.len != length) || (segs_n != 1) || + (mpw.wqe->eseg.flow_table_metadata != + rte_cpu_to_be_32(buf->hash.fdir.hi)) || (mpw.wqe->eseg.cs_flags != cs_flags))) mlx5_mpw_close(txq, &mpw); if (mpw.state == MLX5_MPW_STATE_CLOSED) { @@ -984,6 +990,8 @@ max_wqe -= 2; mlx5_mpw_new(txq, &mpw, length); mpw.wqe->eseg.cs_flags = cs_flags; + mpw.wqe->eseg.flow_table_metadata = + rte_cpu_to_be_32(buf->hash.fdir.hi); } /* Multi-segment packets must be alone in their MPW. */ assert((segs_n == 1) || (mpw.pkts_n == 0)); @@ -1082,7 +1090,7 @@ mpw->wqe->eseg.cs_flags = 0; mpw->wqe->eseg.rsvd0 = 0; mpw->wqe->eseg.rsvd1 = 0; - mpw->wqe->eseg.rsvd2 = 0; + mpw->wqe->eseg.flow_table_metadata = 0; inl = (struct mlx5_wqe_inl_small *) (((uintptr_t)mpw->wqe) + 2 * MLX5_WQE_DWORD_SIZE); mpw->data.raw = (uint8_t *)&inl->raw; @@ -1199,12 +1207,16 @@ if (mpw.state == MLX5_MPW_STATE_OPENED) { if ((mpw.len != length) || (segs_n != 1) || + (mpw.wqe->eseg.flow_table_metadata != + rte_cpu_to_be_32(buf->hash.fdir.hi)) || (mpw.wqe->eseg.cs_flags != cs_flags)) mlx5_mpw_close(txq, &mpw); } else if (mpw.state == MLX5_MPW_INL_STATE_OPENED) { if ((mpw.len != length) || (segs_n != 1) || (length > inline_room) || + (mpw.wqe->eseg.flow_table_metadata != + rte_cpu_to_be_32(buf->hash.fdir.hi)) || (mpw.wqe->eseg.cs_flags != cs_flags)) { mlx5_mpw_inline_close(txq, &mpw); inline_room = @@ -1224,12 +1236,20 @@ max_wqe -= 2; mlx5_mpw_new(txq, &mpw, length); mpw.wqe->eseg.cs_flags = cs_flags; + /* Copy metadata from mbuf if valid */ + if (buf->ol_flags & PKT_TX_METADATA) + mpw.wqe->eseg.flow_table_metadata = + rte_cpu_to_be_32(buf->hash.fdir.hi); } else { if (unlikely(max_wqe < wqe_inl_n)) break; max_wqe -= wqe_inl_n; mlx5_mpw_inline_new(txq, &mpw, length); mpw.wqe->eseg.cs_flags = cs_flags; + /* Copy metadata from mbuf if valid */ + if (buf->ol_flags & PKT_TX_METADATA) + mpw.wqe->eseg.flow_table_metadata = + rte_cpu_to_be_32(buf->hash.fdir.hi); } } /* Multi-segment packets must be alone in their MPW. */ @@ -1482,6 +1502,8 @@ (length <= txq->inline_max_packet_sz && inl_pad + sizeof(inl_hdr) + length > mpw_room) || + (mpw.wqe->eseg.flow_table_metadata != + rte_cpu_to_be_32(buf->hash.fdir.hi)) || (mpw.wqe->eseg.cs_flags != cs_flags)) max_wqe -= mlx5_empw_close(txq, &mpw); } @@ -1505,6 +1527,10 @@ sizeof(inl_hdr) + length <= mpw_room && !txq->mpw_hdr_dseg; mpw.wqe->eseg.cs_flags = cs_flags; + /* Copy metadata from mbuf if valid */ + if (buf->ol_flags & PKT_TX_METADATA) + mpw.wqe->eseg.flow_table_metadata = + rte_cpu_to_be_32(buf->hash.fdir.hi); } else { /* Evaluate whether the next packet can be inlined. * Inlininig is possible when: diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.c b/drivers/net/mlx5/mlx5_rxtx_vec.c index 0a4aed8..db36699 100644 --- a/drivers/net/mlx5/mlx5_rxtx_vec.c +++ b/drivers/net/mlx5/mlx5_rxtx_vec.c @@ -48,25 +48,39 @@ * Number of packets. * @param cs_flags * Pointer of flags to be returned. + * @param txq_offloads + * Offloads enabled on Tx queue * * @return * Number of packets having same ol_flags. + * If PKT_TX_METADATA is set in ol_flags, packets must have same metadata + * as well. */ static inline unsigned int -txq_calc_offload(struct rte_mbuf **pkts, uint16_t pkts_n, uint8_t *cs_flags) +txq_calc_offload(struct rte_mbuf **pkts, uint16_t pkts_n, + uint8_t *cs_flags, const uint64_t txq_offloads) { unsigned int pos; const uint64_t ol_mask = PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM | PKT_TX_TUNNEL_GRE | - PKT_TX_TUNNEL_VXLAN | PKT_TX_OUTER_IP_CKSUM; + PKT_TX_TUNNEL_VXLAN | PKT_TX_OUTER_IP_CKSUM | PKT_TX_METADATA; if (!pkts_n) return 0; /* Count the number of packets having same ol_flags. */ - for (pos = 1; pos < pkts_n; ++pos) - if ((pkts[pos]->ol_flags ^ pkts[0]->ol_flags) & ol_mask) + for (pos = 1; pos < pkts_n; ++pos) { + if ((txq_offloads & MLX5_VEC_TX_CKSUM_OFFLOAD_CAP) && + ((pkts[pos]->ol_flags ^ pkts[0]->ol_flags) & ol_mask)) break; + /* If the metadata ol_flag is set, + * metadata must be same in all packets. + */ + if ((txq_offloads & DEV_TX_OFFLOAD_MATCH_METADATA) && + (pkts[pos]->ol_flags & PKT_TX_METADATA) && + pkts[0]->hash.fdir.hi != pkts[pos]->hash.fdir.hi) + break; + } *cs_flags = txq_ol_cksum_to_cs(pkts[0]); return pos; } @@ -137,8 +151,10 @@ n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST); if (txq->offloads & DEV_TX_OFFLOAD_MULTI_SEGS) n = txq_count_contig_single_seg(&pkts[nb_tx], n); - if (txq->offloads & MLX5_VEC_TX_CKSUM_OFFLOAD_CAP) - n = txq_calc_offload(&pkts[nb_tx], n, &cs_flags); + if (txq->offloads & (MLX5_VEC_TX_CKSUM_OFFLOAD_CAP | + DEV_TX_OFFLOAD_MATCH_METADATA)) + n = txq_calc_offload(&pkts[nb_tx], n, + &cs_flags, txq->offloads); ret = txq_burst_v(txq, &pkts[nb_tx], n, cs_flags); nb_tx += ret; if (!ret) diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.h b/drivers/net/mlx5/mlx5_rxtx_vec.h index fb884f9..fda7004 100644 --- a/drivers/net/mlx5/mlx5_rxtx_vec.h +++ b/drivers/net/mlx5/mlx5_rxtx_vec.h @@ -22,6 +22,7 @@ /* HW offload capabilities of vectorized Tx. */ #define MLX5_VEC_TX_OFFLOAD_CAP \ (MLX5_VEC_TX_CKSUM_OFFLOAD_CAP | \ + DEV_TX_OFFLOAD_MATCH_METADATA | \ DEV_TX_OFFLOAD_MULTI_SEGS) /* diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_neon.h b/drivers/net/mlx5/mlx5_rxtx_vec_neon.h index b37b738..20c9427 100644 --- a/drivers/net/mlx5/mlx5_rxtx_vec_neon.h +++ b/drivers/net/mlx5/mlx5_rxtx_vec_neon.h @@ -237,6 +237,7 @@ uint8x16_t *t_wqe; uint8_t *dseg; uint8x16_t ctrl; + uint32_t md; /* metadata */ /* Make sure all packets can fit into a single WQE. */ assert(elts_n > pkts_n); @@ -293,10 +294,11 @@ ctrl = vqtbl1q_u8(ctrl, ctrl_shuf_m); vst1q_u8((void *)t_wqe, ctrl); /* Fill ESEG in the header. */ + md = pkts[0]->hash.fdir.hi; vst1q_u8((void *)(t_wqe + 1), ((uint8x16_t) { 0, 0, 0, 0, cs_flags, 0, 0, 0, - 0, 0, 0, 0, + md >> 24, md >> 16, md >> 8, md, 0, 0, 0, 0 })); #ifdef MLX5_PMD_SOFT_COUNTERS txq->stats.opackets += pkts_n; diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h index 54b3783..7c8535c 100644 --- a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h +++ b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h @@ -236,6 +236,7 @@ 0, 1, 2, 3 /* bswap32 */); __m128i *t_wqe, *dseg; __m128i ctrl; + uint32_t md; /* metadata */ /* Make sure all packets can fit into a single WQE. */ assert(elts_n > pkts_n); @@ -292,9 +293,10 @@ ctrl = _mm_shuffle_epi8(ctrl, shuf_mask_ctrl); _mm_store_si128(t_wqe, ctrl); /* Fill ESEG in the header. */ + md = pkts[0]->hash.fdir.hi; _mm_store_si128(t_wqe + 1, _mm_set_epi8(0, 0, 0, 0, - 0, 0, 0, 0, + md, md >> 8, md >> 16, md >> 24, 0, 0, 0, cs_flags, 0, 0, 0, 0)); #ifdef MLX5_PMD_SOFT_COUNTERS diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c index f9bc473..7263fb1 100644 --- a/drivers/net/mlx5/mlx5_txq.c +++ b/drivers/net/mlx5/mlx5_txq.c @@ -128,6 +128,12 @@ offloads |= (DEV_TX_OFFLOAD_VXLAN_TNL_TSO | DEV_TX_OFFLOAD_GRE_TNL_TSO); } + +#ifdef HAVE_IBV_FLOW_DV_SUPPORT + if (config->dv_flow_en) + offloads |= DEV_TX_OFFLOAD_MATCH_METADATA; +#endif + return offloads; } -- 1.8.3.1