From: Adrien Mazarguil <adrien.mazarguil@6wind.com>
To: Ferruh Yigit <ferruh.yigit@intel.com>
Cc: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>, dev@dpdk.org
Subject: [dpdk-dev] [PATCH v2 24/29] net/mlx4: allocate queues and mbuf rings together
Date: Thu, 12 Oct 2017 14:19:38 +0200 [thread overview]
Message-ID: <6a50316ed3b2b7a8607baa6c8cf8414e2400a980.1507809961.git.adrien.mazarguil@6wind.com> (raw)
In-Reply-To: <cover.1507809961.git.adrien.mazarguil@6wind.com>
Since live Tx and Rx queues cannot be reused anymore without being
destroyed first, mbuf ring sizes are fixed and known from the start.
This allows a single allocation for queue data structures and mbuf ring
together, saving space and bringing them closer in memory.
Signed-off-by: Adrien Mazarguil <adrien.mazarguil@6wind.com>
Acked-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
drivers/net/mlx4/mlx4_rxq.c | 71 +++++++++++--------------
drivers/net/mlx4/mlx4_rxtx.h | 2 +
drivers/net/mlx4/mlx4_txq.c | 109 +++++++++++---------------------------
3 files changed, 65 insertions(+), 117 deletions(-)
diff --git a/drivers/net/mlx4/mlx4_rxq.c b/drivers/net/mlx4/mlx4_rxq.c
index 30b0654..9978e5d 100644
--- a/drivers/net/mlx4/mlx4_rxq.c
+++ b/drivers/net/mlx4/mlx4_rxq.c
@@ -69,36 +69,30 @@
*
* @param rxq
* Pointer to Rx queue structure.
- * @param elts_n
- * Number of elements to allocate.
*
* @return
* 0 on success, negative errno value otherwise and rte_errno is set.
*/
static int
-mlx4_rxq_alloc_elts(struct rxq *rxq, unsigned int elts_n)
+mlx4_rxq_alloc_elts(struct rxq *rxq)
{
+ struct rxq_elt (*elts)[rxq->elts_n] = rxq->elts;
unsigned int i;
- struct rxq_elt (*elts)[elts_n] =
- rte_calloc_socket("RXQ elements", 1, sizeof(*elts), 0,
- rxq->socket);
- if (elts == NULL) {
- rte_errno = ENOMEM;
- ERROR("%p: can't allocate packets array", (void *)rxq);
- goto error;
- }
/* For each WR (packet). */
- for (i = 0; (i != elts_n); ++i) {
+ for (i = 0; i != RTE_DIM(*elts); ++i) {
struct rxq_elt *elt = &(*elts)[i];
struct ibv_recv_wr *wr = &elt->wr;
struct ibv_sge *sge = &(*elts)[i].sge;
struct rte_mbuf *buf = rte_pktmbuf_alloc(rxq->mp);
if (buf == NULL) {
+ while (i--) {
+ rte_pktmbuf_free_seg((*elts)[i].buf);
+ (*elts)[i].buf = NULL;
+ }
rte_errno = ENOMEM;
- ERROR("%p: empty mbuf pool", (void *)rxq);
- goto error;
+ return -rte_errno;
}
elt->buf = buf;
wr->next = &(*elts)[(i + 1)].wr;
@@ -121,21 +115,7 @@ mlx4_rxq_alloc_elts(struct rxq *rxq, unsigned int elts_n)
}
/* The last WR pointer must be NULL. */
(*elts)[(i - 1)].wr.next = NULL;
- DEBUG("%p: allocated and configured %u single-segment WRs",
- (void *)rxq, elts_n);
- rxq->elts_n = elts_n;
- rxq->elts_head = 0;
- rxq->elts = elts;
return 0;
-error:
- if (elts != NULL) {
- for (i = 0; (i != RTE_DIM(*elts)); ++i)
- rte_pktmbuf_free_seg((*elts)[i].buf);
- rte_free(elts);
- }
- DEBUG("%p: failed, freed everything", (void *)rxq);
- assert(rte_errno > 0);
- return -rte_errno;
}
/**
@@ -148,17 +128,15 @@ static void
mlx4_rxq_free_elts(struct rxq *rxq)
{
unsigned int i;
- unsigned int elts_n = rxq->elts_n;
- struct rxq_elt (*elts)[elts_n] = rxq->elts;
+ struct rxq_elt (*elts)[rxq->elts_n] = rxq->elts;
DEBUG("%p: freeing WRs", (void *)rxq);
- rxq->elts_n = 0;
- rxq->elts = NULL;
- if (elts == NULL)
- return;
- for (i = 0; (i != RTE_DIM(*elts)); ++i)
+ for (i = 0; (i != RTE_DIM(*elts)); ++i) {
+ if (!(*elts)[i].buf)
+ continue;
rte_pktmbuf_free_seg((*elts)[i].buf);
- rte_free(elts);
+ (*elts)[i].buf = NULL;
+ }
}
/**
@@ -187,8 +165,21 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
{
struct priv *priv = dev->data->dev_private;
uint32_t mb_len = rte_pktmbuf_data_room_size(mp);
+ struct rxq_elt (*elts)[desc];
struct rte_flow_error error;
struct rxq *rxq;
+ struct mlx4_malloc_vec vec[] = {
+ {
+ .align = RTE_CACHE_LINE_SIZE,
+ .size = sizeof(*rxq),
+ .addr = (void **)&rxq,
+ },
+ {
+ .align = RTE_CACHE_LINE_SIZE,
+ .size = sizeof(*elts),
+ .addr = (void **)&elts,
+ },
+ };
int ret;
(void)conf; /* Thresholds configuration (ignored). */
@@ -213,9 +204,8 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
return -rte_errno;
}
/* Allocate and initialize Rx queue. */
- rxq = rte_calloc_socket("RXQ", 1, sizeof(*rxq), 0, socket);
+ mlx4_zmallocv_socket("RXQ", vec, RTE_DIM(vec), socket);
if (!rxq) {
- rte_errno = ENOMEM;
ERROR("%p: unable to allocate queue index %u",
(void *)dev, idx);
return -rte_errno;
@@ -224,6 +214,9 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
.priv = priv,
.mp = mp,
.port_id = dev->data->port_id,
+ .elts_n = desc,
+ .elts_head = 0,
+ .elts = elts,
.stats.idx = idx,
.socket = socket,
};
@@ -307,7 +300,7 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
(void *)dev, strerror(rte_errno));
goto error;
}
- ret = mlx4_rxq_alloc_elts(rxq, desc);
+ ret = mlx4_rxq_alloc_elts(rxq);
if (ret) {
ERROR("%p: RXQ allocation failed: %s",
(void *)dev, strerror(rte_errno));
diff --git a/drivers/net/mlx4/mlx4_rxtx.h b/drivers/net/mlx4/mlx4_rxtx.h
index d62120e..d90f2f9 100644
--- a/drivers/net/mlx4/mlx4_rxtx.h
+++ b/drivers/net/mlx4/mlx4_rxtx.h
@@ -81,6 +81,7 @@ struct rxq {
struct rxq_elt (*elts)[]; /**< Rx elements. */
struct mlx4_rxq_stats stats; /**< Rx queue counters. */
unsigned int socket; /**< CPU socket ID for allocations. */
+ uint8_t data[]; /**< Remaining queue resources. */
};
/** Tx element. */
@@ -118,6 +119,7 @@ struct txq {
unsigned int elts_comp_cd_init; /**< Initial value for countdown. */
struct mlx4_txq_stats stats; /**< Tx queue counters. */
unsigned int socket; /**< CPU socket ID for allocations. */
+ uint8_t data[]; /**< Remaining queue resources. */
};
/* mlx4_rxq.c */
diff --git a/drivers/net/mlx4/mlx4_txq.c b/drivers/net/mlx4/mlx4_txq.c
index f102c68..915f8d7 100644
--- a/drivers/net/mlx4/mlx4_txq.c
+++ b/drivers/net/mlx4/mlx4_txq.c
@@ -64,59 +64,6 @@
#include "mlx4_utils.h"
/**
- * Allocate Tx queue elements.
- *
- * @param txq
- * Pointer to Tx queue structure.
- * @param elts_n
- * Number of elements to allocate.
- *
- * @return
- * 0 on success, negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx4_txq_alloc_elts(struct txq *txq, unsigned int elts_n)
-{
- unsigned int i;
- struct txq_elt (*elts)[elts_n] =
- rte_calloc_socket("TXQ", 1, sizeof(*elts), 0, txq->socket);
- int ret = 0;
-
- if (elts == NULL) {
- ERROR("%p: can't allocate packets array", (void *)txq);
- ret = ENOMEM;
- goto error;
- }
- for (i = 0; (i != elts_n); ++i) {
- struct txq_elt *elt = &(*elts)[i];
-
- elt->buf = NULL;
- }
- DEBUG("%p: allocated and configured %u WRs", (void *)txq, elts_n);
- txq->elts_n = elts_n;
- txq->elts = elts;
- txq->elts_head = 0;
- txq->elts_tail = 0;
- txq->elts_comp = 0;
- /*
- * Request send completion every MLX4_PMD_TX_PER_COMP_REQ packets or
- * at least 4 times per ring.
- */
- txq->elts_comp_cd_init =
- ((MLX4_PMD_TX_PER_COMP_REQ < (elts_n / 4)) ?
- MLX4_PMD_TX_PER_COMP_REQ : (elts_n / 4));
- txq->elts_comp_cd = txq->elts_comp_cd_init;
- assert(ret == 0);
- return 0;
-error:
- rte_free(elts);
- DEBUG("%p: failed, freed everything", (void *)txq);
- assert(ret > 0);
- rte_errno = ret;
- return -rte_errno;
-}
-
-/**
* Free Tx queue elements.
*
* @param txq
@@ -125,34 +72,21 @@ mlx4_txq_alloc_elts(struct txq *txq, unsigned int elts_n)
static void
mlx4_txq_free_elts(struct txq *txq)
{
- unsigned int elts_n = txq->elts_n;
unsigned int elts_head = txq->elts_head;
unsigned int elts_tail = txq->elts_tail;
- struct txq_elt (*elts)[elts_n] = txq->elts;
+ struct txq_elt (*elts)[txq->elts_n] = txq->elts;
DEBUG("%p: freeing WRs", (void *)txq);
- txq->elts_n = 0;
- txq->elts_head = 0;
- txq->elts_tail = 0;
- txq->elts_comp = 0;
- txq->elts_comp_cd = 0;
- txq->elts_comp_cd_init = 0;
- txq->elts = NULL;
- if (elts == NULL)
- return;
while (elts_tail != elts_head) {
struct txq_elt *elt = &(*elts)[elts_tail];
assert(elt->buf != NULL);
rte_pktmbuf_free(elt->buf);
-#ifndef NDEBUG
- /* Poisoning. */
- memset(elt, 0x77, sizeof(*elt));
-#endif
- if (++elts_tail == elts_n)
+ elt->buf = NULL;
+ if (++elts_tail == RTE_DIM(*elts))
elts_tail = 0;
}
- rte_free(elts);
+ txq->elts_tail = txq->elts_head;
}
struct txq_mp2mr_mbuf_check_data {
@@ -235,8 +169,21 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
unsigned int socket, const struct rte_eth_txconf *conf)
{
struct priv *priv = dev->data->dev_private;
+ struct txq_elt (*elts)[desc];
struct ibv_qp_init_attr qp_init_attr;
struct txq *txq;
+ struct mlx4_malloc_vec vec[] = {
+ {
+ .align = RTE_CACHE_LINE_SIZE,
+ .size = sizeof(*txq),
+ .addr = (void **)&txq,
+ },
+ {
+ .align = RTE_CACHE_LINE_SIZE,
+ .size = sizeof(*elts),
+ .addr = (void **)&elts,
+ },
+ };
int ret;
(void)conf; /* Thresholds configuration (ignored). */
@@ -261,9 +208,8 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
return -rte_errno;
}
/* Allocate and initialize Tx queue. */
- txq = rte_calloc_socket("TXQ", 1, sizeof(*txq), 0, socket);
+ mlx4_zmallocv_socket("TXQ", vec, RTE_DIM(vec), socket);
if (!txq) {
- rte_errno = ENOMEM;
ERROR("%p: unable to allocate queue index %u",
(void *)dev, idx);
return -rte_errno;
@@ -272,6 +218,19 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
.priv = priv,
.stats.idx = idx,
.socket = socket,
+ .elts_n = desc,
+ .elts = elts,
+ .elts_head = 0,
+ .elts_tail = 0,
+ .elts_comp = 0,
+ /*
+ * Request send completion every MLX4_PMD_TX_PER_COMP_REQ
+ * packets or at least 4 times per ring.
+ */
+ .elts_comp_cd =
+ RTE_MIN(MLX4_PMD_TX_PER_COMP_REQ, desc / 4),
+ .elts_comp_cd_init =
+ RTE_MIN(MLX4_PMD_TX_PER_COMP_REQ, desc / 4),
};
txq->cq = ibv_create_cq(priv->ctx, desc, NULL, NULL, 0);
if (!txq->cq) {
@@ -314,12 +273,6 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
(void *)dev, strerror(rte_errno));
goto error;
}
- ret = mlx4_txq_alloc_elts(txq, desc);
- if (ret) {
- ERROR("%p: TXQ allocation failed: %s",
- (void *)dev, strerror(rte_errno));
- goto error;
- }
ret = ibv_modify_qp
(txq->qp,
&(struct ibv_qp_attr){
--
2.1.4
next prev parent reply other threads:[~2017-10-12 12:20 UTC|newest]
Thread overview: 64+ messages / expand[flat|nested] mbox.gz Atom feed top
2017-10-11 14:35 [dpdk-dev] [PATCH v1 00/29] net/mlx4: restore PMD functionality Adrien Mazarguil
2017-10-11 14:35 ` [dpdk-dev] [PATCH v1 01/29] ethdev: expose flow API error helper Adrien Mazarguil
2017-10-11 14:35 ` [dpdk-dev] [PATCH v1 02/29] net/mlx4: replace bit-field type Adrien Mazarguil
2017-10-11 14:35 ` [dpdk-dev] [PATCH v1 03/29] net/mlx4: remove Rx QP initializer function Adrien Mazarguil
2017-10-11 14:35 ` [dpdk-dev] [PATCH v1 04/29] net/mlx4: enhance header files comments Adrien Mazarguil
2017-10-11 14:35 ` [dpdk-dev] [PATCH v1 05/29] net/mlx4: expose support for flow rule priorities Adrien Mazarguil
2017-10-11 14:35 ` [dpdk-dev] [PATCH v1 06/29] net/mlx4: clarify flow objects naming scheme Adrien Mazarguil
2017-10-11 14:35 ` [dpdk-dev] [PATCH v1 07/29] net/mlx4: tidy up flow rule handling code Adrien Mazarguil
2017-10-11 14:35 ` [dpdk-dev] [PATCH v1 08/29] net/mlx4: compact flow rule error reporting Adrien Mazarguil
2017-10-11 14:35 ` [dpdk-dev] [PATCH v1 09/29] mem: add iovec-like allocation wrappers Adrien Mazarguil
2017-10-11 21:58 ` Ferruh Yigit
2017-10-11 22:00 ` Ferruh Yigit
2017-10-12 11:07 ` Adrien Mazarguil
2017-10-11 14:35 ` [dpdk-dev] [PATCH v1 10/29] net/mlx4: merge flow creation and validation code Adrien Mazarguil
2017-10-11 14:35 ` [dpdk-dev] [PATCH v1 11/29] net/mlx4: allocate drop flow resources on demand Adrien Mazarguil
2017-10-11 14:35 ` [dpdk-dev] [PATCH v1 12/29] net/mlx4: relax check on missing flow rule target Adrien Mazarguil
2017-10-11 14:35 ` [dpdk-dev] [PATCH v1 13/29] net/mlx4: refactor internal flow rules Adrien Mazarguil
2017-10-11 14:35 ` [dpdk-dev] [PATCH v1 14/29] net/mlx4: generalize flow rule priority support Adrien Mazarguil
2017-10-11 14:35 ` [dpdk-dev] [PATCH v1 15/29] net/mlx4: simplify trigger code for flow rules Adrien Mazarguil
2017-10-11 14:35 ` [dpdk-dev] [PATCH v1 16/29] net/mlx4: refactor flow item validation code Adrien Mazarguil
2017-10-11 14:35 ` [dpdk-dev] [PATCH v1 17/29] net/mlx4: add MAC addresses configuration support Adrien Mazarguil
2017-10-11 14:35 ` [dpdk-dev] [PATCH v1 18/29] net/mlx4: add VLAN filter " Adrien Mazarguil
2017-10-11 14:35 ` [dpdk-dev] [PATCH v1 19/29] net/mlx4: add flow support for multicast traffic Adrien Mazarguil
2017-10-11 14:35 ` [dpdk-dev] [PATCH v1 20/29] net/mlx4: restore promisc and allmulti support Adrien Mazarguil
2017-10-11 14:35 ` [dpdk-dev] [PATCH v1 21/29] net/mlx4: update Rx/Tx callbacks consistently Adrien Mazarguil
2017-10-11 14:35 ` [dpdk-dev] [PATCH v1 22/29] net/mlx4: fix invalid errno value sign Adrien Mazarguil
2017-10-11 14:35 ` [dpdk-dev] [PATCH v1 23/29] net/mlx4: drop live queue reconfiguration support Adrien Mazarguil
2017-10-11 14:35 ` [dpdk-dev] [PATCH v1 24/29] net/mlx4: allocate queues and mbuf rings together Adrien Mazarguil
2017-10-11 14:35 ` [dpdk-dev] [PATCH v1 25/29] net/mlx4: convert Rx path to work queues Adrien Mazarguil
2017-10-11 14:35 ` [dpdk-dev] [PATCH v1 26/29] net/mlx4: remove unnecessary check Adrien Mazarguil
2017-10-11 14:35 ` [dpdk-dev] [PATCH v1 27/29] net/mlx4: add RSS flow rule action support Adrien Mazarguil
2017-10-11 14:35 ` [dpdk-dev] [PATCH v1 28/29] net/mlx4: disable UDP support in RSS flow rules Adrien Mazarguil
2017-10-11 14:35 ` [dpdk-dev] [PATCH v1 29/29] net/mlx4: add RSS support outside flow API Adrien Mazarguil
2017-10-12 12:19 ` [dpdk-dev] [PATCH v2 00/29] net/mlx4: restore PMD functionality Adrien Mazarguil
2017-10-12 12:19 ` [dpdk-dev] [PATCH v2 01/29] ethdev: expose flow API error helper Adrien Mazarguil
2017-10-12 12:19 ` [dpdk-dev] [PATCH v2 02/29] net/mlx4: replace bit-field type Adrien Mazarguil
2017-10-12 12:19 ` [dpdk-dev] [PATCH v2 03/29] net/mlx4: remove Rx QP initializer function Adrien Mazarguil
2017-10-12 12:19 ` [dpdk-dev] [PATCH v2 04/29] net/mlx4: enhance header files comments Adrien Mazarguil
2017-10-12 12:19 ` [dpdk-dev] [PATCH v2 05/29] net/mlx4: expose support for flow rule priorities Adrien Mazarguil
2017-10-12 12:19 ` [dpdk-dev] [PATCH v2 06/29] net/mlx4: clarify flow objects naming scheme Adrien Mazarguil
2017-10-12 12:19 ` [dpdk-dev] [PATCH v2 07/29] net/mlx4: tidy up flow rule handling code Adrien Mazarguil
2017-10-12 12:19 ` [dpdk-dev] [PATCH v2 08/29] net/mlx4: compact flow rule error reporting Adrien Mazarguil
2017-10-12 12:19 ` [dpdk-dev] [PATCH v2 09/29] net/mlx4: add iovec-like allocation wrappers Adrien Mazarguil
2017-10-12 12:19 ` [dpdk-dev] [PATCH v2 10/29] net/mlx4: merge flow creation and validation code Adrien Mazarguil
2017-10-12 12:19 ` [dpdk-dev] [PATCH v2 11/29] net/mlx4: allocate drop flow resources on demand Adrien Mazarguil
2017-10-12 12:19 ` [dpdk-dev] [PATCH v2 12/29] net/mlx4: relax check on missing flow rule target Adrien Mazarguil
2017-10-12 12:19 ` [dpdk-dev] [PATCH v2 13/29] net/mlx4: refactor internal flow rules Adrien Mazarguil
2017-10-12 12:19 ` [dpdk-dev] [PATCH v2 14/29] net/mlx4: generalize flow rule priority support Adrien Mazarguil
2017-10-12 12:19 ` [dpdk-dev] [PATCH v2 15/29] net/mlx4: simplify trigger code for flow rules Adrien Mazarguil
2017-10-12 12:19 ` [dpdk-dev] [PATCH v2 16/29] net/mlx4: refactor flow item validation code Adrien Mazarguil
2017-10-12 12:19 ` [dpdk-dev] [PATCH v2 17/29] net/mlx4: add MAC addresses configuration support Adrien Mazarguil
2017-10-12 12:19 ` [dpdk-dev] [PATCH v2 18/29] net/mlx4: add VLAN filter " Adrien Mazarguil
2017-10-12 12:19 ` [dpdk-dev] [PATCH v2 19/29] net/mlx4: add flow support for multicast traffic Adrien Mazarguil
2017-10-12 12:19 ` [dpdk-dev] [PATCH v2 20/29] net/mlx4: restore promisc and allmulti support Adrien Mazarguil
2017-10-12 12:19 ` [dpdk-dev] [PATCH v2 21/29] net/mlx4: update Rx/Tx callbacks consistently Adrien Mazarguil
2017-10-12 12:19 ` [dpdk-dev] [PATCH v2 22/29] net/mlx4: fix invalid errno value sign Adrien Mazarguil
2017-10-12 12:19 ` [dpdk-dev] [PATCH v2 23/29] net/mlx4: drop live queue reconfiguration support Adrien Mazarguil
2017-10-12 12:19 ` Adrien Mazarguil [this message]
2017-10-12 12:19 ` [dpdk-dev] [PATCH v2 25/29] net/mlx4: convert Rx path to work queues Adrien Mazarguil
2017-10-12 12:19 ` [dpdk-dev] [PATCH v2 26/29] net/mlx4: remove unnecessary check Adrien Mazarguil
2017-10-12 12:19 ` [dpdk-dev] [PATCH v2 27/29] net/mlx4: add RSS flow rule action support Adrien Mazarguil
2017-10-12 12:19 ` [dpdk-dev] [PATCH v2 28/29] net/mlx4: disable UDP support in RSS flow rules Adrien Mazarguil
2017-10-12 12:19 ` [dpdk-dev] [PATCH v2 29/29] net/mlx4: add RSS support outside flow API Adrien Mazarguil
2017-10-12 19:12 ` [dpdk-dev] [PATCH v2 00/29] net/mlx4: restore PMD functionality Ferruh Yigit
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=6a50316ed3b2b7a8607baa6c8cf8414e2400a980.1507809961.git.adrien.mazarguil@6wind.com \
--to=adrien.mazarguil@6wind.com \
--cc=dev@dpdk.org \
--cc=ferruh.yigit@intel.com \
--cc=nelio.laranjeiro@6wind.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).