From: Bruce Richardson <bruce.richardson@intel.com>
To: dev@dpdk.org
Subject: [dpdk-dev] [PATCH v2 09/13] ixgbe: rework vector pmd following mbuf changes
Date: Thu, 11 Sep 2014 14:15:43 +0100 [thread overview]
Message-ID: <1410441347-22840-10-git-send-email-bruce.richardson@intel.com> (raw)
In-Reply-To: <1410441347-22840-1-git-send-email-bruce.richardson@intel.com>
The vector PMD expects fields to be in a specific order so that it can
do vector operations on multiple fields at a time. Following mbuf
rework, adjust driver to take account of the new layout and re-enable it
in the config.
Updates in V2:
* Remove extra line at end of file to eliminate git warning
Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
---
config/common_linuxapp | 2 +-
lib/librte_pmd_ixgbe/ixgbe_rxtx.c | 2 +-
lib/librte_pmd_ixgbe/ixgbe_rxtx.h | 4 +-
lib/librte_pmd_ixgbe/ixgbe_rxtx_vec.c | 135 ++++++++++++----------------------
4 files changed, 53 insertions(+), 90 deletions(-)
diff --git a/config/common_linuxapp b/config/common_linuxapp
index b140af7..5bee910 100644
--- a/config/common_linuxapp
+++ b/config/common_linuxapp
@@ -191,7 +191,7 @@ CONFIG_RTE_LIBRTE_IXGBE_DEBUG_DRIVER=n
CONFIG_RTE_LIBRTE_IXGBE_PF_DISABLE_STRIP_CRC=n
CONFIG_RTE_LIBRTE_IXGBE_RX_ALLOW_BULK_ALLOC=y
CONFIG_RTE_LIBRTE_IXGBE_ALLOW_UNSUPPORTED_SFP=n
-CONFIG_RTE_IXGBE_INC_VECTOR=n
+CONFIG_RTE_IXGBE_INC_VECTOR=y
CONFIG_RTE_IXGBE_RX_OLFLAGS_ENABLE=y
#
diff --git a/lib/librte_pmd_ixgbe/ixgbe_rxtx.c b/lib/librte_pmd_ixgbe/ixgbe_rxtx.c
index 26cb176..1a46393 100644
--- a/lib/librte_pmd_ixgbe/ixgbe_rxtx.c
+++ b/lib/librte_pmd_ixgbe/ixgbe_rxtx.c
@@ -2166,7 +2166,7 @@ ixgbe_dev_rx_queue_setup(struct rte_eth_dev *dev,
if (!ixgbe_rx_vec_condition_check(dev)) {
PMD_INIT_LOG(INFO, "Vector rx enabled, please make "
"sure RX burst size no less than 32.\n");
- ixgbe_rxq_vec_setup(rxq, socket_id);
+ ixgbe_rxq_vec_setup(rxq);
dev->rx_pkt_burst = ixgbe_recv_pkts_vec;
}
#endif
diff --git a/lib/librte_pmd_ixgbe/ixgbe_rxtx.h b/lib/librte_pmd_ixgbe/ixgbe_rxtx.h
index 0d633d6..e92a864 100644
--- a/lib/librte_pmd_ixgbe/ixgbe_rxtx.h
+++ b/lib/librte_pmd_ixgbe/ixgbe_rxtx.h
@@ -115,6 +115,7 @@ struct igb_rx_queue {
struct igb_rx_entry *sw_ring; /**< address of RX software ring. */
struct rte_mbuf *pkt_first_seg; /**< First segment of current packet. */
struct rte_mbuf *pkt_last_seg; /**< Last segment of current packet. */
+ uint64_t mbuf_initializer; /**< value to init mbufs */
uint16_t nb_rx_desc; /**< number of RX descriptors. */
uint16_t rx_tail; /**< current value of RDT register. */
uint16_t nb_rx_hold; /**< number of held free RX desc. */
@@ -126,7 +127,6 @@ struct igb_rx_queue {
#ifdef RTE_IXGBE_INC_VECTOR
uint16_t rxrearm_nb; /**< the idx we start the re-arming from */
uint16_t rxrearm_start; /**< number of remaining to be re-armed */
- __m128i misc_info; /**< cache XMM combine port_id/crc/nb_segs */
#endif
uint16_t rx_free_thresh; /**< max free RX desc to hold. */
uint16_t queue_id; /**< RX queue index. */
@@ -259,7 +259,7 @@ struct ixgbe_txq_ops {
uint16_t ixgbe_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
uint16_t ixgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
int ixgbe_txq_vec_setup(struct igb_tx_queue *txq, unsigned int socket_id);
-int ixgbe_rxq_vec_setup(struct igb_rx_queue *rxq, unsigned int socket_id);
+int ixgbe_rxq_vec_setup(struct igb_rx_queue *rxq);
int ixgbe_rx_vec_condition_check(struct rte_eth_dev *dev);
#endif
diff --git a/lib/librte_pmd_ixgbe/ixgbe_rxtx_vec.c b/lib/librte_pmd_ixgbe/ixgbe_rxtx_vec.c
index bafb215..d53e239 100644
--- a/lib/librte_pmd_ixgbe/ixgbe_rxtx_vec.c
+++ b/lib/librte_pmd_ixgbe/ixgbe_rxtx_vec.c
@@ -47,15 +47,11 @@
static inline void
ixgbe_rxq_rearm(struct igb_rx_queue *rxq)
{
- static const struct rte_mbuf mb_def = {
- .nb_segs = 1,
- };
int i;
uint16_t rx_id;
volatile union ixgbe_adv_rx_desc *rxdp;
struct igb_rx_entry *rxep = &rxq->sw_ring[rxq->rxrearm_start];
struct rte_mbuf *mb0, *mb1;
- __m128i def_low;
__m128i hdr_room = _mm_set_epi64x(RTE_PKTMBUF_HEADROOM,
RTE_PKTMBUF_HEADROOM);
@@ -66,8 +62,6 @@ ixgbe_rxq_rearm(struct igb_rx_queue *rxq)
rxdp = rxq->rx_ring + rxq->rxrearm_start;
- def_low = _mm_load_si128((__m128i *)&(mb_def.next));
-
/* Initialize the mbufs in vector, process 2 mbufs in one loop */
for (i = 0; i < RTE_IXGBE_RXQ_REARM_THRESH; i += 2, rxep += 2) {
__m128i dma_addr0, dma_addr1;
@@ -76,33 +70,25 @@ ixgbe_rxq_rearm(struct igb_rx_queue *rxq)
mb0 = rxep[0].mbuf;
mb1 = rxep[1].mbuf;
+ /* flush mbuf with pkt template */
+ mb0->rearm_data[0] = rxq->mbuf_initializer;
+ mb1->rearm_data[0] = rxq->mbuf_initializer;
+
/* load buf_addr(lo 64bit) and buf_physaddr(hi 64bit) */
vaddr0 = _mm_loadu_si128((__m128i *)&(mb0->buf_addr));
vaddr1 = _mm_loadu_si128((__m128i *)&(mb1->buf_addr));
- /* calc va/pa of pkt data point */
- vaddr0 = _mm_add_epi64(vaddr0, hdr_room);
- vaddr1 = _mm_add_epi64(vaddr1, hdr_room);
-
/* convert pa to dma_addr hdr/data */
dma_addr0 = _mm_unpackhi_epi64(vaddr0, vaddr0);
dma_addr1 = _mm_unpackhi_epi64(vaddr1, vaddr1);
- /* fill va into t0 def pkt template */
- vaddr0 = _mm_unpacklo_epi64(def_low, vaddr0);
- vaddr1 = _mm_unpacklo_epi64(def_low, vaddr1);
+ /* add headroom to pa values */
+ dma_addr0 = _mm_add_epi64(dma_addr0, hdr_room);
+ dma_addr1 = _mm_add_epi64(dma_addr1, hdr_room);
/* flush desc with pa dma_addr */
_mm_store_si128((__m128i *)&rxdp++->read, dma_addr0);
_mm_store_si128((__m128i *)&rxdp++->read, dma_addr1);
-
- /* flush mbuf with pkt template */
- _mm_store_si128((__m128i *)&mb0->next, vaddr0);
- _mm_store_si128((__m128i *)&mb1->next, vaddr1);
-
- /* update refcnt per pkt */
- rte_mbuf_refcnt_set(mb0, 1);
- rte_mbuf_refcnt_set(mb1, 1);
}
rxq->rxrearm_start += RTE_IXGBE_RXQ_REARM_THRESH;
@@ -189,7 +175,13 @@ ixgbe_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
int pos;
uint64_t var;
__m128i shuf_msk;
- __m128i in_port;
+ __m128i crc_adjust = _mm_set_epi16(
+ 0, 0, 0, 0, /* ignore non-length fields */
+ 0, /* ignore high-16bits of pkt_len */
+ -rxq->crc_len, /* sub crc on pkt_len */
+ -rxq->crc_len, /* sub crc on data_len */
+ 0 /* ignore pkt_type field */
+ );
__m128i dd_check;
if (unlikely(nb_pkts < RTE_IXGBE_VPMD_RX_BURST))
@@ -222,8 +214,8 @@ ixgbe_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
15, 14, /* octet 14~15, low 16 bits vlan_macip */
0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */
13, 12, /* octet 12~13, low 16 bits pkt_len */
- 0xFF, 0xFF, /* skip nb_segs and in_port, zero out */
- 13, 12 /* octet 12~13, 16 bits data_len */
+ 13, 12, /* octet 12~13, 16 bits data_len */
+ 0xFF, 0xFF /* skip pkt_type field */
);
@@ -231,9 +223,6 @@ ixgbe_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
* the next 'n' mbufs into the cache */
sw_ring = &rxq->sw_ring[rxq->rx_tail];
- /* in_port, nb_seg = 1, crc_len */
- in_port = rxq->misc_info;
-
/*
* A. load 4 packet in one loop
* B. copy 4 mbuf point from swring to rx_pkts
@@ -285,8 +274,8 @@ ixgbe_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
desc_to_olflags_v(descs, &rx_pkts[pos]);
/* D.2 pkt 3,4 set in_port/nb_seg and remove crc */
- pkt_mb4 = _mm_add_epi16(pkt_mb4, in_port);
- pkt_mb3 = _mm_add_epi16(pkt_mb3, in_port);
+ pkt_mb4 = _mm_add_epi16(pkt_mb4, crc_adjust);
+ pkt_mb3 = _mm_add_epi16(pkt_mb3, crc_adjust);
/* D.1 pkt 1,2 convert format from desc to pktmbuf */
pkt_mb2 = _mm_shuffle_epi8(descs[1], shuf_msk);
@@ -297,23 +286,23 @@ ixgbe_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
staterr = _mm_unpacklo_epi32(sterr_tmp1, sterr_tmp2);
/* D.3 copy final 3,4 data to rx_pkts */
- _mm_storeu_si128((__m128i *)&(rx_pkts[pos+3]->data_len),
+ _mm_storeu_si128((void *)&rx_pkts[pos+3]->rx_descriptor_fields1,
pkt_mb4);
- _mm_storeu_si128((__m128i *)&(rx_pkts[pos+2]->data_len),
+ _mm_storeu_si128((void *)&rx_pkts[pos+2]->rx_descriptor_fields1,
pkt_mb3);
/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */
- pkt_mb2 = _mm_add_epi16(pkt_mb2, in_port);
- pkt_mb1 = _mm_add_epi16(pkt_mb1, in_port);
+ pkt_mb2 = _mm_add_epi16(pkt_mb2, crc_adjust);
+ pkt_mb1 = _mm_add_epi16(pkt_mb1, crc_adjust);
/* C.3 calc avaialbe number of desc */
staterr = _mm_and_si128(staterr, dd_check);
staterr = _mm_packs_epi32(staterr, zero);
/* D.3 copy final 1,2 data to rx_pkts */
- _mm_storeu_si128((__m128i *)&(rx_pkts[pos+1]->data_len),
+ _mm_storeu_si128((void *)&rx_pkts[pos+1]->rx_descriptor_fields1,
pkt_mb2);
- _mm_storeu_si128((__m128i *)&(rx_pkts[pos]->data_len),
+ _mm_storeu_si128((void *)&rx_pkts[pos]->rx_descriptor_fields1,
pkt_mb1);
/* C.4 calc avaialbe number of desc */
@@ -330,46 +319,19 @@ ixgbe_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
return nb_pkts_recd;
}
-
static inline void
vtx1(volatile union ixgbe_adv_tx_desc *txdp,
- struct rte_mbuf *pkt, __m128i flags)
+ struct rte_mbuf *pkt, uint64_t flags)
{
- __m128i t0, t1, offset, ols, ba, ctl;
-
- /* load buf_addr/buf_physaddr in t0 */
- t0 = _mm_loadu_si128((__m128i *)&(pkt->buf_addr));
- /* load data, ... pkt_len in t1 */
- t1 = _mm_loadu_si128((__m128i *)&(pkt->data));
-
- /* calc offset = (data - buf_adr) */
- offset = _mm_sub_epi64(t1, t0);
-
- /* cmd_type_len: pkt_len |= DCMD_DTYP_FLAGS */
- ctl = _mm_or_si128(t1, flags);
-
- /* reorder as buf_physaddr/buf_addr */
- offset = _mm_shuffle_epi32(offset, 0x4E);
-
- /* olinfo_stats: pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT */
- ols = _mm_slli_epi32(t1, IXGBE_ADVTXD_PAYLEN_SHIFT);
-
- /* buffer_addr = buf_physaddr + offset */
- ba = _mm_add_epi64(t0, offset);
-
- /* format cmd_type_len/olinfo_status */
- ctl = _mm_unpackhi_epi32(ctl, ols);
-
- /* format buf_physaddr/cmd_type_len */
- ba = _mm_unpackhi_epi64(ba, ctl);
-
- /* write desc */
- _mm_store_si128((__m128i *)&txdp->read, ba);
+ __m128i descriptor = _mm_set_epi64x((uint64_t)pkt->pkt_len << 46 |
+ flags | pkt->data_len,
+ pkt->buf_physaddr + pkt->data_off);
+ _mm_store_si128((__m128i *)&txdp->read, descriptor);
}
static inline void
vtx(volatile union ixgbe_adv_tx_desc *txdp,
- struct rte_mbuf **pkt, uint16_t nb_pkts, __m128i flags)
+ struct rte_mbuf **pkt, uint16_t nb_pkts, uint64_t flags)
{
int i;
for (i = 0; i < nb_pkts; ++i, ++txdp, ++pkt)
@@ -456,9 +418,8 @@ ixgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
struct igb_tx_entry_v *txep;
struct igb_tx_entry_seq *txsp;
uint16_t n, nb_commit, tx_id;
- __m128i flags = _mm_set_epi32(DCMD_DTYP_FLAGS, 0, 0, 0);
- __m128i rs = _mm_set_epi32(IXGBE_ADVTXD_DCMD_RS|DCMD_DTYP_FLAGS,
- 0, 0, 0);
+ uint64_t flags = DCMD_DTYP_FLAGS;
+ uint64_t rs = IXGBE_ADVTXD_DCMD_RS|DCMD_DTYP_FLAGS;
int i;
if (unlikely(nb_pkts > RTE_IXGBE_VPMD_TX_BURST))
@@ -610,6 +571,23 @@ static struct ixgbe_txq_ops vec_txq_ops = {
.reset = ixgbe_reset_tx_queue,
};
+int
+ixgbe_rxq_vec_setup(struct igb_rx_queue *rxq)
+{
+ static struct rte_mbuf mb_def = {
+ .nb_segs = 1,
+ .data_off = RTE_PKTMBUF_HEADROOM,
+#ifdef RTE_MBUF_REFCNT
+ .refcnt = 1,
+#endif
+ };
+
+ mb_def.buf_len = rxq->mb_pool->elt_size - sizeof(struct rte_mbuf);
+ mb_def.port = rxq->port_id;
+ rxq->mbuf_initializer = *((uint64_t *)&mb_def.rearm_data);
+ return 0;
+}
+
int ixgbe_txq_vec_setup(struct igb_tx_queue *txq,
unsigned int socket_id)
{
@@ -637,21 +615,6 @@ int ixgbe_txq_vec_setup(struct igb_tx_queue *txq,
return 0;
}
-int ixgbe_rxq_vec_setup(struct igb_rx_queue *rxq,
- __rte_unused unsigned int socket_id)
-{
- rxq->misc_info =
- _mm_set_epi16(
- 0, 0, 0, 0, 0,
- (uint16_t)-rxq->crc_len, /* sub crc on pkt_len */
- (uint16_t)(rxq->port_id << 8 | 1),
- /* 8b port_id and 8b nb_seg*/
- (uint16_t)-rxq->crc_len /* sub crc on data_len */
- );
-
- return 0;
-}
-
int ixgbe_rx_vec_condition_check(struct rte_eth_dev *dev)
{
#ifndef RTE_LIBRTE_IEEE1588
--
1.9.3
next prev parent reply other threads:[~2014-09-11 13:11 UTC|newest]
Thread overview: 62+ messages / expand[flat|nested] mbox.gz Atom feed top
2014-09-03 15:49 [dpdk-dev] [PATCH 00/13] Mbuf Structure Rework, part 2 Bruce Richardson
2014-09-03 15:49 ` [dpdk-dev] [PATCH 01/13] mbuf: replace data pointer by an offset Bruce Richardson
2014-09-08 9:52 ` Olivier MATZ
2014-09-08 9:55 ` Olivier MATZ
2014-09-03 15:49 ` [dpdk-dev] [PATCH 02/13] mbuf: reorder fields by time of use Bruce Richardson
2014-09-08 10:17 ` Olivier MATZ
2014-09-03 15:49 ` [dpdk-dev] [PATCH 03/13] mbuf: add packet_type field Bruce Richardson
2014-09-08 10:17 ` Olivier MATZ
2014-09-08 10:33 ` Yerden Zhumabekov
2014-09-08 11:17 ` Olivier MATZ
2014-09-09 3:59 ` Zhang, Helin
[not found] ` <540EB428.9060706@6wind.com>
2014-09-09 8:45 ` Zhang, Helin
2014-09-09 9:47 ` Richardson, Bruce
2014-09-09 15:05 ` Jim Thompson
2014-09-03 15:49 ` [dpdk-dev] [PATCH 04/13] mbuf: expand ol_flags field to 64-bits Bruce Richardson
2014-09-08 10:25 ` Olivier MATZ
2014-09-09 9:00 ` Richardson, Bruce
2014-09-03 15:49 ` [dpdk-dev] [PATCH 05/13] mbuf: introduce a flag to indicate a control mbuf Bruce Richardson
2014-09-08 11:53 ` Olivier MATZ
2014-09-03 15:49 ` [dpdk-dev] [PATCH 06/13] mbuf: minor changes for readability Bruce Richardson
2014-09-08 12:03 ` Olivier MATZ
2014-09-03 15:49 ` [dpdk-dev] [PATCH 07/13] mbuf: use macros only to access the mbuf metadata Bruce Richardson
2014-09-08 12:05 ` Olivier MATZ
2014-09-09 9:01 ` Richardson, Bruce
2014-09-12 16:56 ` Dumitrescu, Cristian
2014-09-12 21:02 ` Olivier MATZ
2014-09-16 20:07 ` Dumitrescu, Cristian
2014-09-16 22:06 ` Ramia, Kannan Babu
2014-09-17 10:31 ` Richardson, Bruce
2014-09-17 14:01 ` Thomas Monjalon
2014-09-10 15:09 ` Bruce Richardson
2014-09-10 15:31 ` Olivier MATZ
2014-09-03 15:49 ` [dpdk-dev] [PATCH 08/13] mbuf: add named points inside the mbuf structure Bruce Richardson
2014-09-08 12:08 ` Olivier MATZ
2014-09-03 15:49 ` [dpdk-dev] [PATCH 09/13] ixgbe: rework vector pmd following mbuf changes Bruce Richardson
2014-09-03 15:49 ` [dpdk-dev] [PATCH 10/13] mbuf: split mbuf across two cache lines Bruce Richardson
2014-09-08 12:10 ` Olivier MATZ
2014-09-03 15:49 ` [dpdk-dev] [PATCH 11/13] mbuf: move l2_len and l3_len to second cache line Bruce Richardson
2014-09-04 5:08 ` Yerden Zhumabekov
2014-09-04 10:27 ` Bruce Richardson
2014-09-04 11:00 ` Yerden Zhumabekov
2014-09-04 11:55 ` Bruce Richardson
2014-09-03 15:49 ` [dpdk-dev] [PATCH 12/13] ixgbe: Fix perf regression due to moved pool ptr Bruce Richardson
2014-09-03 15:49 ` [dpdk-dev] [PATCH 13/13] ixgbe: Improve slow-path perf: vector scattered RX Bruce Richardson
2014-09-11 13:15 ` [dpdk-dev] [PATCH v2 00/13] Mbuf Structure Rework, part 2 Bruce Richardson
2014-09-11 13:15 ` [dpdk-dev] [PATCH v2 01/13] mbuf: replace data pointer by an offset Bruce Richardson
2014-09-11 13:15 ` [dpdk-dev] [PATCH v2 02/13] mbuf: reorder fields by time of use Bruce Richardson
2014-09-15 7:11 ` Liu, Jijiang
2014-09-15 8:19 ` Richardson, Bruce
2014-09-11 13:15 ` [dpdk-dev] [PATCH v2 03/13] mbuf: expand ol_flags field to 64-bits Bruce Richardson
2014-09-11 13:15 ` [dpdk-dev] [PATCH v2 04/13] mbuf: introduce a flag to indicate a control mbuf Bruce Richardson
2014-09-11 13:15 ` [dpdk-dev] [PATCH v2 05/13] mbuf: minor changes for readability Bruce Richardson
2014-09-11 13:15 ` [dpdk-dev] [PATCH v2 06/13] mbuf: use macros only to access the mbuf metadata Bruce Richardson
2014-09-11 13:15 ` [dpdk-dev] [PATCH v2 07/13] mbuf: move metadata macros to rte_port library Bruce Richardson
2014-09-11 13:15 ` [dpdk-dev] [PATCH v2 08/13] mbuf: add named points inside the mbuf structure Bruce Richardson
2014-09-11 13:15 ` Bruce Richardson [this message]
2014-09-11 13:15 ` [dpdk-dev] [PATCH v2 10/13] mbuf: split mbuf across two cache lines Bruce Richardson
2014-09-11 13:15 ` [dpdk-dev] [PATCH v2 11/13] mbuf: move l2_len and l3_len to second cache line Bruce Richardson
2014-09-11 13:15 ` [dpdk-dev] [PATCH v2 12/13] ixgbe: Fix perf regression due to moved pool ptr Bruce Richardson
2014-09-15 16:20 ` [dpdk-dev] [PATCH v3 " Bruce Richardson
2014-09-11 13:15 ` [dpdk-dev] [PATCH v2 13/13] ixgbe: Improve slow-path perf: vector scattered RX Bruce Richardson
2014-09-17 22:35 ` [dpdk-dev] [PATCH v2 00/13] Mbuf Structure Rework, part 2 Thomas Monjalon
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1410441347-22840-10-git-send-email-bruce.richardson@intel.com \
--to=bruce.richardson@intel.com \
--cc=dev@dpdk.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).