[PATCH v3] mbuf: deprecate GCC marker in rte mbuf struct

[Tyler Retzlaff <roretzla@linux.microsoft.com>]

Provide a macro that allows conditional expansion of RTE_MARKER fields
to empty to allow rte_mbuf to be used with MSVC. It is proposed that
we announce the fields to be __rte_deprecated (currently disabled).

Introduce C11 anonymous unions to permit aliasing of well-known
offsets by name into the rte_mbuf structure by a *new* name and to
provide padding for cache alignment.

Signed-off-by: Tyler Retzlaff <roretzla@linux.microsoft.com>
---
 lib/eal/include/rte_common.h |   6 +
 lib/mbuf/rte_mbuf_core.h     | 365 +++++++++++++++++++++++--------------------
 2 files changed, 201 insertions(+), 170 deletions(-)

diff --git a/lib/eal/include/rte_common.h b/lib/eal/include/rte_common.h
index d7d6390..af73f67 100644
--- a/lib/eal/include/rte_common.h
+++ b/lib/eal/include/rte_common.h
@@ -582,6 +582,12 @@ static void __attribute__((destructor(RTE_PRIO(prio)), used)) func(void)
 /** Marker for 8B alignment in a structure. */
 __extension__ typedef uint64_t RTE_MARKER64[0];
 
+#define __rte_marker(type, name) type name /* __rte_deprecated */
+
+#else
+
+#define __rte_marker(type, name)
+
 #endif
 
 /*********** Macros for calculating min and max **********/
diff --git a/lib/mbuf/rte_mbuf_core.h b/lib/mbuf/rte_mbuf_core.h
index 5688683..fd02353 100644
--- a/lib/mbuf/rte_mbuf_core.h
+++ b/lib/mbuf/rte_mbuf_core.h
@@ -16,7 +16,10 @@
  * New fields and flags should fit in the "dynamic space".
  */
 
+#include <assert.h>
+#include <stdalign.h>
 #include <stdint.h>
+#include <stddef.h>
 
 #include <rte_byteorder.h>
 #include <rte_stdatomic.h>
@@ -464,206 +467,228 @@ enum {
  * The generic rte_mbuf, containing a packet mbuf.
  */
 struct rte_mbuf {
-	RTE_MARKER cacheline0;
-
-	void *buf_addr;           /**< Virtual address of segment buffer. */
+	__rte_marker(RTE_MARKER, cacheline0);
+	union {
+		char mbuf_cacheline0[RTE_CACHE_LINE_MIN_SIZE];
+		struct {
+			void *buf_addr;           /**< Virtual address of segment buffer. */
 #if RTE_IOVA_IN_MBUF
-	/**
-	 * Physical address of segment buffer.
-	 * This field is undefined if the build is configured to use only
-	 * virtual address as IOVA (i.e. RTE_IOVA_IN_MBUF is 0).
-	 * Force alignment to 8-bytes, so as to ensure we have the exact
-	 * same mbuf cacheline0 layout for 32-bit and 64-bit. This makes
-	 * working on vector drivers easier.
-	 */
-	rte_iova_t buf_iova __rte_aligned(sizeof(rte_iova_t));
+			/**
+			 * Physical address of segment buffer.
+			 * This field is undefined if the build is configured to use only
+			 * virtual address as IOVA (i.e. RTE_IOVA_IN_MBUF is 0).
+			 * Force alignment to 8-bytes, so as to ensure we have the exact
+			 * same mbuf cacheline0 layout for 32-bit and 64-bit. This makes
+			 * working on vector drivers easier.
+			 */
+			rte_iova_t buf_iova __rte_aligned(sizeof(rte_iova_t));
 #else
-	/**
-	 * Next segment of scattered packet.
-	 * This field is valid when physical address field is undefined.
-	 * Otherwise next pointer in the second cache line will be used.
-	 */
-	struct rte_mbuf *next;
+			/**
+			 * Next segment of scattered packet.
+			 * This field is valid when physical address field is undefined.
+			 * Otherwise next pointer in the second cache line will be used.
+			 */
+			struct rte_mbuf *next;
 #endif
 
-	/* next 8 bytes are initialised on RX descriptor rearm */
-	RTE_MARKER64 rearm_data;
-	uint16_t data_off;
-
-	/**
-	 * Reference counter. Its size should at least equal to the size
-	 * of port field (16 bits), to support zero-copy broadcast.
-	 * It should only be accessed using the following functions:
-	 * rte_mbuf_refcnt_update(), rte_mbuf_refcnt_read(), and
-	 * rte_mbuf_refcnt_set(). The functionality of these functions (atomic,
-	 * or non-atomic) is controlled by the RTE_MBUF_REFCNT_ATOMIC flag.
-	 */
-	RTE_ATOMIC(uint16_t) refcnt;
-
-	/**
-	 * Number of segments. Only valid for the first segment of an mbuf
-	 * chain.
-	 */
-	uint16_t nb_segs;
-
-	/** Input port (16 bits to support more than 256 virtual ports).
-	 * The event eth Tx adapter uses this field to specify the output port.
-	 */
-	uint16_t port;
-
-	uint64_t ol_flags;        /**< Offload features. */
+			/* next 8 bytes are initialised on RX descriptor rearm */
+			__rte_marker(RTE_MARKER64, rearm_data);
+			union {
+				char mbuf_rearm_data[8];
+				struct {
+					uint16_t data_off;
+
+					/**
+					 * Reference counter. Its size should at least equal to the size
+					 * of port field (16 bits), to support zero-copy broadcast.
+					 * It should only be accessed using the following functions:
+					 * rte_mbuf_refcnt_update(), rte_mbuf_refcnt_read(), and
+					 * rte_mbuf_refcnt_set(). The functionality of these functions (atomic,
+					 * or non-atomic) is controlled by the RTE_MBUF_REFCNT_ATOMIC flag.
+					 */
+					RTE_ATOMIC(uint16_t) refcnt;
+
+					/**
+					 * Number of segments. Only valid for the first segment of an mbuf
+					 * chain.
+					 */
+					uint16_t nb_segs;
+
+					/** Input port (16 bits to support more than 256 virtual ports).
+					 * The event eth Tx adapter uses this field to specify the output port.
+					 */
+					uint16_t port;
+				};
+			};
 
-	/* remaining bytes are set on RX when pulling packet from descriptor */
-	RTE_MARKER rx_descriptor_fields1;
+			uint64_t ol_flags;        /**< Offload features. */
 
-	/*
-	 * The packet type, which is the combination of outer/inner L2, L3, L4
-	 * and tunnel types. The packet_type is about data really present in the
-	 * mbuf. Example: if vlan stripping is enabled, a received vlan packet
-	 * would have RTE_PTYPE_L2_ETHER and not RTE_PTYPE_L2_VLAN because the
-	 * vlan is stripped from the data.
-	 */
-	union {
-		uint32_t packet_type; /**< L2/L3/L4 and tunnel information. */
-		__extension__
-		struct {
-			uint8_t l2_type:4;   /**< (Outer) L2 type. */
-			uint8_t l3_type:4;   /**< (Outer) L3 type. */
-			uint8_t l4_type:4;   /**< (Outer) L4 type. */
-			uint8_t tun_type:4;  /**< Tunnel type. */
+			/* remaining bytes are set on RX when pulling packet from descriptor */
+			__rte_marker(RTE_MARKER, rx_descriptor_fields1);
 			union {
-				uint8_t inner_esp_next_proto;
-				/**< ESP next protocol type, valid if
-				 * RTE_PTYPE_TUNNEL_ESP tunnel type is set
-				 * on both Tx and Rx.
-				 */
-				__extension__
+				char mbuf_rx_descriptor_fields1[sizeof(void *)];
 				struct {
-					uint8_t inner_l2_type:4;
-					/**< Inner L2 type. */
-					uint8_t inner_l3_type:4;
-					/**< Inner L3 type. */
+					/*
+					 * The packet type, which is the combination of outer/inner L2, L3, L4
+					 * and tunnel types. The packet_type is about data really present in the
+					 * mbuf. Example: if vlan stripping is enabled, a received vlan packet
+					 * would have RTE_PTYPE_L2_ETHER and not RTE_PTYPE_L2_VLAN because the
+					 * vlan is stripped from the data.
+					 */
+					union {
+						uint32_t packet_type; /**< L2/L3/L4 and tunnel information. */
+						__extension__
+						struct {
+							uint8_t l2_type:4;   /**< (Outer) L2 type. */
+							uint8_t l3_type:4;   /**< (Outer) L3 type. */
+							uint8_t l4_type:4;   /**< (Outer) L4 type. */
+							uint8_t tun_type:4;  /**< Tunnel type. */
+							union {
+								uint8_t inner_esp_next_proto;
+								/**< ESP next protocol type, valid if
+								 * RTE_PTYPE_TUNNEL_ESP tunnel type is set
+								 * on both Tx and Rx.
+								 */
+								__extension__
+								struct {
+									uint8_t inner_l2_type:4;
+									/**< Inner L2 type. */
+									uint8_t inner_l3_type:4;
+									/**< Inner L3 type. */
+								};
+							};
+							uint8_t inner_l4_type:4; /**< Inner L4 type. */
+						};
+					};
+					uint32_t pkt_len;         /**< Total pkt len: sum of all segments. */
 				};
 			};
-			uint8_t inner_l4_type:4; /**< Inner L4 type. */
-		};
-	};
 
-	uint32_t pkt_len;         /**< Total pkt len: sum of all segments. */
-	uint16_t data_len;        /**< Amount of data in segment buffer. */
-	/** VLAN TCI (CPU order), valid if RTE_MBUF_F_RX_VLAN is set. */
-	uint16_t vlan_tci;
+			uint16_t data_len;        /**< Amount of data in segment buffer. */
+			/** VLAN TCI (CPU order), valid if RTE_MBUF_F_RX_VLAN is set. */
+			uint16_t vlan_tci;
 
-	union {
-		union {
-			uint32_t rss;     /**< RSS hash result if RSS enabled */
-			struct {
+			union {
 				union {
+					uint32_t rss;     /**< RSS hash result if RSS enabled */
 					struct {
-						uint16_t hash;
-						uint16_t id;
-					};
-					uint32_t lo;
-					/**< Second 4 flexible bytes */
-				};
-				uint32_t hi;
-				/**< First 4 flexible bytes or FD ID, dependent
-				 * on RTE_MBUF_F_RX_FDIR_* flag in ol_flags.
-				 */
-			} fdir;	/**< Filter identifier if FDIR enabled */
-			struct rte_mbuf_sched sched;
-			/**< Hierarchical scheduler : 8 bytes */
-			struct {
-				uint32_t reserved1;
-				uint16_t reserved2;
-				uint16_t txq;
-				/**< The event eth Tx adapter uses this field
-				 * to store Tx queue id.
-				 * @see rte_event_eth_tx_adapter_txq_set()
-				 */
-			} txadapter; /**< Eventdev ethdev Tx adapter */
-			uint32_t usr;
-			/**< User defined tags. See rte_distributor_process() */
-		} hash;                   /**< hash information */
-	};
+						union {
+							struct {
+								uint16_t hash;
+								uint16_t id;
+							};
+							uint32_t lo;
+							/**< Second 4 flexible bytes */
+						};
+						uint32_t hi;
+						/**< First 4 flexible bytes or FD ID, dependent
+						 * on RTE_MBUF_F_RX_FDIR_* flag in ol_flags.
+						 */
+					} fdir;	/**< Filter identifier if FDIR enabled */
+					struct rte_mbuf_sched sched;
+					/**< Hierarchical scheduler : 8 bytes */
+					struct {
+						uint32_t reserved1;
+						uint16_t reserved2;
+						uint16_t txq;
+						/**< The event eth Tx adapter uses this field
+						 * to store Tx queue id.
+						 * @see rte_event_eth_tx_adapter_txq_set()
+						 */
+					} txadapter; /**< Eventdev ethdev Tx adapter */
+					uint32_t usr;
+					/**< User defined tags. See rte_distributor_process() */
+				} hash;                   /**< hash information */
+			};
 
-	/** Outer VLAN TCI (CPU order), valid if RTE_MBUF_F_RX_QINQ is set. */
-	uint16_t vlan_tci_outer;
+			/** Outer VLAN TCI (CPU order), valid if RTE_MBUF_F_RX_QINQ is set. */
+			uint16_t vlan_tci_outer;
 
-	uint16_t buf_len;         /**< Length of segment buffer. */
+			uint16_t buf_len;         /**< Length of segment buffer. */
 
-	struct rte_mempool *pool; /**< Pool from which mbuf was allocated. */
+			struct rte_mempool *pool; /**< Pool from which mbuf was allocated. */
+		};
+	};
 
 	/* second cache line - fields only used in slow path or on TX */
-	RTE_MARKER cacheline1 __rte_cache_min_aligned;
-
-#if RTE_IOVA_IN_MBUF
-	/**
-	 * Next segment of scattered packet. Must be NULL in the last
-	 * segment or in case of non-segmented packet.
-	 */
-	struct rte_mbuf *next;
-#else
-	/**
-	 * Reserved for dynamic fields
-	 * when the next pointer is in first cache line (i.e. RTE_IOVA_IN_MBUF is 0).
-	 */
-	uint64_t dynfield2;
-#endif
-
-	/* fields to support TX offloads */
+	__rte_marker(RTE_MARKER, cacheline1);
 	union {
-		uint64_t tx_offload;       /**< combined for easy fetch */
-		__extension__
+		char mbuf_cacheline1[RTE_CACHE_LINE_MIN_SIZE];
 		struct {
-			uint64_t l2_len:RTE_MBUF_L2_LEN_BITS;
-			/**< L2 (MAC) Header Length for non-tunneling pkt.
-			 * Outer_L4_len + ... + Inner_L2_len for tunneling pkt.
+#if RTE_IOVA_IN_MBUF
+			/**
+			 * Next segment of scattered packet. Must be NULL in the last
+			 * segment or in case of non-segmented packet.
 			 */
-			uint64_t l3_len:RTE_MBUF_L3_LEN_BITS;
-			/**< L3 (IP) Header Length. */
-			uint64_t l4_len:RTE_MBUF_L4_LEN_BITS;
-			/**< L4 (TCP/UDP) Header Length. */
-			uint64_t tso_segsz:RTE_MBUF_TSO_SEGSZ_BITS;
-			/**< TCP TSO segment size */
-
-			/*
-			 * Fields for Tx offloading of tunnels.
-			 * These are undefined for packets which don't request
-			 * any tunnel offloads (outer IP or UDP checksum,
-			 * tunnel TSO).
-			 *
-			 * PMDs should not use these fields unconditionally
-			 * when calculating offsets.
-			 *
-			 * Applications are expected to set appropriate tunnel
-			 * offload flags when they fill in these fields.
+			struct rte_mbuf *next;
+#else
+			/**
+			 * Reserved for dynamic fields
+			 * when the next pointer is in first cache line (i.e. RTE_IOVA_IN_MBUF is 0).
 			 */
-			uint64_t outer_l3_len:RTE_MBUF_OUTL3_LEN_BITS;
-			/**< Outer L3 (IP) Hdr Length. */
-			uint64_t outer_l2_len:RTE_MBUF_OUTL2_LEN_BITS;
-			/**< Outer L2 (MAC) Hdr Length. */
+			uint64_t dynfield2;
+#endif
 
-			/* uint64_t unused:RTE_MBUF_TXOFLD_UNUSED_BITS; */
-		};
-	};
+			/* fields to support TX offloads */
+			union {
+				uint64_t tx_offload;       /**< combined for easy fetch */
+				__extension__
+				struct {
+					uint64_t l2_len:RTE_MBUF_L2_LEN_BITS;
+					/**< L2 (MAC) Header Length for non-tunneling pkt.
+					 * Outer_L4_len + ... + Inner_L2_len for tunneling pkt.
+					 */
+					uint64_t l3_len:RTE_MBUF_L3_LEN_BITS;
+					/**< L3 (IP) Header Length. */
+					uint64_t l4_len:RTE_MBUF_L4_LEN_BITS;
+					/**< L4 (TCP/UDP) Header Length. */
+					uint64_t tso_segsz:RTE_MBUF_TSO_SEGSZ_BITS;
+					/**< TCP TSO segment size */
+
+					/*
+					 * Fields for Tx offloading of tunnels.
+					 * These are undefined for packets which don't request
+					 * any tunnel offloads (outer IP or UDP checksum,
+					 * tunnel TSO).
+					 *
+					 * PMDs should not use these fields unconditionally
+					 * when calculating offsets.
+					 *
+					 * Applications are expected to set appropriate tunnel
+					 * offload flags when they fill in these fields.
+					 */
+					uint64_t outer_l3_len:RTE_MBUF_OUTL3_LEN_BITS;
+					/**< Outer L3 (IP) Hdr Length. */
+					uint64_t outer_l2_len:RTE_MBUF_OUTL2_LEN_BITS;
+					/**< Outer L2 (MAC) Hdr Length. */
+
+					/* uint64_t unused:RTE_MBUF_TXOFLD_UNUSED_BITS; */
+				};
+			};
 
-	/** Shared data for external buffer attached to mbuf. See
-	 * rte_pktmbuf_attach_extbuf().
-	 */
-	struct rte_mbuf_ext_shared_info *shinfo;
+			/** Shared data for external buffer attached to mbuf. See
+			 * rte_pktmbuf_attach_extbuf().
+			 */
+			struct rte_mbuf_ext_shared_info *shinfo;
 
-	/** Size of the application private data. In case of an indirect
-	 * mbuf, it stores the direct mbuf private data size.
-	 */
-	uint16_t priv_size;
+			/** Size of the application private data. In case of an indirect
+			 * mbuf, it stores the direct mbuf private data size.
+			 */
+			uint16_t priv_size;
 
-	/** Timesync flags for use with IEEE1588. */
-	uint16_t timesync;
+			/** Timesync flags for use with IEEE1588. */
+			uint16_t timesync;
 
-	uint32_t dynfield1[9]; /**< Reserved for dynamic fields. */
+			uint32_t dynfield1[9]; /**< Reserved for dynamic fields. */
+		};
+	};
 } __rte_cache_aligned;
 
+static_assert(alignof(struct rte_mbuf) == RTE_CACHE_LINE_SIZE, "alignof rte_mbuf");
+static_assert(offsetof(struct rte_mbuf, cacheline0) == 0, "cacheline0");
+static_assert(offsetof(struct rte_mbuf, cacheline1) == RTE_CACHE_LINE_MIN_SIZE, "cacheline1");
+static_assert(sizeof(struct rte_mbuf) == RTE_CACHE_LINE_MIN_SIZE * 2, "sizeof rte_mbuf");
+
 /**
  * Function typedef of callback to free externally attached buffer.
  */
-- 
1.8.3.1