* [dpdk-dev] [PATCH 5/8] lib: remove dead code from timer
2019-09-30 9:21 8% [dpdk-dev] [PATCH 1/8] config: change ABI versioning for global Marcin Baran
` (2 preceding siblings ...)
2019-09-30 9:21 1% ` [dpdk-dev] [PATCH 4/8] build: change ABI version to 20.0 Marcin Baran
@ 2019-09-30 9:21 2% ` Marcin Baran
3 siblings, 0 replies; 200+ results
From: Marcin Baran @ 2019-09-30 9:21 UTC (permalink / raw)
To: dev, bruce.richardson, ray.kinsella; +Cc: Marcin Baran
After updating ABI policy old and unused
code needs to be removed and all libraries
symbols version should be set to v20.
Signed-off-by: Marcin Baran <marcinx.baran@intel.com>
---
lib/librte_timer/rte_timer.c | 85 +++---------------------------------
lib/librte_timer/rte_timer.h | 15 -------
2 files changed, 5 insertions(+), 95 deletions(-)
diff --git a/lib/librte_timer/rte_timer.c b/lib/librte_timer/rte_timer.c
index 34c864b60..de6959b80 100644
--- a/lib/librte_timer/rte_timer.c
+++ b/lib/librte_timer/rte_timer.c
@@ -68,9 +68,6 @@ static struct rte_timer_data *rte_timer_data_arr;
static const uint32_t default_data_id;
static uint32_t rte_timer_subsystem_initialized;
-/* For maintaining older interfaces for a period */
-static struct rte_timer_data default_timer_data;
-
/* when debug is enabled, store some statistics */
#ifdef RTE_LIBRTE_TIMER_DEBUG
#define __TIMER_STAT_ADD(priv_timer, name, n) do { \
@@ -131,21 +128,6 @@ rte_timer_data_dealloc(uint32_t id)
return 0;
}
-void
-rte_timer_subsystem_init_v20(void)
-{
- unsigned lcore_id;
- struct priv_timer *priv_timer = default_timer_data.priv_timer;
-
- /* since priv_timer is static, it's zeroed by default, so only init some
- * fields.
- */
- for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id ++) {
- rte_spinlock_init(&priv_timer[lcore_id].list_lock);
- priv_timer[lcore_id].prev_lcore = lcore_id;
- }
-}
-
/* Init the timer library. Allocate an array of timer data structs in shared
* memory, and allocate the zeroth entry for use with original timer
* APIs. Since the intersection of the sets of lcore ids in primary and
@@ -153,7 +135,7 @@ rte_timer_subsystem_init_v20(void)
* multiple processes.
*/
int
-rte_timer_subsystem_init_v1905(void)
+rte_timer_subsystem_init(void)
{
const struct rte_memzone *mz;
struct rte_timer_data *data;
@@ -208,9 +190,6 @@ rte_timer_subsystem_init_v1905(void)
return 0;
}
-MAP_STATIC_SYMBOL(int rte_timer_subsystem_init(void),
- rte_timer_subsystem_init_v1905);
-BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 20.0);
void
rte_timer_subsystem_finalize(void)
@@ -551,41 +530,13 @@ __rte_timer_reset(struct rte_timer *tim, uint64_t expire,
/* Reset and start the timer associated with the timer handle tim */
int
-rte_timer_reset_v20(struct rte_timer *tim, uint64_t ticks,
- enum rte_timer_type type, unsigned int tim_lcore,
- rte_timer_cb_t fct, void *arg)
-{
- uint64_t cur_time = rte_get_timer_cycles();
- uint64_t period;
-
- if (unlikely((tim_lcore != (unsigned)LCORE_ID_ANY) &&
- !(rte_lcore_is_enabled(tim_lcore) ||
- rte_lcore_has_role(tim_lcore, ROLE_SERVICE))))
- return -1;
-
- if (type == PERIODICAL)
- period = ticks;
- else
- period = 0;
-
- return __rte_timer_reset(tim, cur_time + ticks, period, tim_lcore,
- fct, arg, 0, &default_timer_data);
-}
-
-int
-rte_timer_reset_v1905(struct rte_timer *tim, uint64_t ticks,
+rte_timer_reset(struct rte_timer *tim, uint64_t ticks,
enum rte_timer_type type, unsigned int tim_lcore,
rte_timer_cb_t fct, void *arg)
{
return rte_timer_alt_reset(default_data_id, tim, ticks, type,
tim_lcore, fct, arg);
}
-MAP_STATIC_SYMBOL(int rte_timer_reset(struct rte_timer *tim, uint64_t ticks,
- enum rte_timer_type type,
- unsigned int tim_lcore,
- rte_timer_cb_t fct, void *arg),
- rte_timer_reset_v1905);
-BIND_DEFAULT_SYMBOL(rte_timer_reset, _v1905, 20.0);
int
rte_timer_alt_reset(uint32_t timer_data_id, struct rte_timer *tim,
@@ -656,19 +607,10 @@ __rte_timer_stop(struct rte_timer *tim, int local_is_locked,
/* Stop the timer associated with the timer handle tim */
int
-rte_timer_stop_v20(struct rte_timer *tim)
-{
- return __rte_timer_stop(tim, 0, &default_timer_data);
-}
-
-int
-rte_timer_stop_v1905(struct rte_timer *tim)
+rte_timer_stop(struct rte_timer *tim)
{
return rte_timer_alt_stop(default_data_id, tim);
}
-MAP_STATIC_SYMBOL(int rte_timer_stop(struct rte_timer *tim),
- rte_timer_stop_v1905);
-BIND_DEFAULT_SYMBOL(rte_timer_stop, _v1905, 20.0);
int
rte_timer_alt_stop(uint32_t timer_data_id, struct rte_timer *tim)
@@ -814,14 +756,8 @@ __rte_timer_manage(struct rte_timer_data *timer_data)
priv_timer[lcore_id].running_tim = NULL;
}
-void
-rte_timer_manage_v20(void)
-{
- __rte_timer_manage(&default_timer_data);
-}
-
int
-rte_timer_manage_v1905(void)
+rte_timer_manage(void)
{
struct rte_timer_data *timer_data;
@@ -831,8 +767,6 @@ rte_timer_manage_v1905(void)
return 0;
}
-MAP_STATIC_SYMBOL(int rte_timer_manage(void), rte_timer_manage_v1905);
-BIND_DEFAULT_SYMBOL(rte_timer_manage, _v1905, 20.0);
int
rte_timer_alt_manage(uint32_t timer_data_id,
@@ -1070,20 +1004,11 @@ __rte_timer_dump_stats(struct rte_timer_data *timer_data __rte_unused, FILE *f)
#endif
}
-void
-rte_timer_dump_stats_v20(FILE *f)
-{
- __rte_timer_dump_stats(&default_timer_data, f);
-}
-
int
-rte_timer_dump_stats_v1905(FILE *f)
+rte_timer_dump_stats(FILE *f)
{
return rte_timer_alt_dump_stats(default_data_id, f);
}
-MAP_STATIC_SYMBOL(int rte_timer_dump_stats(FILE *f),
- rte_timer_dump_stats_v1905);
-BIND_DEFAULT_SYMBOL(rte_timer_dump_stats, _v1905, 20.0);
int
rte_timer_alt_dump_stats(uint32_t timer_data_id __rte_unused, FILE *f)
diff --git a/lib/librte_timer/rte_timer.h b/lib/librte_timer/rte_timer.h
index 05d287d8f..9dc5fc309 100644
--- a/lib/librte_timer/rte_timer.h
+++ b/lib/librte_timer/rte_timer.h
@@ -181,8 +181,6 @@ int rte_timer_data_dealloc(uint32_t id);
* subsystem
*/
int rte_timer_subsystem_init(void);
-int rte_timer_subsystem_init_v1905(void);
-void rte_timer_subsystem_init_v20(void);
/**
* @warning
@@ -250,13 +248,6 @@ void rte_timer_init(struct rte_timer *tim);
int rte_timer_reset(struct rte_timer *tim, uint64_t ticks,
enum rte_timer_type type, unsigned tim_lcore,
rte_timer_cb_t fct, void *arg);
-int rte_timer_reset_v1905(struct rte_timer *tim, uint64_t ticks,
- enum rte_timer_type type, unsigned int tim_lcore,
- rte_timer_cb_t fct, void *arg);
-int rte_timer_reset_v20(struct rte_timer *tim, uint64_t ticks,
- enum rte_timer_type type, unsigned int tim_lcore,
- rte_timer_cb_t fct, void *arg);
-
/**
* Loop until rte_timer_reset() succeeds.
@@ -313,8 +304,6 @@ rte_timer_reset_sync(struct rte_timer *tim, uint64_t ticks,
* - (-1): The timer is in the RUNNING or CONFIG state.
*/
int rte_timer_stop(struct rte_timer *tim);
-int rte_timer_stop_v1905(struct rte_timer *tim);
-int rte_timer_stop_v20(struct rte_timer *tim);
/**
* Loop until rte_timer_stop() succeeds.
@@ -358,8 +347,6 @@ int rte_timer_pending(struct rte_timer *tim);
* - -EINVAL: timer subsystem not yet initialized
*/
int rte_timer_manage(void);
-int rte_timer_manage_v1905(void);
-void rte_timer_manage_v20(void);
/**
* Dump statistics about timers.
@@ -371,8 +358,6 @@ void rte_timer_manage_v20(void);
* - -EINVAL: timer subsystem not yet initialized
*/
int rte_timer_dump_stats(FILE *f);
-int rte_timer_dump_stats_v1905(FILE *f);
-void rte_timer_dump_stats_v20(FILE *f);
/**
* @warning
--
2.22.0.windows.1
^ permalink raw reply [relevance 2%]
* [dpdk-dev] [PATCH 4/8] build: change ABI version to 20.0
2019-09-30 9:21 8% [dpdk-dev] [PATCH 1/8] config: change ABI versioning for global Marcin Baran
2019-09-30 9:21 10% ` [dpdk-dev] [PATCH 2/8] buildtools: scripts for updating symbols abi version Marcin Baran
2019-09-30 9:21 23% ` [dpdk-dev] [PATCH 3/8] buildtools: add ABI versioning check script Marcin Baran
@ 2019-09-30 9:21 1% ` Marcin Baran
2019-09-30 9:21 2% ` [dpdk-dev] [PATCH 5/8] lib: remove dead code from timer Marcin Baran
3 siblings, 0 replies; 200+ results
From: Marcin Baran @ 2019-09-30 9:21 UTC (permalink / raw)
To: dev, bruce.richardson, ray.kinsella; +Cc: Pawel Modrak
From: Pawel Modrak <pawelx.modrak@intel.com>
Merge all vesions in linker version script files to DPDK_20.0.
Use version 20.0 as default version in source files.
Signed-off-by: Pawel Modrak <pawelx.modrak@intel.com>
---
.../rte_pmd_bbdev_fpga_lte_fec_version.map | 5 +-
.../null/rte_pmd_bbdev_null_version.map | 3 +-
.../rte_pmd_bbdev_turbo_sw_version.map | 3 +-
drivers/bus/dpaa/rte_bus_dpaa_version.map | 98 ++--
drivers/bus/fslmc/rte_bus_fslmc_version.map | 153 +++----
drivers/bus/ifpga/rte_bus_ifpga_version.map | 14 +-
drivers/bus/pci/rte_bus_pci_version.map | 4 +-
drivers/bus/vdev/rte_bus_vdev_version.map | 12 +-
drivers/bus/vmbus/rte_bus_vmbus_version.map | 12 +-
drivers/common/cpt/rte_common_cpt_version.map | 3 +-
.../common/dpaax/rte_common_dpaax_version.map | 6 +-
.../common/mvep/rte_common_mvep_version.map | 5 +-
.../octeontx/rte_common_octeontx_version.map | 5 +-
.../rte_common_octeontx2_version.map | 18 +-
.../compress/isal/rte_pmd_isal_version.map | 3 +-
.../rte_pmd_octeontx_compress_version.map | 3 +-
drivers/compress/qat/rte_pmd_qat_version.map | 3 +-
.../compress/zlib/rte_pmd_zlib_version.map | 3 +-
.../aesni_gcm/rte_pmd_aesni_gcm_version.map | 3 +-
.../aesni_mb/rte_pmd_aesni_mb_version.map | 3 +-
.../crypto/armv8/rte_pmd_armv8_version.map | 3 +-
.../caam_jr/rte_pmd_caam_jr_version.map | 2 +-
drivers/crypto/ccp/rte_pmd_ccp_version.map | 2 +-
.../dpaa2_sec/rte_pmd_dpaa2_sec_version.map | 10 +-
.../dpaa_sec/rte_pmd_dpaa_sec_version.map | 2 +-
.../crypto/kasumi/rte_pmd_kasumi_version.map | 3 +-
.../crypto/mvsam/rte_pmd_mvsam_version.map | 3 +-
.../null/rte_pmd_null_crypto_version.map | 3 +-
.../rte_pmd_octeontx_crypto_version.map | 2 +-
.../openssl/rte_pmd_openssl_version.map | 3 +-
.../rte_pmd_crypto_scheduler_version.map | 18 +-
.../crypto/snow3g/rte_pmd_snow3g_version.map | 3 +-
.../virtio/rte_pmd_virtio_crypto_version.map | 3 +-
drivers/crypto/zuc/rte_pmd_zuc_version.map | 3 +-
.../event/dpaa/rte_pmd_dpaa_event_version.map | 2 +-
.../dpaa2/rte_pmd_dpaa2_event_version.map | 3 +-
.../event/dsw/rte_pmd_dsw_event_version.map | 3 +-
.../rte_pmd_octeontx_event_version.map | 3 +-
.../rte_pmd_octeontx2_event_version.map | 4 +-
.../event/opdl/rte_pmd_opdl_event_version.map | 3 +-
.../rte_pmd_skeleton_event_version.map | 2 +-
drivers/event/sw/rte_pmd_sw_event_version.map | 3 +-
.../bucket/rte_mempool_bucket_version.map | 2 +-
.../mempool/dpaa/rte_mempool_dpaa_version.map | 4 +-
.../dpaa2/rte_mempool_dpaa2_version.map | 12 +-
.../octeontx/rte_mempool_octeontx_version.map | 3 +-
.../rte_mempool_octeontx2_version.map | 6 +-
.../mempool/ring/rte_mempool_ring_version.map | 2 +-
.../stack/rte_mempool_stack_version.map | 2 +-
.../af_packet/rte_pmd_af_packet_version.map | 2 +-
drivers/net/af_xdp/rte_pmd_af_xdp_version.map | 3 +-
drivers/net/ark/rte_pmd_ark_version.map | 4 +-
.../net/atlantic/rte_pmd_atlantic_version.map | 12 +-
drivers/net/avp/rte_pmd_avp_version.map | 3 +-
drivers/net/axgbe/rte_pmd_axgbe_version.map | 3 +-
drivers/net/bnx2x/rte_pmd_bnx2x_version.map | 2 +-
drivers/net/bnxt/rte_pmd_bnxt_version.map | 6 +-
drivers/net/bonding/rte_pmd_bond_version.map | 47 +-
drivers/net/cxgbe/rte_pmd_cxgbe_version.map | 2 +-
drivers/net/dpaa/rte_pmd_dpaa_version.map | 11 +-
drivers/net/dpaa2/rte_pmd_dpaa2_version.map | 13 +-
drivers/net/e1000/rte_pmd_e1000_version.map | 2 +-
drivers/net/ena/rte_pmd_ena_version.map | 2 +-
drivers/net/enetc/rte_pmd_enetc_version.map | 2 +-
drivers/net/enic/rte_pmd_enic_version.map | 2 +-
.../net/failsafe/rte_pmd_failsafe_version.map | 2 +-
drivers/net/fm10k/rte_pmd_fm10k_version.map | 2 +-
drivers/net/hinic/rte_pmd_hinic_version.map | 2 +-
drivers/net/i40e/rte_pmd_i40e_version.map | 65 +--
drivers/net/iavf/rte_pmd_iavf_version.map | 2 +-
drivers/net/ice/rte_pmd_ice_version.map | 2 +-
drivers/net/ifc/rte_pmd_ifc_version.map | 2 +-
drivers/net/ipn3ke/rte_pmd_ipn3ke_version.map | 2 +-
drivers/net/ixgbe/rte_pmd_ixgbe_version.map | 63 +--
drivers/net/kni/rte_pmd_kni_version.map | 2 +-
.../net/liquidio/rte_pmd_liquidio_version.map | 2 +-
drivers/net/memif/rte_pmd_memif_version.map | 4 +-
drivers/net/mlx4/rte_pmd_mlx4_version.map | 2 +-
drivers/net/mlx5/rte_pmd_mlx5_version.map | 3 +-
drivers/net/mvneta/rte_pmd_mvneta_version.map | 3 +-
drivers/net/mvpp2/rte_pmd_mvpp2_version.map | 3 +-
drivers/net/netvsc/rte_pmd_netvsc_version.map | 3 +-
drivers/net/nfb/rte_pmd_nfb_version.map | 2 +-
drivers/net/nfp/rte_pmd_nfp_version.map | 3 +-
drivers/net/null/rte_pmd_null_version.map | 2 +-
.../net/octeontx/rte_pmd_octeontx_version.map | 10 +-
.../octeontx2/rte_pmd_octeontx2_version.map | 2 +-
drivers/net/pcap/rte_pmd_pcap_version.map | 2 +-
drivers/net/qede/rte_pmd_qede_version.map | 2 +-
drivers/net/ring/rte_pmd_ring_version.map | 10 +-
drivers/net/sfc/rte_pmd_sfc_version.map | 2 +-
.../net/softnic/rte_pmd_softnic_version.map | 3 +-
.../net/szedata2/rte_pmd_szedata2_version.map | 3 +-
drivers/net/tap/rte_pmd_tap_version.map | 2 +-
.../net/thunderx/rte_pmd_thunderx_version.map | 2 +-
.../rte_pmd_vdev_netvsc_version.map | 2 +-
drivers/net/vhost/rte_pmd_vhost_version.map | 9 +-
drivers/net/virtio/rte_pmd_virtio_version.map | 2 +-
.../net/vmxnet3/rte_pmd_vmxnet3_version.map | 2 +-
.../rte_rawdev_dpaa2_cmdif_version.map | 2 +-
.../rte_rawdev_dpaa2_qdma_version.map | 6 +-
.../raw/ifpga/rte_rawdev_ifpga_version.map | 2 +-
drivers/raw/ioat/rte_rawdev_ioat_version.map | 2 +-
drivers/raw/ntb/rte_rawdev_ntb_version.map | 4 +-
.../rte_rawdev_octeontx2_dma_version.map | 2 +-
.../skeleton/rte_rawdev_skeleton_version.map | 2 +-
lib/librte_acl/rte_acl_version.map | 4 +-
lib/librte_bbdev/rte_bbdev_version.map | 1 -
.../rte_bitratestats_version.map | 4 +-
lib/librte_bpf/rte_bpf_version.map | 1 -
lib/librte_cfgfile/rte_cfgfile_version.map | 34 +-
lib/librte_cmdline/rte_cmdline_version.map | 10 +-
.../rte_compressdev_version.map | 13 +-
.../rte_cryptodev_version.map | 107 ++---
lib/librte_distributor/rte_distributor.c | 18 +-
lib/librte_distributor/rte_distributor_v20.c | 9 -
.../rte_distributor_version.map | 16 +-
lib/librte_eal/rte_eal_version.map | 428 +++++++-----------
lib/librte_efd/rte_efd_version.map | 4 +-
lib/librte_ethdev/rte_ethdev_version.map | 212 +++------
lib/librte_eventdev/rte_eventdev_version.map | 130 ++----
.../rte_flow_classify_version.map | 1 -
lib/librte_gro/rte_gro_version.map | 4 +-
lib/librte_gso/rte_gso_version.map | 4 +-
lib/librte_hash/rte_hash_version.map | 45 +-
lib/librte_ip_frag/rte_ip_frag_version.map | 11 +-
lib/librte_ipsec/rte_ipsec_version.map | 1 -
lib/librte_jobstats/rte_jobstats_version.map | 10 +-
lib/librte_kni/rte_kni_version.map | 3 +-
lib/librte_kvargs/rte_kvargs_version.map | 6 +-
.../rte_latencystats_version.map | 4 +-
lib/librte_lpm/rte_lpm.c | 21 +-
lib/librte_lpm/rte_lpm6.c | 12 +-
lib/librte_lpm/rte_lpm_version.map | 35 +-
lib/librte_mbuf/rte_mbuf_version.map | 40 +-
lib/librte_member/rte_member_version.map | 4 +-
lib/librte_mempool/rte_mempool_version.map | 45 +-
lib/librte_meter/rte_meter_version.map | 12 +-
lib/librte_metrics/rte_metrics_version.map | 3 +-
lib/librte_net/rte_net_version.map | 26 +-
lib/librte_pci/rte_pci_version.map | 4 +-
lib/librte_pdump/rte_pdump_version.map | 4 +-
lib/librte_pipeline/rte_pipeline_version.map | 39 +-
lib/librte_port/rte_port_version.map | 64 +--
lib/librte_power/rte_power_version.map | 25 +-
lib/librte_rawdev/rte_rawdev_version.map | 6 +-
lib/librte_rcu/rte_rcu_version.map | 1 -
lib/librte_reorder/rte_reorder_version.map | 10 +-
lib/librte_ring/rte_ring_version.map | 12 +-
lib/librte_sched/rte_sched_version.map | 15 +-
lib/librte_security/rte_security_version.map | 3 +-
lib/librte_stack/rte_stack_version.map | 1 -
lib/librte_table/rte_table_version.map | 4 +-
.../rte_telemetry_version.map | 1 -
lib/librte_timer/rte_timer.c | 15 +-
lib/librte_timer/rte_timer_version.map | 13 +-
lib/librte_vhost/rte_vhost_version.map | 79 +---
157 files changed, 840 insertions(+), 1567 deletions(-)
diff --git a/drivers/baseband/fpga_lte_fec/rte_pmd_bbdev_fpga_lte_fec_version.map b/drivers/baseband/fpga_lte_fec/rte_pmd_bbdev_fpga_lte_fec_version.map
index e92327075..0322b777c 100644
--- a/drivers/baseband/fpga_lte_fec/rte_pmd_bbdev_fpga_lte_fec_version.map
+++ b/drivers/baseband/fpga_lte_fec/rte_pmd_bbdev_fpga_lte_fec_version.map
@@ -1,3 +1,4 @@
-DPDK_19.08 {
- local: *;
+DPDK_20.0 {
+
+ local: *;
};
diff --git a/drivers/baseband/null/rte_pmd_bbdev_null_version.map b/drivers/baseband/null/rte_pmd_bbdev_null_version.map
index 58b94270d..0322b777c 100644
--- a/drivers/baseband/null/rte_pmd_bbdev_null_version.map
+++ b/drivers/baseband/null/rte_pmd_bbdev_null_version.map
@@ -1,3 +1,4 @@
-DPDK_18.02 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/baseband/turbo_sw/rte_pmd_bbdev_turbo_sw_version.map b/drivers/baseband/turbo_sw/rte_pmd_bbdev_turbo_sw_version.map
index 58b94270d..0322b777c 100644
--- a/drivers/baseband/turbo_sw/rte_pmd_bbdev_turbo_sw_version.map
+++ b/drivers/baseband/turbo_sw/rte_pmd_bbdev_turbo_sw_version.map
@@ -1,3 +1,4 @@
-DPDK_18.02 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/bus/dpaa/rte_bus_dpaa_version.map b/drivers/bus/dpaa/rte_bus_dpaa_version.map
index c88deaf7f..cfd3b55b6 100644
--- a/drivers/bus/dpaa/rte_bus_dpaa_version.map
+++ b/drivers/bus/dpaa/rte_bus_dpaa_version.map
@@ -1,4 +1,4 @@
-DPDK_17.11 {
+DPDK_20.0 {
global:
bman_acquire;
@@ -8,32 +8,37 @@ DPDK_17.11 {
bman_new_pool;
bman_query_free_buffers;
bman_release;
+ bman_thread_irq;
+ dpaa_logtype_eventdev;
dpaa_logtype_mempool;
dpaa_logtype_pmd;
dpaa_netcfg;
+ dpaa_svr_family;
fman_ccsr_map_fd;
fman_dealloc_bufs_mask_hi;
fman_dealloc_bufs_mask_lo;
fman_if_add_mac_addr;
fman_if_clear_mac_addr;
fman_if_disable_rx;
- fman_if_enable_rx;
fman_if_discard_rx_errors;
- fman_if_get_fc_threshold;
+ fman_if_enable_rx;
fman_if_get_fc_quanta;
+ fman_if_get_fc_threshold;
fman_if_get_fdoff;
+ fman_if_get_sg_enable;
fman_if_loopback_disable;
fman_if_loopback_enable;
fman_if_promiscuous_disable;
fman_if_promiscuous_enable;
fman_if_reset_mcast_filter_table;
fman_if_set_bp;
- fman_if_set_fc_threshold;
fman_if_set_fc_quanta;
+ fman_if_set_fc_threshold;
fman_if_set_fdoff;
fman_if_set_ic_params;
fman_if_set_maxfrm;
fman_if_set_mcast_filter_table;
+ fman_if_set_sg;
fman_if_stats_get;
fman_if_stats_get_all;
fman_if_stats_reset;
@@ -41,85 +46,52 @@ DPDK_17.11 {
netcfg_acquire;
netcfg_release;
of_find_compatible_node;
+ of_get_mac_address;
of_get_property;
+ per_lcore_dpaa_io;
+ per_lcore_held_bufs;
qm_channel_caam;
+ qm_channel_pool1;
+ qman_alloc_cgrid_range;
+ qman_alloc_pool_range;
+ qman_clear_irq;
+ qman_create_cgr;
qman_create_fq;
+ qman_dca_index;
+ qman_delete_cgr;
qman_dequeue;
qman_dqrr_consume;
qman_enqueue;
qman_enqueue_multi;
+ qman_enqueue_multi_fq;
qman_fq_fqid;
qman_fq_state;
qman_global_init;
qman_init_fq;
- qman_poll_dqrr;
- qman_query_fq_np;
- qman_set_vdq;
- qman_reserve_fqid_range;
- qman_volatile_dequeue;
- rte_dpaa_driver_register;
- rte_dpaa_driver_unregister;
- rte_dpaa_mem_ptov;
- rte_dpaa_portal_init;
-
- local: *;
-};
-
-DPDK_18.02 {
- global:
-
- dpaa_logtype_eventdev;
- dpaa_svr_family;
- per_lcore_dpaa_io;
- per_lcore_held_bufs;
- qm_channel_pool1;
- qman_alloc_cgrid_range;
- qman_alloc_pool_range;
- qman_create_cgr;
- qman_dca_index;
- qman_delete_cgr;
- qman_enqueue_multi_fq;
+ qman_irqsource_add;
+ qman_irqsource_remove;
qman_modify_cgr;
qman_oos_fq;
+ qman_poll_dqrr;
qman_portal_dequeue;
qman_portal_poll_rx;
qman_query_fq_frm_cnt;
+ qman_query_fq_np;
qman_release_cgrid_range;
+ qman_reserve_fqid_range;
qman_retire_fq;
+ qman_set_fq_lookup_table;
+ qman_set_vdq;
qman_static_dequeue_add;
- rte_dpaa_portal_fq_close;
- rte_dpaa_portal_fq_init;
-
- local: *;
-} DPDK_17.11;
-
-DPDK_18.08 {
- global:
- fman_if_get_sg_enable;
- fman_if_set_sg;
- of_get_mac_address;
-
- local: *;
-} DPDK_18.02;
-
-DPDK_18.11 {
- global:
- bman_thread_irq;
- fman_if_get_sg_enable;
- fman_if_set_sg;
- qman_clear_irq;
-
- qman_irqsource_add;
- qman_irqsource_remove;
qman_thread_fd;
qman_thread_irq;
-
+ qman_volatile_dequeue;
+ rte_dpaa_driver_register;
+ rte_dpaa_driver_unregister;
+ rte_dpaa_mem_ptov;
+ rte_dpaa_portal_fq_close;
+ rte_dpaa_portal_fq_init;
+ rte_dpaa_portal_init;
local: *;
-} DPDK_18.08;
-
-DPDK_19.05 {
- global:
- qman_set_fq_lookup_table;
+};
- local: *;
-} DPDK_18.11;
diff --git a/drivers/bus/fslmc/rte_bus_fslmc_version.map b/drivers/bus/fslmc/rte_bus_fslmc_version.map
index 4da787236..87810f789 100644
--- a/drivers/bus/fslmc/rte_bus_fslmc_version.map
+++ b/drivers/bus/fslmc/rte_bus_fslmc_version.map
@@ -1,32 +1,67 @@
-DPDK_17.05 {
+DPDK_20.0 {
global:
+ dpaa2_affine_qbman_ethrx_swp;
dpaa2_affine_qbman_swp;
dpaa2_alloc_dpbp_dev;
dpaa2_alloc_dq_storage;
+ dpaa2_dpbp_supported;
+ dpaa2_dqrr_size;
+ dpaa2_eqcr_size;
dpaa2_free_dpbp_dev;
dpaa2_free_dq_storage;
+ dpaa2_free_eq_descriptors;
+ dpaa2_get_qbman_swp;
+ dpaa2_io_portal;
+ dpaa2_svr_family;
+ dpaa2_virt_mode;
dpbp_disable;
dpbp_enable;
dpbp_get_attributes;
dpbp_get_num_free_bufs;
dpbp_open;
dpbp_reset;
+ dpci_get_opr;
+ dpci_set_opr;
+ dpci_set_rx_queue;
+ dpcon_get_attributes;
+ dpcon_open;
+ dpdmai_close;
+ dpdmai_disable;
+ dpdmai_enable;
+ dpdmai_get_attributes;
+ dpdmai_get_rx_queue;
+ dpdmai_get_tx_queue;
+ dpdmai_open;
+ dpdmai_set_rx_queue;
+ dpio_add_static_dequeue_channel;
dpio_close;
dpio_disable;
dpio_enable;
dpio_get_attributes;
dpio_open;
+ dpio_remove_static_dequeue_channel;
dpio_reset;
dpio_set_stashing_destination;
+ mc_get_soc_version;
+ mc_get_version;
mc_send_command;
per_lcore__dpaa2_io;
+ per_lcore_dpaa2_held_bufs;
qbman_check_command_complete;
+ qbman_check_new_result;
qbman_eq_desc_clear;
+ qbman_eq_desc_set_dca;
qbman_eq_desc_set_fq;
qbman_eq_desc_set_no_orp;
+ qbman_eq_desc_set_orp;
qbman_eq_desc_set_qd;
qbman_eq_desc_set_response;
+ qbman_eq_desc_set_token;
+ qbman_fq_query_state;
+ qbman_fq_state_frame_count;
+ qbman_get_dqrr_from_idx;
+ qbman_get_dqrr_idx;
qbman_pull_desc_clear;
qbman_pull_desc_set_fq;
qbman_pull_desc_set_numframes;
@@ -35,112 +70,42 @@ DPDK_17.05 {
qbman_release_desc_set_bpid;
qbman_result_DQ_fd;
qbman_result_DQ_flags;
- qbman_result_has_new_result;
- qbman_swp_acquire;
- qbman_swp_pull;
- qbman_swp_release;
- rte_fslmc_driver_register;
- rte_fslmc_driver_unregister;
- rte_fslmc_vfio_dmamap;
- rte_mcp_ptr_list;
-
- local: *;
-};
-
-DPDK_17.08 {
- global:
-
- dpaa2_io_portal;
- dpaa2_get_qbman_swp;
- dpci_set_rx_queue;
- dpcon_open;
- dpcon_get_attributes;
- dpio_add_static_dequeue_channel;
- dpio_remove_static_dequeue_channel;
- mc_get_soc_version;
- mc_get_version;
- qbman_check_new_result;
- qbman_eq_desc_set_dca;
- qbman_get_dqrr_from_idx;
- qbman_get_dqrr_idx;
qbman_result_DQ_fqd_ctx;
+ qbman_result_DQ_odpid;
+ qbman_result_DQ_seqnum;
qbman_result_SCN_state;
+ qbman_result_eqresp_fd;
+ qbman_result_eqresp_rc;
+ qbman_result_eqresp_rspid;
+ qbman_result_eqresp_set_rspid;
+ qbman_result_has_new_result;
+ qbman_swp_acquire;
qbman_swp_dqrr_consume;
+ qbman_swp_dqrr_idx_consume;
qbman_swp_dqrr_next;
qbman_swp_enqueue_multiple;
qbman_swp_enqueue_multiple_desc;
+ qbman_swp_enqueue_multiple_fd;
qbman_swp_interrupt_clear_status;
+ qbman_swp_prefetch_dqrr_next;
+ qbman_swp_pull;
qbman_swp_push_set;
+ qbman_swp_release;
rte_dpaa2_alloc_dpci_dev;
- rte_fslmc_object_register;
- rte_global_active_dqs_list;
-
-} DPDK_17.05;
-
-DPDK_17.11 {
- global:
-
- dpaa2_dpbp_supported;
rte_dpaa2_dev_type;
+ rte_dpaa2_free_dpci_dev;
rte_dpaa2_intr_disable;
rte_dpaa2_intr_enable;
-
-} DPDK_17.08;
-
-DPDK_18.02 {
- global:
-
- dpaa2_svr_family;
- dpaa2_virt_mode;
- per_lcore_dpaa2_held_bufs;
- qbman_fq_query_state;
- qbman_fq_state_frame_count;
- qbman_swp_dqrr_idx_consume;
- qbman_swp_prefetch_dqrr_next;
- rte_fslmc_get_device_count;
-
-} DPDK_17.11;
-
-DPDK_18.05 {
- global:
-
- dpaa2_affine_qbman_ethrx_swp;
- dpdmai_close;
- dpdmai_disable;
- dpdmai_enable;
- dpdmai_get_attributes;
- dpdmai_get_rx_queue;
- dpdmai_get_tx_queue;
- dpdmai_open;
- dpdmai_set_rx_queue;
- rte_dpaa2_free_dpci_dev;
rte_dpaa2_memsegs;
-
-} DPDK_18.02;
-
-DPDK_18.11 {
- global:
- dpaa2_dqrr_size;
- dpaa2_eqcr_size;
- dpci_get_opr;
- dpci_set_opr;
-
-} DPDK_18.05;
-
-DPDK_19.05 {
- global:
- dpaa2_free_eq_descriptors;
-
- qbman_eq_desc_set_orp;
- qbman_eq_desc_set_token;
- qbman_result_DQ_odpid;
- qbman_result_DQ_seqnum;
- qbman_result_eqresp_fd;
- qbman_result_eqresp_rc;
- qbman_result_eqresp_rspid;
- qbman_result_eqresp_set_rspid;
- qbman_swp_enqueue_multiple_fd;
-} DPDK_18.11;
+ rte_fslmc_driver_register;
+ rte_fslmc_driver_unregister;
+ rte_fslmc_get_device_count;
+ rte_fslmc_object_register;
+ rte_fslmc_vfio_dmamap;
+ rte_global_active_dqs_list;
+ rte_mcp_ptr_list;
+ local: *;
+};
EXPERIMENTAL {
global:
diff --git a/drivers/bus/ifpga/rte_bus_ifpga_version.map b/drivers/bus/ifpga/rte_bus_ifpga_version.map
index 964c9a9c4..701d794d6 100644
--- a/drivers/bus/ifpga/rte_bus_ifpga_version.map
+++ b/drivers/bus/ifpga/rte_bus_ifpga_version.map
@@ -1,17 +1,11 @@
-DPDK_18.05 {
+DPDK_20.0 {
global:
- rte_ifpga_get_integer32_arg;
- rte_ifpga_get_string_arg;
rte_ifpga_driver_register;
rte_ifpga_driver_unregister;
-
+ rte_ifpga_find_afu_by_name;
+ rte_ifpga_get_integer32_arg;
+ rte_ifpga_get_string_arg;
local: *;
};
-DPDK_19.05 {
- global:
-
- rte_ifpga_find_afu_by_name;
-
-} DPDK_18.05;
diff --git a/drivers/bus/pci/rte_bus_pci_version.map b/drivers/bus/pci/rte_bus_pci_version.map
index 27e9c4f10..6d4dfc8ff 100644
--- a/drivers/bus/pci/rte_bus_pci_version.map
+++ b/drivers/bus/pci/rte_bus_pci_version.map
@@ -1,4 +1,4 @@
-DPDK_17.11 {
+DPDK_20.0 {
global:
rte_pci_dump;
@@ -13,6 +13,6 @@ DPDK_17.11 {
rte_pci_unmap_device;
rte_pci_unregister;
rte_pci_write_config;
-
local: *;
};
+
diff --git a/drivers/bus/vdev/rte_bus_vdev_version.map b/drivers/bus/vdev/rte_bus_vdev_version.map
index 590cf9b43..019e7cb8d 100644
--- a/drivers/bus/vdev/rte_bus_vdev_version.map
+++ b/drivers/bus/vdev/rte_bus_vdev_version.map
@@ -1,18 +1,12 @@
-DPDK_17.11 {
+DPDK_20.0 {
global:
+ rte_vdev_add_custom_scan;
rte_vdev_init;
rte_vdev_register;
+ rte_vdev_remove_custom_scan;
rte_vdev_uninit;
rte_vdev_unregister;
-
local: *;
};
-DPDK_18.02 {
- global:
-
- rte_vdev_add_custom_scan;
- rte_vdev_remove_custom_scan;
-
-} DPDK_17.11;
diff --git a/drivers/bus/vmbus/rte_bus_vmbus_version.map b/drivers/bus/vmbus/rte_bus_vmbus_version.map
index ae231ad32..0064b37e4 100644
--- a/drivers/bus/vmbus/rte_bus_vmbus_version.map
+++ b/drivers/bus/vmbus/rte_bus_vmbus_version.map
@@ -1,6 +1,4 @@
-/* SPDX-License-Identifier: BSD-3-Clause */
-
-DPDK_18.08 {
+DPDK_20.0 {
global:
rte_vmbus_chan_close;
@@ -20,17 +18,11 @@ DPDK_18.08 {
rte_vmbus_probe;
rte_vmbus_register;
rte_vmbus_scan;
+ rte_vmbus_set_latency;
rte_vmbus_sub_channel_index;
rte_vmbus_subchan_open;
rte_vmbus_unmap_device;
rte_vmbus_unregister;
-
local: *;
};
-DPDK_18.11 {
- global:
-
- rte_vmbus_set_latency;
-
-} DPDK_18.08;
diff --git a/drivers/common/cpt/rte_common_cpt_version.map b/drivers/common/cpt/rte_common_cpt_version.map
index dec614f0d..d0d14b450 100644
--- a/drivers/common/cpt/rte_common_cpt_version.map
+++ b/drivers/common/cpt/rte_common_cpt_version.map
@@ -1,6 +1,7 @@
-DPDK_18.11 {
+DPDK_20.0 {
global:
cpt_pmd_ops_helper_get_mlen_direct_mode;
cpt_pmd_ops_helper_get_mlen_sg_mode;
};
+
diff --git a/drivers/common/dpaax/rte_common_dpaax_version.map b/drivers/common/dpaax/rte_common_dpaax_version.map
index 8131c9e30..98d70eacb 100644
--- a/drivers/common/dpaax/rte_common_dpaax_version.map
+++ b/drivers/common/dpaax/rte_common_dpaax_version.map
@@ -1,11 +1,11 @@
-DPDK_18.11 {
+DPDK_20.0 {
global:
- dpaax_iova_table_update;
dpaax_iova_table_depopulate;
dpaax_iova_table_dump;
dpaax_iova_table_p;
dpaax_iova_table_populate;
-
+ dpaax_iova_table_update;
local: *;
};
+
diff --git a/drivers/common/mvep/rte_common_mvep_version.map b/drivers/common/mvep/rte_common_mvep_version.map
index c71722d79..79f8c5e2c 100644
--- a/drivers/common/mvep/rte_common_mvep_version.map
+++ b/drivers/common/mvep/rte_common_mvep_version.map
@@ -1,6 +1,7 @@
-DPDK_18.11 {
+DPDK_20.0 {
global:
- rte_mvep_init;
rte_mvep_deinit;
+ rte_mvep_init;
};
+
diff --git a/drivers/common/octeontx/rte_common_octeontx_version.map b/drivers/common/octeontx/rte_common_octeontx_version.map
index f04b3b7f8..7d6f3a617 100644
--- a/drivers/common/octeontx/rte_common_octeontx_version.map
+++ b/drivers/common/octeontx/rte_common_octeontx_version.map
@@ -1,7 +1,8 @@
-DPDK_18.05 {
+DPDK_20.0 {
global:
+ octeontx_mbox_send;
octeontx_mbox_set_ram_mbox_base;
octeontx_mbox_set_reg;
- octeontx_mbox_send;
};
+
diff --git a/drivers/common/octeontx2/rte_common_octeontx2_version.map b/drivers/common/octeontx2/rte_common_octeontx2_version.map
index 4400120da..fa1bececa 100644
--- a/drivers/common/octeontx2/rte_common_octeontx2_version.map
+++ b/drivers/common/octeontx2/rte_common_octeontx2_version.map
@@ -1,39 +1,35 @@
-DPDK_19.08 {
+DPDK_20.0 {
global:
otx2_dev_active_vfs;
otx2_dev_fini;
otx2_dev_priv_init;
-
+ otx2_disable_irqs;
+ otx2_intra_dev_get_cfg;
otx2_logtype_base;
otx2_logtype_dpi;
otx2_logtype_mbox;
+ otx2_logtype_nix;
otx2_logtype_npa;
otx2_logtype_npc;
- otx2_logtype_nix;
otx2_logtype_sso;
- otx2_logtype_tm;
otx2_logtype_tim;
-
+ otx2_logtype_tm;
otx2_mbox_alloc_msg_rsp;
otx2_mbox_get_rsp;
otx2_mbox_get_rsp_tmo;
otx2_mbox_id2name;
otx2_mbox_msg_send;
otx2_mbox_wait_for_rsp;
-
- otx2_intra_dev_get_cfg;
otx2_npa_lf_active;
otx2_npa_lf_obj_get;
otx2_npa_lf_obj_ref;
otx2_npa_pf_func_get;
otx2_npa_set_defaults;
+ otx2_register_irq;
otx2_sso_pf_func_get;
otx2_sso_pf_func_set;
-
- otx2_disable_irqs;
otx2_unregister_irq;
- otx2_register_irq;
-
local: *;
};
+
diff --git a/drivers/compress/isal/rte_pmd_isal_version.map b/drivers/compress/isal/rte_pmd_isal_version.map
index de8e412ff..0322b777c 100644
--- a/drivers/compress/isal/rte_pmd_isal_version.map
+++ b/drivers/compress/isal/rte_pmd_isal_version.map
@@ -1,3 +1,4 @@
-DPDK_18.05 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/compress/octeontx/rte_pmd_octeontx_compress_version.map b/drivers/compress/octeontx/rte_pmd_octeontx_compress_version.map
index ad6e191e4..0322b777c 100644
--- a/drivers/compress/octeontx/rte_pmd_octeontx_compress_version.map
+++ b/drivers/compress/octeontx/rte_pmd_octeontx_compress_version.map
@@ -1,3 +1,4 @@
-DPDK_18.08 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/compress/qat/rte_pmd_qat_version.map b/drivers/compress/qat/rte_pmd_qat_version.map
index ad6e191e4..0322b777c 100644
--- a/drivers/compress/qat/rte_pmd_qat_version.map
+++ b/drivers/compress/qat/rte_pmd_qat_version.map
@@ -1,3 +1,4 @@
-DPDK_18.08 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/compress/zlib/rte_pmd_zlib_version.map b/drivers/compress/zlib/rte_pmd_zlib_version.map
index ad6e191e4..0322b777c 100644
--- a/drivers/compress/zlib/rte_pmd_zlib_version.map
+++ b/drivers/compress/zlib/rte_pmd_zlib_version.map
@@ -1,3 +1,4 @@
-DPDK_18.08 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/crypto/aesni_gcm/rte_pmd_aesni_gcm_version.map b/drivers/crypto/aesni_gcm/rte_pmd_aesni_gcm_version.map
index dc4d417b7..0322b777c 100644
--- a/drivers/crypto/aesni_gcm/rte_pmd_aesni_gcm_version.map
+++ b/drivers/crypto/aesni_gcm/rte_pmd_aesni_gcm_version.map
@@ -1,3 +1,4 @@
-DPDK_16.04 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/crypto/aesni_mb/rte_pmd_aesni_mb_version.map b/drivers/crypto/aesni_mb/rte_pmd_aesni_mb_version.map
index ad607bbed..0322b777c 100644
--- a/drivers/crypto/aesni_mb/rte_pmd_aesni_mb_version.map
+++ b/drivers/crypto/aesni_mb/rte_pmd_aesni_mb_version.map
@@ -1,3 +1,4 @@
-DPDK_2.2 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/crypto/armv8/rte_pmd_armv8_version.map b/drivers/crypto/armv8/rte_pmd_armv8_version.map
index 1f84b68a8..0322b777c 100644
--- a/drivers/crypto/armv8/rte_pmd_armv8_version.map
+++ b/drivers/crypto/armv8/rte_pmd_armv8_version.map
@@ -1,3 +1,4 @@
-DPDK_17.02 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/crypto/caam_jr/rte_pmd_caam_jr_version.map b/drivers/crypto/caam_jr/rte_pmd_caam_jr_version.map
index 521e51f41..0322b777c 100644
--- a/drivers/crypto/caam_jr/rte_pmd_caam_jr_version.map
+++ b/drivers/crypto/caam_jr/rte_pmd_caam_jr_version.map
@@ -1,4 +1,4 @@
-DPDK_18.11 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/crypto/ccp/rte_pmd_ccp_version.map b/drivers/crypto/ccp/rte_pmd_ccp_version.map
index 9b9ab1a4c..0322b777c 100644
--- a/drivers/crypto/ccp/rte_pmd_ccp_version.map
+++ b/drivers/crypto/ccp/rte_pmd_ccp_version.map
@@ -1,4 +1,4 @@
-DPDK_18.05 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/crypto/dpaa2_sec/rte_pmd_dpaa2_sec_version.map b/drivers/crypto/dpaa2_sec/rte_pmd_dpaa2_sec_version.map
index 0bfb986d0..a176d44fd 100644
--- a/drivers/crypto/dpaa2_sec/rte_pmd_dpaa2_sec_version.map
+++ b/drivers/crypto/dpaa2_sec/rte_pmd_dpaa2_sec_version.map
@@ -1,12 +1,8 @@
-DPDK_17.05 {
-
- local: *;
-};
-
-DPDK_18.11 {
+DPDK_20.0 {
global:
dpaa2_sec_eventq_attach;
dpaa2_sec_eventq_detach;
+ local: *;
+};
-} DPDK_17.05;
diff --git a/drivers/crypto/dpaa_sec/rte_pmd_dpaa_sec_version.map b/drivers/crypto/dpaa_sec/rte_pmd_dpaa_sec_version.map
index a70bd197b..0322b777c 100644
--- a/drivers/crypto/dpaa_sec/rte_pmd_dpaa_sec_version.map
+++ b/drivers/crypto/dpaa_sec/rte_pmd_dpaa_sec_version.map
@@ -1,4 +1,4 @@
-DPDK_17.11 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/crypto/kasumi/rte_pmd_kasumi_version.map b/drivers/crypto/kasumi/rte_pmd_kasumi_version.map
index 8ffeca934..0322b777c 100644
--- a/drivers/crypto/kasumi/rte_pmd_kasumi_version.map
+++ b/drivers/crypto/kasumi/rte_pmd_kasumi_version.map
@@ -1,3 +1,4 @@
-DPDK_16.07 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/crypto/mvsam/rte_pmd_mvsam_version.map b/drivers/crypto/mvsam/rte_pmd_mvsam_version.map
index a75303172..0322b777c 100644
--- a/drivers/crypto/mvsam/rte_pmd_mvsam_version.map
+++ b/drivers/crypto/mvsam/rte_pmd_mvsam_version.map
@@ -1,3 +1,4 @@
-DPDK_17.11 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/crypto/null/rte_pmd_null_crypto_version.map b/drivers/crypto/null/rte_pmd_null_crypto_version.map
index dc4d417b7..0322b777c 100644
--- a/drivers/crypto/null/rte_pmd_null_crypto_version.map
+++ b/drivers/crypto/null/rte_pmd_null_crypto_version.map
@@ -1,3 +1,4 @@
-DPDK_16.04 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/crypto/octeontx/rte_pmd_octeontx_crypto_version.map b/drivers/crypto/octeontx/rte_pmd_octeontx_crypto_version.map
index 521e51f41..0322b777c 100644
--- a/drivers/crypto/octeontx/rte_pmd_octeontx_crypto_version.map
+++ b/drivers/crypto/octeontx/rte_pmd_octeontx_crypto_version.map
@@ -1,4 +1,4 @@
-DPDK_18.11 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/crypto/openssl/rte_pmd_openssl_version.map b/drivers/crypto/openssl/rte_pmd_openssl_version.map
index cc5829e30..0322b777c 100644
--- a/drivers/crypto/openssl/rte_pmd_openssl_version.map
+++ b/drivers/crypto/openssl/rte_pmd_openssl_version.map
@@ -1,3 +1,4 @@
-DPDK_16.11 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/crypto/scheduler/rte_pmd_crypto_scheduler_version.map b/drivers/crypto/scheduler/rte_pmd_crypto_scheduler_version.map
index 5c43127cf..8f4f9140e 100644
--- a/drivers/crypto/scheduler/rte_pmd_crypto_scheduler_version.map
+++ b/drivers/crypto/scheduler/rte_pmd_crypto_scheduler_version.map
@@ -1,21 +1,15 @@
-DPDK_17.02 {
+DPDK_20.0 {
global:
rte_cryptodev_scheduler_load_user_scheduler;
- rte_cryptodev_scheduler_slave_attach;
- rte_cryptodev_scheduler_slave_detach;
- rte_cryptodev_scheduler_ordering_set;
- rte_cryptodev_scheduler_ordering_get;
-
-};
-
-DPDK_17.05 {
- global:
-
rte_cryptodev_scheduler_mode_get;
rte_cryptodev_scheduler_mode_set;
rte_cryptodev_scheduler_option_get;
rte_cryptodev_scheduler_option_set;
+ rte_cryptodev_scheduler_ordering_get;
+ rte_cryptodev_scheduler_ordering_set;
+ rte_cryptodev_scheduler_slave_attach;
+ rte_cryptodev_scheduler_slave_detach;
rte_cryptodev_scheduler_slaves_get;
+};
-} DPDK_17.02;
diff --git a/drivers/crypto/snow3g/rte_pmd_snow3g_version.map b/drivers/crypto/snow3g/rte_pmd_snow3g_version.map
index dc4d417b7..0322b777c 100644
--- a/drivers/crypto/snow3g/rte_pmd_snow3g_version.map
+++ b/drivers/crypto/snow3g/rte_pmd_snow3g_version.map
@@ -1,3 +1,4 @@
-DPDK_16.04 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/crypto/virtio/rte_pmd_virtio_crypto_version.map b/drivers/crypto/virtio/rte_pmd_virtio_crypto_version.map
index de8e412ff..0322b777c 100644
--- a/drivers/crypto/virtio/rte_pmd_virtio_crypto_version.map
+++ b/drivers/crypto/virtio/rte_pmd_virtio_crypto_version.map
@@ -1,3 +1,4 @@
-DPDK_18.05 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/crypto/zuc/rte_pmd_zuc_version.map b/drivers/crypto/zuc/rte_pmd_zuc_version.map
index cc5829e30..0322b777c 100644
--- a/drivers/crypto/zuc/rte_pmd_zuc_version.map
+++ b/drivers/crypto/zuc/rte_pmd_zuc_version.map
@@ -1,3 +1,4 @@
-DPDK_16.11 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/event/dpaa/rte_pmd_dpaa_event_version.map b/drivers/event/dpaa/rte_pmd_dpaa_event_version.map
index 179140fb8..0322b777c 100644
--- a/drivers/event/dpaa/rte_pmd_dpaa_event_version.map
+++ b/drivers/event/dpaa/rte_pmd_dpaa_event_version.map
@@ -1,4 +1,4 @@
-DPDK_18.02 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/event/dpaa2/rte_pmd_dpaa2_event_version.map b/drivers/event/dpaa2/rte_pmd_dpaa2_event_version.map
index 1c0b7559d..0322b777c 100644
--- a/drivers/event/dpaa2/rte_pmd_dpaa2_event_version.map
+++ b/drivers/event/dpaa2/rte_pmd_dpaa2_event_version.map
@@ -1,3 +1,4 @@
-DPDK_17.08 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/event/dsw/rte_pmd_dsw_event_version.map b/drivers/event/dsw/rte_pmd_dsw_event_version.map
index 24bd5cdb3..0322b777c 100644
--- a/drivers/event/dsw/rte_pmd_dsw_event_version.map
+++ b/drivers/event/dsw/rte_pmd_dsw_event_version.map
@@ -1,3 +1,4 @@
-DPDK_18.11 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/event/octeontx/rte_pmd_octeontx_event_version.map b/drivers/event/octeontx/rte_pmd_octeontx_event_version.map
index 5352e7e3b..0322b777c 100644
--- a/drivers/event/octeontx/rte_pmd_octeontx_event_version.map
+++ b/drivers/event/octeontx/rte_pmd_octeontx_event_version.map
@@ -1,3 +1,4 @@
-DPDK_17.05 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/event/octeontx2/rte_pmd_octeontx2_event_version.map b/drivers/event/octeontx2/rte_pmd_octeontx2_event_version.map
index 41c65c8c9..0322b777c 100644
--- a/drivers/event/octeontx2/rte_pmd_octeontx2_event_version.map
+++ b/drivers/event/octeontx2/rte_pmd_octeontx2_event_version.map
@@ -1,4 +1,4 @@
-DPDK_19.08 {
+DPDK_20.0 {
+
local: *;
};
-
diff --git a/drivers/event/opdl/rte_pmd_opdl_event_version.map b/drivers/event/opdl/rte_pmd_opdl_event_version.map
index 58b94270d..0322b777c 100644
--- a/drivers/event/opdl/rte_pmd_opdl_event_version.map
+++ b/drivers/event/opdl/rte_pmd_opdl_event_version.map
@@ -1,3 +1,4 @@
-DPDK_18.02 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/event/skeleton/rte_pmd_skeleton_event_version.map b/drivers/event/skeleton/rte_pmd_skeleton_event_version.map
index 8591cc0b1..0322b777c 100644
--- a/drivers/event/skeleton/rte_pmd_skeleton_event_version.map
+++ b/drivers/event/skeleton/rte_pmd_skeleton_event_version.map
@@ -1,4 +1,4 @@
-DPDK_17.05 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/event/sw/rte_pmd_sw_event_version.map b/drivers/event/sw/rte_pmd_sw_event_version.map
index 5352e7e3b..0322b777c 100644
--- a/drivers/event/sw/rte_pmd_sw_event_version.map
+++ b/drivers/event/sw/rte_pmd_sw_event_version.map
@@ -1,3 +1,4 @@
-DPDK_17.05 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/mempool/bucket/rte_mempool_bucket_version.map b/drivers/mempool/bucket/rte_mempool_bucket_version.map
index 9b9ab1a4c..0322b777c 100644
--- a/drivers/mempool/bucket/rte_mempool_bucket_version.map
+++ b/drivers/mempool/bucket/rte_mempool_bucket_version.map
@@ -1,4 +1,4 @@
-DPDK_18.05 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/mempool/dpaa/rte_mempool_dpaa_version.map b/drivers/mempool/dpaa/rte_mempool_dpaa_version.map
index 60bf50b2d..f20d86db8 100644
--- a/drivers/mempool/dpaa/rte_mempool_dpaa_version.map
+++ b/drivers/mempool/dpaa/rte_mempool_dpaa_version.map
@@ -1,8 +1,8 @@
-DPDK_17.11 {
+DPDK_20.0 {
global:
rte_dpaa_bpid_info;
rte_dpaa_memsegs;
-
local: *;
};
+
diff --git a/drivers/mempool/dpaa2/rte_mempool_dpaa2_version.map b/drivers/mempool/dpaa2/rte_mempool_dpaa2_version.map
index b45e7a9ac..b4024be42 100644
--- a/drivers/mempool/dpaa2/rte_mempool_dpaa2_version.map
+++ b/drivers/mempool/dpaa2/rte_mempool_dpaa2_version.map
@@ -1,16 +1,10 @@
-DPDK_17.05 {
+DPDK_20.0 {
global:
rte_dpaa2_bpid_info;
rte_dpaa2_mbuf_alloc_bulk;
-
- local: *;
-};
-
-DPDK_18.05 {
- global:
-
rte_dpaa2_mbuf_from_buf_addr;
rte_dpaa2_mbuf_pool_bpid;
+ local: *;
+};
-} DPDK_17.05;
diff --git a/drivers/mempool/octeontx/rte_mempool_octeontx_version.map b/drivers/mempool/octeontx/rte_mempool_octeontx_version.map
index a75303172..0322b777c 100644
--- a/drivers/mempool/octeontx/rte_mempool_octeontx_version.map
+++ b/drivers/mempool/octeontx/rte_mempool_octeontx_version.map
@@ -1,3 +1,4 @@
-DPDK_17.11 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/mempool/octeontx2/rte_mempool_octeontx2_version.map b/drivers/mempool/octeontx2/rte_mempool_octeontx2_version.map
index d703368c3..4f8889204 100644
--- a/drivers/mempool/octeontx2/rte_mempool_octeontx2_version.map
+++ b/drivers/mempool/octeontx2/rte_mempool_octeontx2_version.map
@@ -1,8 +1,8 @@
-DPDK_19.08 {
+DPDK_20.0 {
global:
- otx2_npa_lf_init;
otx2_npa_lf_fini;
-
+ otx2_npa_lf_init;
local: *;
};
+
diff --git a/drivers/mempool/ring/rte_mempool_ring_version.map b/drivers/mempool/ring/rte_mempool_ring_version.map
index 8591cc0b1..0322b777c 100644
--- a/drivers/mempool/ring/rte_mempool_ring_version.map
+++ b/drivers/mempool/ring/rte_mempool_ring_version.map
@@ -1,4 +1,4 @@
-DPDK_17.05 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/mempool/stack/rte_mempool_stack_version.map b/drivers/mempool/stack/rte_mempool_stack_version.map
index 8591cc0b1..0322b777c 100644
--- a/drivers/mempool/stack/rte_mempool_stack_version.map
+++ b/drivers/mempool/stack/rte_mempool_stack_version.map
@@ -1,4 +1,4 @@
-DPDK_17.05 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/net/af_packet/rte_pmd_af_packet_version.map b/drivers/net/af_packet/rte_pmd_af_packet_version.map
index ef3539840..0322b777c 100644
--- a/drivers/net/af_packet/rte_pmd_af_packet_version.map
+++ b/drivers/net/af_packet/rte_pmd_af_packet_version.map
@@ -1,4 +1,4 @@
-DPDK_2.0 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/net/af_xdp/rte_pmd_af_xdp_version.map b/drivers/net/af_xdp/rte_pmd_af_xdp_version.map
index c6db030fe..0322b777c 100644
--- a/drivers/net/af_xdp/rte_pmd_af_xdp_version.map
+++ b/drivers/net/af_xdp/rte_pmd_af_xdp_version.map
@@ -1,3 +1,4 @@
-DPDK_19.05 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/net/ark/rte_pmd_ark_version.map b/drivers/net/ark/rte_pmd_ark_version.map
index 1062e0429..0322b777c 100644
--- a/drivers/net/ark/rte_pmd_ark_version.map
+++ b/drivers/net/ark/rte_pmd_ark_version.map
@@ -1,4 +1,4 @@
-DPDK_17.05 {
- local: *;
+DPDK_20.0 {
+ local: *;
};
diff --git a/drivers/net/atlantic/rte_pmd_atlantic_version.map b/drivers/net/atlantic/rte_pmd_atlantic_version.map
index b16faa999..fdf813df0 100644
--- a/drivers/net/atlantic/rte_pmd_atlantic_version.map
+++ b/drivers/net/atlantic/rte_pmd_atlantic_version.map
@@ -1,16 +1,14 @@
-DPDK_18.11 {
+DPDK_20.0 {
local: *;
};
-
EXPERIMENTAL {
global:
- rte_pmd_atl_macsec_enable;
- rte_pmd_atl_macsec_disable;
- rte_pmd_atl_macsec_config_txsc;
rte_pmd_atl_macsec_config_rxsc;
- rte_pmd_atl_macsec_select_txsa;
+ rte_pmd_atl_macsec_config_txsc;
+ rte_pmd_atl_macsec_disable;
+ rte_pmd_atl_macsec_enable;
rte_pmd_atl_macsec_select_rxsa;
+ rte_pmd_atl_macsec_select_txsa;
};
-
diff --git a/drivers/net/avp/rte_pmd_avp_version.map b/drivers/net/avp/rte_pmd_avp_version.map
index 5352e7e3b..0322b777c 100644
--- a/drivers/net/avp/rte_pmd_avp_version.map
+++ b/drivers/net/avp/rte_pmd_avp_version.map
@@ -1,3 +1,4 @@
-DPDK_17.05 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/net/axgbe/rte_pmd_axgbe_version.map b/drivers/net/axgbe/rte_pmd_axgbe_version.map
index de8e412ff..0322b777c 100644
--- a/drivers/net/axgbe/rte_pmd_axgbe_version.map
+++ b/drivers/net/axgbe/rte_pmd_axgbe_version.map
@@ -1,3 +1,4 @@
-DPDK_18.05 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/net/bnx2x/rte_pmd_bnx2x_version.map b/drivers/net/bnx2x/rte_pmd_bnx2x_version.map
index bd8138a03..0322b777c 100644
--- a/drivers/net/bnx2x/rte_pmd_bnx2x_version.map
+++ b/drivers/net/bnx2x/rte_pmd_bnx2x_version.map
@@ -1,4 +1,4 @@
-DPDK_2.1 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/net/bnxt/rte_pmd_bnxt_version.map b/drivers/net/bnxt/rte_pmd_bnxt_version.map
index 4750d40ad..a83fd4036 100644
--- a/drivers/net/bnxt/rte_pmd_bnxt_version.map
+++ b/drivers/net/bnxt/rte_pmd_bnxt_version.map
@@ -1,4 +1,4 @@
-DPDK_17.08 {
+DPDK_20.0 {
global:
rte_pmd_bnxt_get_vf_rx_status;
@@ -10,13 +10,13 @@ DPDK_17.08 {
rte_pmd_bnxt_set_tx_loopback;
rte_pmd_bnxt_set_vf_mac_addr;
rte_pmd_bnxt_set_vf_mac_anti_spoof;
+ rte_pmd_bnxt_set_vf_persist_stats;
rte_pmd_bnxt_set_vf_rate_limit;
rte_pmd_bnxt_set_vf_rxmode;
rte_pmd_bnxt_set_vf_vlan_anti_spoof;
rte_pmd_bnxt_set_vf_vlan_filter;
rte_pmd_bnxt_set_vf_vlan_insert;
rte_pmd_bnxt_set_vf_vlan_stripq;
- rte_pmd_bnxt_set_vf_persist_stats;
-
local: *;
};
+
diff --git a/drivers/net/bonding/rte_pmd_bond_version.map b/drivers/net/bonding/rte_pmd_bond_version.map
index 00d955c48..11da7191f 100644
--- a/drivers/net/bonding/rte_pmd_bond_version.map
+++ b/drivers/net/bonding/rte_pmd_bond_version.map
@@ -1,9 +1,21 @@
-DPDK_2.0 {
+DPDK_20.0 {
global:
+ rte_eth_bond_8023ad_agg_selection_get;
+ rte_eth_bond_8023ad_agg_selection_set;
+ rte_eth_bond_8023ad_conf_get;
+ rte_eth_bond_8023ad_dedicated_queues_disable;
+ rte_eth_bond_8023ad_dedicated_queues_enable;
+ rte_eth_bond_8023ad_ext_collect;
+ rte_eth_bond_8023ad_ext_collect_get;
+ rte_eth_bond_8023ad_ext_distrib;
+ rte_eth_bond_8023ad_ext_distrib_get;
+ rte_eth_bond_8023ad_ext_slowtx;
+ rte_eth_bond_8023ad_setup;
rte_eth_bond_8023ad_slave_info;
rte_eth_bond_active_slaves_get;
rte_eth_bond_create;
+ rte_eth_bond_free;
rte_eth_bond_link_monitoring_set;
rte_eth_bond_mac_address_reset;
rte_eth_bond_mac_address_set;
@@ -16,39 +28,6 @@ DPDK_2.0 {
rte_eth_bond_slaves_get;
rte_eth_bond_xmit_policy_get;
rte_eth_bond_xmit_policy_set;
-
local: *;
};
-DPDK_2.1 {
- global:
-
- rte_eth_bond_free;
-
-} DPDK_2.0;
-
-DPDK_16.04 {
-};
-
-DPDK_16.07 {
- global:
-
- rte_eth_bond_8023ad_ext_collect;
- rte_eth_bond_8023ad_ext_collect_get;
- rte_eth_bond_8023ad_ext_distrib;
- rte_eth_bond_8023ad_ext_distrib_get;
- rte_eth_bond_8023ad_ext_slowtx;
-
-} DPDK_16.04;
-
-DPDK_17.08 {
- global:
-
- rte_eth_bond_8023ad_dedicated_queues_enable;
- rte_eth_bond_8023ad_dedicated_queues_disable;
- rte_eth_bond_8023ad_agg_selection_get;
- rte_eth_bond_8023ad_agg_selection_set;
- rte_eth_bond_8023ad_conf_get;
- rte_eth_bond_8023ad_setup;
-
-} DPDK_16.07;
diff --git a/drivers/net/cxgbe/rte_pmd_cxgbe_version.map b/drivers/net/cxgbe/rte_pmd_cxgbe_version.map
index bd8138a03..0322b777c 100644
--- a/drivers/net/cxgbe/rte_pmd_cxgbe_version.map
+++ b/drivers/net/cxgbe/rte_pmd_cxgbe_version.map
@@ -1,4 +1,4 @@
-DPDK_2.1 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/net/dpaa/rte_pmd_dpaa_version.map b/drivers/net/dpaa/rte_pmd_dpaa_version.map
index 8cb4500b5..94fa246bb 100644
--- a/drivers/net/dpaa/rte_pmd_dpaa_version.map
+++ b/drivers/net/dpaa/rte_pmd_dpaa_version.map
@@ -1,12 +1,9 @@
-DPDK_17.11 {
-
- local: *;
-};
-
-DPDK_18.08 {
+DPDK_20.0 {
global:
dpaa_eth_eventq_attach;
dpaa_eth_eventq_detach;
rte_pmd_dpaa_set_tx_loopback;
-} DPDK_17.11;
+ local: *;
+};
+
diff --git a/drivers/net/dpaa2/rte_pmd_dpaa2_version.map b/drivers/net/dpaa2/rte_pmd_dpaa2_version.map
index d1b4cdb23..5e0bcef9d 100644
--- a/drivers/net/dpaa2/rte_pmd_dpaa2_version.map
+++ b/drivers/net/dpaa2/rte_pmd_dpaa2_version.map
@@ -1,15 +1,10 @@
-DPDK_17.05 {
-
- local: *;
-};
-
-DPDK_17.11 {
+DPDK_20.0 {
global:
dpaa2_eth_eventq_attach;
dpaa2_eth_eventq_detach;
-
-} DPDK_17.05;
+ local: *;
+};
EXPERIMENTAL {
global:
@@ -17,4 +12,4 @@ EXPERIMENTAL {
rte_pmd_dpaa2_mux_flow_create;
rte_pmd_dpaa2_set_custom_hash;
rte_pmd_dpaa2_set_timestamp;
-} DPDK_17.11;
+};
diff --git a/drivers/net/e1000/rte_pmd_e1000_version.map b/drivers/net/e1000/rte_pmd_e1000_version.map
index ef3539840..0322b777c 100644
--- a/drivers/net/e1000/rte_pmd_e1000_version.map
+++ b/drivers/net/e1000/rte_pmd_e1000_version.map
@@ -1,4 +1,4 @@
-DPDK_2.0 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/net/ena/rte_pmd_ena_version.map b/drivers/net/ena/rte_pmd_ena_version.map
index 349c6e1c2..0322b777c 100644
--- a/drivers/net/ena/rte_pmd_ena_version.map
+++ b/drivers/net/ena/rte_pmd_ena_version.map
@@ -1,4 +1,4 @@
-DPDK_16.04 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/net/enetc/rte_pmd_enetc_version.map b/drivers/net/enetc/rte_pmd_enetc_version.map
index 521e51f41..0322b777c 100644
--- a/drivers/net/enetc/rte_pmd_enetc_version.map
+++ b/drivers/net/enetc/rte_pmd_enetc_version.map
@@ -1,4 +1,4 @@
-DPDK_18.11 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/net/enic/rte_pmd_enic_version.map b/drivers/net/enic/rte_pmd_enic_version.map
index ef3539840..0322b777c 100644
--- a/drivers/net/enic/rte_pmd_enic_version.map
+++ b/drivers/net/enic/rte_pmd_enic_version.map
@@ -1,4 +1,4 @@
-DPDK_2.0 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/net/failsafe/rte_pmd_failsafe_version.map b/drivers/net/failsafe/rte_pmd_failsafe_version.map
index b6d2840be..0322b777c 100644
--- a/drivers/net/failsafe/rte_pmd_failsafe_version.map
+++ b/drivers/net/failsafe/rte_pmd_failsafe_version.map
@@ -1,4 +1,4 @@
-DPDK_17.08 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/net/fm10k/rte_pmd_fm10k_version.map b/drivers/net/fm10k/rte_pmd_fm10k_version.map
index ef3539840..0322b777c 100644
--- a/drivers/net/fm10k/rte_pmd_fm10k_version.map
+++ b/drivers/net/fm10k/rte_pmd_fm10k_version.map
@@ -1,4 +1,4 @@
-DPDK_2.0 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/net/hinic/rte_pmd_hinic_version.map b/drivers/net/hinic/rte_pmd_hinic_version.map
index 9a61188cd..0322b777c 100644
--- a/drivers/net/hinic/rte_pmd_hinic_version.map
+++ b/drivers/net/hinic/rte_pmd_hinic_version.map
@@ -1,4 +1,4 @@
-DPDK_19.08 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/net/i40e/rte_pmd_i40e_version.map b/drivers/net/i40e/rte_pmd_i40e_version.map
index cccd5768c..e8ecfa1d7 100644
--- a/drivers/net/i40e/rte_pmd_i40e_version.map
+++ b/drivers/net/i40e/rte_pmd_i40e_version.map
@@ -1,67 +1,40 @@
-DPDK_2.0 {
-
- local: *;
-};
-
-DPDK_17.02 {
+DPDK_20.0 {
global:
+ rte_pmd_i40e_add_vf_mac_addr;
+ rte_pmd_i40e_flow_add_del_packet_template;
+ rte_pmd_i40e_flow_type_mapping_get;
+ rte_pmd_i40e_flow_type_mapping_reset;
+ rte_pmd_i40e_flow_type_mapping_update;
+ rte_pmd_i40e_get_ddp_info;
+ rte_pmd_i40e_get_ddp_list;
rte_pmd_i40e_get_vf_stats;
+ rte_pmd_i40e_inset_get;
+ rte_pmd_i40e_inset_set;
rte_pmd_i40e_ping_vfs;
+ rte_pmd_i40e_process_ddp_package;
rte_pmd_i40e_ptype_mapping_get;
rte_pmd_i40e_ptype_mapping_replace;
rte_pmd_i40e_ptype_mapping_reset;
rte_pmd_i40e_ptype_mapping_update;
+ rte_pmd_i40e_query_vfid_by_mac;
rte_pmd_i40e_reset_vf_stats;
+ rte_pmd_i40e_rss_queue_region_conf;
+ rte_pmd_i40e_set_tc_strict_prio;
rte_pmd_i40e_set_tx_loopback;
rte_pmd_i40e_set_vf_broadcast;
rte_pmd_i40e_set_vf_mac_addr;
rte_pmd_i40e_set_vf_mac_anti_spoof;
+ rte_pmd_i40e_set_vf_max_bw;
rte_pmd_i40e_set_vf_multicast_promisc;
+ rte_pmd_i40e_set_vf_tc_bw_alloc;
+ rte_pmd_i40e_set_vf_tc_max_bw;
rte_pmd_i40e_set_vf_unicast_promisc;
rte_pmd_i40e_set_vf_vlan_anti_spoof;
rte_pmd_i40e_set_vf_vlan_filter;
rte_pmd_i40e_set_vf_vlan_insert;
rte_pmd_i40e_set_vf_vlan_stripq;
rte_pmd_i40e_set_vf_vlan_tag;
+ local: *;
+};
-} DPDK_2.0;
-
-DPDK_17.05 {
- global:
-
- rte_pmd_i40e_set_tc_strict_prio;
- rte_pmd_i40e_set_vf_max_bw;
- rte_pmd_i40e_set_vf_tc_bw_alloc;
- rte_pmd_i40e_set_vf_tc_max_bw;
- rte_pmd_i40e_process_ddp_package;
- rte_pmd_i40e_get_ddp_list;
-
-} DPDK_17.02;
-
-DPDK_17.08 {
- global:
-
- rte_pmd_i40e_get_ddp_info;
-
-} DPDK_17.05;
-
-DPDK_17.11 {
- global:
-
- rte_pmd_i40e_add_vf_mac_addr;
- rte_pmd_i40e_flow_add_del_packet_template;
- rte_pmd_i40e_flow_type_mapping_update;
- rte_pmd_i40e_flow_type_mapping_get;
- rte_pmd_i40e_flow_type_mapping_reset;
- rte_pmd_i40e_query_vfid_by_mac;
- rte_pmd_i40e_rss_queue_region_conf;
-
-} DPDK_17.08;
-
-DPDK_18.02 {
- global:
-
- rte_pmd_i40e_inset_get;
- rte_pmd_i40e_inset_set;
-} DPDK_17.11;
\ No newline at end of file
diff --git a/drivers/net/iavf/rte_pmd_iavf_version.map b/drivers/net/iavf/rte_pmd_iavf_version.map
index 179140fb8..0322b777c 100644
--- a/drivers/net/iavf/rte_pmd_iavf_version.map
+++ b/drivers/net/iavf/rte_pmd_iavf_version.map
@@ -1,4 +1,4 @@
-DPDK_18.02 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/net/ice/rte_pmd_ice_version.map b/drivers/net/ice/rte_pmd_ice_version.map
index 7b23b609d..0322b777c 100644
--- a/drivers/net/ice/rte_pmd_ice_version.map
+++ b/drivers/net/ice/rte_pmd_ice_version.map
@@ -1,4 +1,4 @@
-DPDK_19.02 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/net/ifc/rte_pmd_ifc_version.map b/drivers/net/ifc/rte_pmd_ifc_version.map
index 9b9ab1a4c..0322b777c 100644
--- a/drivers/net/ifc/rte_pmd_ifc_version.map
+++ b/drivers/net/ifc/rte_pmd_ifc_version.map
@@ -1,4 +1,4 @@
-DPDK_18.05 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/net/ipn3ke/rte_pmd_ipn3ke_version.map b/drivers/net/ipn3ke/rte_pmd_ipn3ke_version.map
index fc8c95e91..0322b777c 100644
--- a/drivers/net/ipn3ke/rte_pmd_ipn3ke_version.map
+++ b/drivers/net/ipn3ke/rte_pmd_ipn3ke_version.map
@@ -1,4 +1,4 @@
-DPDK_19.05 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/net/ixgbe/rte_pmd_ixgbe_version.map b/drivers/net/ixgbe/rte_pmd_ixgbe_version.map
index c814f96d7..b117c8d95 100644
--- a/drivers/net/ixgbe/rte_pmd_ixgbe_version.map
+++ b/drivers/net/ixgbe/rte_pmd_ixgbe_version.map
@@ -1,57 +1,38 @@
-DPDK_2.0 {
-
- local: *;
-};
-
-DPDK_16.11 {
- global:
-
- rte_pmd_ixgbe_set_all_queues_drop_en;
- rte_pmd_ixgbe_set_tx_loopback;
- rte_pmd_ixgbe_set_vf_mac_addr;
- rte_pmd_ixgbe_set_vf_mac_anti_spoof;
- rte_pmd_ixgbe_set_vf_split_drop_en;
- rte_pmd_ixgbe_set_vf_vlan_anti_spoof;
- rte_pmd_ixgbe_set_vf_vlan_insert;
- rte_pmd_ixgbe_set_vf_vlan_stripq;
-} DPDK_2.0;
-
-DPDK_17.02 {
+DPDK_20.0 {
global:
+ rte_pmd_ixgbe_bypass_event_show;
+ rte_pmd_ixgbe_bypass_event_store;
+ rte_pmd_ixgbe_bypass_init;
+ rte_pmd_ixgbe_bypass_state_set;
+ rte_pmd_ixgbe_bypass_state_show;
+ rte_pmd_ixgbe_bypass_ver_show;
+ rte_pmd_ixgbe_bypass_wd_reset;
+ rte_pmd_ixgbe_bypass_wd_timeout_show;
+ rte_pmd_ixgbe_bypass_wd_timeout_store;
rte_pmd_ixgbe_macsec_config_rxsc;
rte_pmd_ixgbe_macsec_config_txsc;
rte_pmd_ixgbe_macsec_disable;
rte_pmd_ixgbe_macsec_enable;
rte_pmd_ixgbe_macsec_select_rxsa;
rte_pmd_ixgbe_macsec_select_txsa;
+ rte_pmd_ixgbe_ping_vf;
+ rte_pmd_ixgbe_set_all_queues_drop_en;
+ rte_pmd_ixgbe_set_tc_bw_alloc;
+ rte_pmd_ixgbe_set_tx_loopback;
+ rte_pmd_ixgbe_set_vf_mac_addr;
+ rte_pmd_ixgbe_set_vf_mac_anti_spoof;
rte_pmd_ixgbe_set_vf_rate_limit;
rte_pmd_ixgbe_set_vf_rx;
rte_pmd_ixgbe_set_vf_rxmode;
+ rte_pmd_ixgbe_set_vf_split_drop_en;
rte_pmd_ixgbe_set_vf_tx;
+ rte_pmd_ixgbe_set_vf_vlan_anti_spoof;
rte_pmd_ixgbe_set_vf_vlan_filter;
-} DPDK_16.11;
-
-DPDK_17.05 {
- global:
-
- rte_pmd_ixgbe_ping_vf;
- rte_pmd_ixgbe_set_tc_bw_alloc;
-} DPDK_17.02;
-
-DPDK_17.08 {
- global:
-
- rte_pmd_ixgbe_bypass_event_show;
- rte_pmd_ixgbe_bypass_event_store;
- rte_pmd_ixgbe_bypass_init;
- rte_pmd_ixgbe_bypass_state_set;
- rte_pmd_ixgbe_bypass_state_show;
- rte_pmd_ixgbe_bypass_ver_show;
- rte_pmd_ixgbe_bypass_wd_reset;
- rte_pmd_ixgbe_bypass_wd_timeout_show;
- rte_pmd_ixgbe_bypass_wd_timeout_store;
-} DPDK_17.05;
+ rte_pmd_ixgbe_set_vf_vlan_insert;
+ rte_pmd_ixgbe_set_vf_vlan_stripq;
+ local: *;
+};
EXPERIMENTAL {
global:
diff --git a/drivers/net/kni/rte_pmd_kni_version.map b/drivers/net/kni/rte_pmd_kni_version.map
index 8591cc0b1..0322b777c 100644
--- a/drivers/net/kni/rte_pmd_kni_version.map
+++ b/drivers/net/kni/rte_pmd_kni_version.map
@@ -1,4 +1,4 @@
-DPDK_17.05 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/net/liquidio/rte_pmd_liquidio_version.map b/drivers/net/liquidio/rte_pmd_liquidio_version.map
index 8591cc0b1..0322b777c 100644
--- a/drivers/net/liquidio/rte_pmd_liquidio_version.map
+++ b/drivers/net/liquidio/rte_pmd_liquidio_version.map
@@ -1,4 +1,4 @@
-DPDK_17.05 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/net/memif/rte_pmd_memif_version.map b/drivers/net/memif/rte_pmd_memif_version.map
index 8861484fb..0322b777c 100644
--- a/drivers/net/memif/rte_pmd_memif_version.map
+++ b/drivers/net/memif/rte_pmd_memif_version.map
@@ -1,4 +1,4 @@
-DPDK_19.08 {
+DPDK_20.0 {
- local: *;
+ local: *;
};
diff --git a/drivers/net/mlx4/rte_pmd_mlx4_version.map b/drivers/net/mlx4/rte_pmd_mlx4_version.map
index ef3539840..0322b777c 100644
--- a/drivers/net/mlx4/rte_pmd_mlx4_version.map
+++ b/drivers/net/mlx4/rte_pmd_mlx4_version.map
@@ -1,4 +1,4 @@
-DPDK_2.0 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/net/mlx5/rte_pmd_mlx5_version.map b/drivers/net/mlx5/rte_pmd_mlx5_version.map
index ad607bbed..0322b777c 100644
--- a/drivers/net/mlx5/rte_pmd_mlx5_version.map
+++ b/drivers/net/mlx5/rte_pmd_mlx5_version.map
@@ -1,3 +1,4 @@
-DPDK_2.2 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/net/mvneta/rte_pmd_mvneta_version.map b/drivers/net/mvneta/rte_pmd_mvneta_version.map
index 24bd5cdb3..0322b777c 100644
--- a/drivers/net/mvneta/rte_pmd_mvneta_version.map
+++ b/drivers/net/mvneta/rte_pmd_mvneta_version.map
@@ -1,3 +1,4 @@
-DPDK_18.11 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/net/mvpp2/rte_pmd_mvpp2_version.map b/drivers/net/mvpp2/rte_pmd_mvpp2_version.map
index a75303172..0322b777c 100644
--- a/drivers/net/mvpp2/rte_pmd_mvpp2_version.map
+++ b/drivers/net/mvpp2/rte_pmd_mvpp2_version.map
@@ -1,3 +1,4 @@
-DPDK_17.11 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/net/netvsc/rte_pmd_netvsc_version.map b/drivers/net/netvsc/rte_pmd_netvsc_version.map
index d534019a6..0322b777c 100644
--- a/drivers/net/netvsc/rte_pmd_netvsc_version.map
+++ b/drivers/net/netvsc/rte_pmd_netvsc_version.map
@@ -1,5 +1,4 @@
-/* SPDX-License-Identifier: BSD-3-Clause */
+DPDK_20.0 {
-DPDK_18.08 {
local: *;
};
diff --git a/drivers/net/nfb/rte_pmd_nfb_version.map b/drivers/net/nfb/rte_pmd_nfb_version.map
index fc8c95e91..0322b777c 100644
--- a/drivers/net/nfb/rte_pmd_nfb_version.map
+++ b/drivers/net/nfb/rte_pmd_nfb_version.map
@@ -1,4 +1,4 @@
-DPDK_19.05 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/net/nfp/rte_pmd_nfp_version.map b/drivers/net/nfp/rte_pmd_nfp_version.map
index ad607bbed..0322b777c 100644
--- a/drivers/net/nfp/rte_pmd_nfp_version.map
+++ b/drivers/net/nfp/rte_pmd_nfp_version.map
@@ -1,3 +1,4 @@
-DPDK_2.2 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/net/null/rte_pmd_null_version.map b/drivers/net/null/rte_pmd_null_version.map
index ef3539840..0322b777c 100644
--- a/drivers/net/null/rte_pmd_null_version.map
+++ b/drivers/net/null/rte_pmd_null_version.map
@@ -1,4 +1,4 @@
-DPDK_2.0 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/net/octeontx/rte_pmd_octeontx_version.map b/drivers/net/octeontx/rte_pmd_octeontx_version.map
index a3161b14d..13ac26b39 100644
--- a/drivers/net/octeontx/rte_pmd_octeontx_version.map
+++ b/drivers/net/octeontx/rte_pmd_octeontx_version.map
@@ -1,11 +1,7 @@
-DPDK_17.11 {
-
- local: *;
-};
-
-DPDK_18.02 {
+DPDK_20.0 {
global:
rte_octeontx_pchan_map;
+ local: *;
+};
-} DPDK_17.11;
diff --git a/drivers/net/octeontx2/rte_pmd_octeontx2_version.map b/drivers/net/octeontx2/rte_pmd_octeontx2_version.map
index 9a61188cd..0322b777c 100644
--- a/drivers/net/octeontx2/rte_pmd_octeontx2_version.map
+++ b/drivers/net/octeontx2/rte_pmd_octeontx2_version.map
@@ -1,4 +1,4 @@
-DPDK_19.08 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/net/pcap/rte_pmd_pcap_version.map b/drivers/net/pcap/rte_pmd_pcap_version.map
index ef3539840..0322b777c 100644
--- a/drivers/net/pcap/rte_pmd_pcap_version.map
+++ b/drivers/net/pcap/rte_pmd_pcap_version.map
@@ -1,4 +1,4 @@
-DPDK_2.0 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/net/qede/rte_pmd_qede_version.map b/drivers/net/qede/rte_pmd_qede_version.map
index 349c6e1c2..0322b777c 100644
--- a/drivers/net/qede/rte_pmd_qede_version.map
+++ b/drivers/net/qede/rte_pmd_qede_version.map
@@ -1,4 +1,4 @@
-DPDK_16.04 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/net/ring/rte_pmd_ring_version.map b/drivers/net/ring/rte_pmd_ring_version.map
index 1f785d940..4190d385a 100644
--- a/drivers/net/ring/rte_pmd_ring_version.map
+++ b/drivers/net/ring/rte_pmd_ring_version.map
@@ -1,14 +1,8 @@
-DPDK_2.0 {
+DPDK_20.0 {
global:
+ rte_eth_from_ring;
rte_eth_from_rings;
-
local: *;
};
-DPDK_2.2 {
- global:
-
- rte_eth_from_ring;
-
-} DPDK_2.0;
diff --git a/drivers/net/sfc/rte_pmd_sfc_version.map b/drivers/net/sfc/rte_pmd_sfc_version.map
index 31eca32eb..0322b777c 100644
--- a/drivers/net/sfc/rte_pmd_sfc_version.map
+++ b/drivers/net/sfc/rte_pmd_sfc_version.map
@@ -1,4 +1,4 @@
-DPDK_17.02 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/net/softnic/rte_pmd_softnic_version.map b/drivers/net/softnic/rte_pmd_softnic_version.map
index bc44b06f9..eb4f0b07a 100644
--- a/drivers/net/softnic/rte_pmd_softnic_version.map
+++ b/drivers/net/softnic/rte_pmd_softnic_version.map
@@ -1,8 +1,7 @@
-DPDK_17.11 {
+DPDK_20.0 {
global:
rte_pmd_softnic_run;
-
local: *;
};
diff --git a/drivers/net/szedata2/rte_pmd_szedata2_version.map b/drivers/net/szedata2/rte_pmd_szedata2_version.map
index ad607bbed..0322b777c 100644
--- a/drivers/net/szedata2/rte_pmd_szedata2_version.map
+++ b/drivers/net/szedata2/rte_pmd_szedata2_version.map
@@ -1,3 +1,4 @@
-DPDK_2.2 {
+DPDK_20.0 {
+
local: *;
};
diff --git a/drivers/net/tap/rte_pmd_tap_version.map b/drivers/net/tap/rte_pmd_tap_version.map
index 31eca32eb..0322b777c 100644
--- a/drivers/net/tap/rte_pmd_tap_version.map
+++ b/drivers/net/tap/rte_pmd_tap_version.map
@@ -1,4 +1,4 @@
-DPDK_17.02 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/net/thunderx/rte_pmd_thunderx_version.map b/drivers/net/thunderx/rte_pmd_thunderx_version.map
index 1901bcb3b..0322b777c 100644
--- a/drivers/net/thunderx/rte_pmd_thunderx_version.map
+++ b/drivers/net/thunderx/rte_pmd_thunderx_version.map
@@ -1,4 +1,4 @@
-DPDK_16.07 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/net/vdev_netvsc/rte_pmd_vdev_netvsc_version.map b/drivers/net/vdev_netvsc/rte_pmd_vdev_netvsc_version.map
index 179140fb8..0322b777c 100644
--- a/drivers/net/vdev_netvsc/rte_pmd_vdev_netvsc_version.map
+++ b/drivers/net/vdev_netvsc/rte_pmd_vdev_netvsc_version.map
@@ -1,4 +1,4 @@
-DPDK_18.02 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/net/vhost/rte_pmd_vhost_version.map b/drivers/net/vhost/rte_pmd_vhost_version.map
index 695db8574..9f7e613f2 100644
--- a/drivers/net/vhost/rte_pmd_vhost_version.map
+++ b/drivers/net/vhost/rte_pmd_vhost_version.map
@@ -1,13 +1,8 @@
-DPDK_16.04 {
+DPDK_20.0 {
global:
rte_eth_vhost_get_queue_event;
-
+ rte_eth_vhost_get_vid_from_port_id;
local: *;
};
-DPDK_16.11 {
- global:
-
- rte_eth_vhost_get_vid_from_port_id;
-};
diff --git a/drivers/net/virtio/rte_pmd_virtio_version.map b/drivers/net/virtio/rte_pmd_virtio_version.map
index ef3539840..0322b777c 100644
--- a/drivers/net/virtio/rte_pmd_virtio_version.map
+++ b/drivers/net/virtio/rte_pmd_virtio_version.map
@@ -1,4 +1,4 @@
-DPDK_2.0 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/net/vmxnet3/rte_pmd_vmxnet3_version.map b/drivers/net/vmxnet3/rte_pmd_vmxnet3_version.map
index ef3539840..0322b777c 100644
--- a/drivers/net/vmxnet3/rte_pmd_vmxnet3_version.map
+++ b/drivers/net/vmxnet3/rte_pmd_vmxnet3_version.map
@@ -1,4 +1,4 @@
-DPDK_2.0 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/raw/dpaa2_cmdif/rte_rawdev_dpaa2_cmdif_version.map b/drivers/raw/dpaa2_cmdif/rte_rawdev_dpaa2_cmdif_version.map
index 9b9ab1a4c..0322b777c 100644
--- a/drivers/raw/dpaa2_cmdif/rte_rawdev_dpaa2_cmdif_version.map
+++ b/drivers/raw/dpaa2_cmdif/rte_rawdev_dpaa2_cmdif_version.map
@@ -1,4 +1,4 @@
-DPDK_18.05 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/raw/dpaa2_qdma/rte_rawdev_dpaa2_qdma_version.map b/drivers/raw/dpaa2_qdma/rte_rawdev_dpaa2_qdma_version.map
index d16a136fc..a2420771f 100644
--- a/drivers/raw/dpaa2_qdma/rte_rawdev_dpaa2_qdma_version.map
+++ b/drivers/raw/dpaa2_qdma/rte_rawdev_dpaa2_qdma_version.map
@@ -1,4 +1,4 @@
-DPDK_19.05 {
+DPDK_20.0 {
global:
rte_qdma_attr_get;
@@ -9,12 +9,12 @@ DPDK_19.05 {
rte_qdma_start;
rte_qdma_stop;
rte_qdma_vq_create;
- rte_qdma_vq_destroy;
rte_qdma_vq_dequeue;
rte_qdma_vq_dequeue_multi;
+ rte_qdma_vq_destroy;
rte_qdma_vq_enqueue;
rte_qdma_vq_enqueue_multi;
rte_qdma_vq_stats;
-
local: *;
};
+
diff --git a/drivers/raw/ifpga/rte_rawdev_ifpga_version.map b/drivers/raw/ifpga/rte_rawdev_ifpga_version.map
index 9b9ab1a4c..0322b777c 100644
--- a/drivers/raw/ifpga/rte_rawdev_ifpga_version.map
+++ b/drivers/raw/ifpga/rte_rawdev_ifpga_version.map
@@ -1,4 +1,4 @@
-DPDK_18.05 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/raw/ioat/rte_rawdev_ioat_version.map b/drivers/raw/ioat/rte_rawdev_ioat_version.map
index 9a61188cd..0322b777c 100644
--- a/drivers/raw/ioat/rte_rawdev_ioat_version.map
+++ b/drivers/raw/ioat/rte_rawdev_ioat_version.map
@@ -1,4 +1,4 @@
-DPDK_19.08 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/raw/ntb/rte_rawdev_ntb_version.map b/drivers/raw/ntb/rte_rawdev_ntb_version.map
index 8861484fb..0322b777c 100644
--- a/drivers/raw/ntb/rte_rawdev_ntb_version.map
+++ b/drivers/raw/ntb/rte_rawdev_ntb_version.map
@@ -1,4 +1,4 @@
-DPDK_19.08 {
+DPDK_20.0 {
- local: *;
+ local: *;
};
diff --git a/drivers/raw/octeontx2_dma/rte_rawdev_octeontx2_dma_version.map b/drivers/raw/octeontx2_dma/rte_rawdev_octeontx2_dma_version.map
index 9a61188cd..0322b777c 100644
--- a/drivers/raw/octeontx2_dma/rte_rawdev_octeontx2_dma_version.map
+++ b/drivers/raw/octeontx2_dma/rte_rawdev_octeontx2_dma_version.map
@@ -1,4 +1,4 @@
-DPDK_19.08 {
+DPDK_20.0 {
local: *;
};
diff --git a/drivers/raw/skeleton/rte_rawdev_skeleton_version.map b/drivers/raw/skeleton/rte_rawdev_skeleton_version.map
index 179140fb8..0322b777c 100644
--- a/drivers/raw/skeleton/rte_rawdev_skeleton_version.map
+++ b/drivers/raw/skeleton/rte_rawdev_skeleton_version.map
@@ -1,4 +1,4 @@
-DPDK_18.02 {
+DPDK_20.0 {
local: *;
};
diff --git a/lib/librte_acl/rte_acl_version.map b/lib/librte_acl/rte_acl_version.map
index b09370a10..6a5d7034a 100644
--- a/lib/librte_acl/rte_acl_version.map
+++ b/lib/librte_acl/rte_acl_version.map
@@ -1,4 +1,4 @@
-DPDK_2.0 {
+DPDK_20.0 {
global:
rte_acl_add_rules;
@@ -14,6 +14,6 @@ DPDK_2.0 {
rte_acl_reset;
rte_acl_reset_rules;
rte_acl_set_ctx_classify;
-
local: *;
};
+
diff --git a/lib/librte_bbdev/rte_bbdev_version.map b/lib/librte_bbdev/rte_bbdev_version.map
index 3624eb1cb..7ca27ca67 100644
--- a/lib/librte_bbdev/rte_bbdev_version.map
+++ b/lib/librte_bbdev/rte_bbdev_version.map
@@ -36,6 +36,5 @@ EXPERIMENTAL {
rte_bbdev_stats_get;
rte_bbdev_stats_reset;
rte_bbdev_stop;
-
local: *;
};
diff --git a/lib/librte_bitratestats/rte_bitratestats_version.map b/lib/librte_bitratestats/rte_bitratestats_version.map
index fe7454452..0261427b9 100644
--- a/lib/librte_bitratestats/rte_bitratestats_version.map
+++ b/lib/librte_bitratestats/rte_bitratestats_version.map
@@ -1,9 +1,9 @@
-DPDK_17.05 {
+DPDK_20.0 {
global:
rte_stats_bitrate_calc;
rte_stats_bitrate_create;
rte_stats_bitrate_reg;
-
local: *;
};
+
diff --git a/lib/librte_bpf/rte_bpf_version.map b/lib/librte_bpf/rte_bpf_version.map
index a203e088e..b183ba566 100644
--- a/lib/librte_bpf/rte_bpf_version.map
+++ b/lib/librte_bpf/rte_bpf_version.map
@@ -11,6 +11,5 @@ EXPERIMENTAL {
rte_bpf_exec_burst;
rte_bpf_get_jit;
rte_bpf_load;
-
local: *;
};
diff --git a/lib/librte_cfgfile/rte_cfgfile_version.map b/lib/librte_cfgfile/rte_cfgfile_version.map
index a0a11cea8..255523103 100644
--- a/lib/librte_cfgfile/rte_cfgfile_version.map
+++ b/lib/librte_cfgfile/rte_cfgfile_version.map
@@ -1,40 +1,22 @@
-DPDK_2.0 {
+DPDK_20.0 {
global:
+ rte_cfgfile_add_entry;
+ rte_cfgfile_add_section;
rte_cfgfile_close;
+ rte_cfgfile_create;
rte_cfgfile_get_entry;
rte_cfgfile_has_entry;
rte_cfgfile_has_section;
rte_cfgfile_load;
+ rte_cfgfile_load_with_params;
rte_cfgfile_num_sections;
+ rte_cfgfile_save;
rte_cfgfile_section_entries;
+ rte_cfgfile_section_entries_by_index;
rte_cfgfile_section_num_entries;
rte_cfgfile_sections;
-
+ rte_cfgfile_set_entry;
local: *;
};
-DPDK_16.04 {
- global:
-
- rte_cfgfile_section_entries_by_index;
-
-} DPDK_2.0;
-
-DPDK_17.05 {
- global:
-
- rte_cfgfile_load_with_params;
-
-} DPDK_16.04;
-
-DPDK_17.11 {
- global:
-
- rte_cfgfile_add_entry;
- rte_cfgfile_add_section;
- rte_cfgfile_create;
- rte_cfgfile_save;
- rte_cfgfile_set_entry;
-
-} DPDK_17.05;
diff --git a/lib/librte_cmdline/rte_cmdline_version.map b/lib/librte_cmdline/rte_cmdline_version.map
index 04bcb387f..29788e1ef 100644
--- a/lib/librte_cmdline/rte_cmdline_version.map
+++ b/lib/librte_cmdline/rte_cmdline_version.map
@@ -1,4 +1,4 @@
-DPDK_2.0 {
+DPDK_20.0 {
global:
cirbuf_add_buf_head;
@@ -40,6 +40,7 @@ DPDK_2.0 {
cmdline_parse_num;
cmdline_parse_portlist;
cmdline_parse_string;
+ cmdline_poll;
cmdline_printf;
cmdline_quit;
cmdline_set_prompt;
@@ -65,13 +66,6 @@ DPDK_2.0 {
rdline_stop;
vt100_init;
vt100_parser;
-
local: *;
};
-DPDK_2.1 {
- global:
-
- cmdline_poll;
-
-} DPDK_2.0;
diff --git a/lib/librte_compressdev/rte_compressdev_version.map b/lib/librte_compressdev/rte_compressdev_version.map
index e2a108b65..6e3acd83e 100644
--- a/lib/librte_compressdev/rte_compressdev_version.map
+++ b/lib/librte_compressdev/rte_compressdev_version.map
@@ -1,6 +1,12 @@
EXPERIMENTAL {
global:
+ rte_comp_get_feature_name;
+ rte_comp_op_alloc;
+ rte_comp_op_bulk_alloc;
+ rte_comp_op_bulk_free;
+ rte_comp_op_free;
+ rte_comp_op_pool_create;
rte_compressdev_capability_get;
rte_compressdev_close;
rte_compressdev_configure;
@@ -29,12 +35,5 @@ EXPERIMENTAL {
rte_compressdev_stop;
rte_compressdev_stream_create;
rte_compressdev_stream_free;
- rte_comp_get_feature_name;
- rte_comp_op_alloc;
- rte_comp_op_bulk_alloc;
- rte_comp_op_bulk_free;
- rte_comp_op_free;
- rte_comp_op_pool_create;
-
local: *;
};
diff --git a/lib/librte_cryptodev/rte_cryptodev_version.map b/lib/librte_cryptodev/rte_cryptodev_version.map
index 3deb265ac..a95776567 100644
--- a/lib/librte_cryptodev/rte_cryptodev_version.map
+++ b/lib/librte_cryptodev/rte_cryptodev_version.map
@@ -1,96 +1,67 @@
-DPDK_16.04 {
+DPDK_20.0 {
global:
- rte_cryptodevs;
+ rte_crypto_aead_algorithm_strings;
+ rte_crypto_aead_operation_strings;
+ rte_crypto_auth_algorithm_strings;
+ rte_crypto_auth_operation_strings;
+ rte_crypto_cipher_algorithm_strings;
+ rte_crypto_cipher_operation_strings;
+ rte_crypto_op_pool_create;
+ rte_cryptodev_allocate_driver;
rte_cryptodev_callback_register;
rte_cryptodev_callback_unregister;
rte_cryptodev_close;
- rte_cryptodev_count;
rte_cryptodev_configure;
+ rte_cryptodev_count;
+ rte_cryptodev_device_count_by_driver;
+ rte_cryptodev_devices_get;
+ rte_cryptodev_driver_id_get;
+ rte_cryptodev_driver_name_get;
+ rte_cryptodev_get_aead_algo_enum;
+ rte_cryptodev_get_auth_algo_enum;
+ rte_cryptodev_get_cipher_algo_enum;
rte_cryptodev_get_dev_id;
rte_cryptodev_get_feature_name;
+ rte_cryptodev_get_sec_ctx;
rte_cryptodev_info_get;
+ rte_cryptodev_name_get;
rte_cryptodev_pmd_allocate;
rte_cryptodev_pmd_callback_process;
+ rte_cryptodev_pmd_create;
+ rte_cryptodev_pmd_create_dev_name;
+ rte_cryptodev_pmd_destroy;
+ rte_cryptodev_pmd_get_dev;
+ rte_cryptodev_pmd_get_named_dev;
+ rte_cryptodev_pmd_is_valid_dev;
+ rte_cryptodev_pmd_parse_input_args;
rte_cryptodev_pmd_release_device;
- rte_cryptodev_sym_session_create;
- rte_cryptodev_sym_session_free;
+ rte_cryptodev_queue_pair_count;
+ rte_cryptodev_queue_pair_setup;
rte_cryptodev_socket_id;
rte_cryptodev_start;
rte_cryptodev_stats_get;
rte_cryptodev_stats_reset;
rte_cryptodev_stop;
- rte_cryptodev_queue_pair_count;
- rte_cryptodev_queue_pair_setup;
- rte_crypto_op_pool_create;
-
- local: *;
-};
-
-DPDK_17.02 {
- global:
-
- rte_cryptodev_devices_get;
- rte_cryptodev_pmd_create_dev_name;
- rte_cryptodev_pmd_get_dev;
- rte_cryptodev_pmd_get_named_dev;
- rte_cryptodev_pmd_is_valid_dev;
+ rte_cryptodev_sym_capability_check_aead;
rte_cryptodev_sym_capability_check_auth;
rte_cryptodev_sym_capability_check_cipher;
rte_cryptodev_sym_capability_get;
- rte_crypto_auth_algorithm_strings;
- rte_crypto_auth_operation_strings;
- rte_crypto_cipher_algorithm_strings;
- rte_crypto_cipher_operation_strings;
-
-} DPDK_16.04;
-
-DPDK_17.05 {
- global:
-
- rte_cryptodev_get_auth_algo_enum;
- rte_cryptodev_get_cipher_algo_enum;
-
-} DPDK_17.02;
-
-DPDK_17.08 {
- global:
-
- rte_cryptodev_allocate_driver;
- rte_cryptodev_device_count_by_driver;
- rte_cryptodev_driver_id_get;
- rte_cryptodev_driver_name_get;
- rte_cryptodev_get_aead_algo_enum;
- rte_cryptodev_sym_capability_check_aead;
- rte_cryptodev_sym_session_init;
- rte_cryptodev_sym_session_clear;
- rte_crypto_aead_algorithm_strings;
- rte_crypto_aead_operation_strings;
-
-} DPDK_17.05;
-
-DPDK_17.11 {
- global:
-
- rte_cryptodev_get_sec_ctx;
- rte_cryptodev_name_get;
- rte_cryptodev_pmd_create;
- rte_cryptodev_pmd_destroy;
- rte_cryptodev_pmd_parse_input_args;
-
-} DPDK_17.08;
-
-DPDK_18.05 {
- global:
-
rte_cryptodev_sym_get_header_session_size;
rte_cryptodev_sym_get_private_session_size;
-
-} DPDK_17.11;
+ rte_cryptodev_sym_session_clear;
+ rte_cryptodev_sym_session_create;
+ rte_cryptodev_sym_session_free;
+ rte_cryptodev_sym_session_init;
+ rte_cryptodevs;
+ local: *;
+};
EXPERIMENTAL {
global:
+ rte_crypto_asym_op_strings;
+ rte_crypto_asym_xform_strings;
rte_cryptodev_asym_capability_get;
rte_cryptodev_asym_get_header_session_size;
rte_cryptodev_asym_get_private_session_size;
@@ -105,6 +76,4 @@ EXPERIMENTAL {
rte_cryptodev_sym_session_get_user_data;
rte_cryptodev_sym_session_pool_create;
rte_cryptodev_sym_session_set_user_data;
- rte_crypto_asym_op_strings;
- rte_crypto_asym_xform_strings;
};
diff --git a/lib/librte_distributor/rte_distributor.c b/lib/librte_distributor/rte_distributor.c
index 21eb1fb0a..edc942317 100644
--- a/lib/librte_distributor/rte_distributor.c
+++ b/lib/librte_distributor/rte_distributor.c
@@ -78,7 +78,7 @@ rte_distributor_request_pkt_v1705(struct rte_distributor *d,
*/
*retptr64 |= RTE_DISTRIB_GET_BUF;
}
-BIND_DEFAULT_SYMBOL(rte_distributor_request_pkt, _v1705, 17.05);
+BIND_DEFAULT_SYMBOL(rte_distributor_request_pkt, _v1705, 20.0);
MAP_STATIC_SYMBOL(void rte_distributor_request_pkt(struct rte_distributor *d,
unsigned int worker_id, struct rte_mbuf **oldpkt,
unsigned int count),
@@ -119,7 +119,7 @@ rte_distributor_poll_pkt_v1705(struct rte_distributor *d,
return count;
}
-BIND_DEFAULT_SYMBOL(rte_distributor_poll_pkt, _v1705, 17.05);
+BIND_DEFAULT_SYMBOL(rte_distributor_poll_pkt, _v1705, 20.0);
MAP_STATIC_SYMBOL(int rte_distributor_poll_pkt(struct rte_distributor *d,
unsigned int worker_id, struct rte_mbuf **pkts),
rte_distributor_poll_pkt_v1705);
@@ -153,7 +153,7 @@ rte_distributor_get_pkt_v1705(struct rte_distributor *d,
}
return count;
}
-BIND_DEFAULT_SYMBOL(rte_distributor_get_pkt, _v1705, 17.05);
+BIND_DEFAULT_SYMBOL(rte_distributor_get_pkt, _v1705, 20.0);
MAP_STATIC_SYMBOL(int rte_distributor_get_pkt(struct rte_distributor *d,
unsigned int worker_id, struct rte_mbuf **pkts,
struct rte_mbuf **oldpkt, unsigned int return_count),
@@ -187,7 +187,7 @@ rte_distributor_return_pkt_v1705(struct rte_distributor *d,
return 0;
}
-BIND_DEFAULT_SYMBOL(rte_distributor_return_pkt, _v1705, 17.05);
+BIND_DEFAULT_SYMBOL(rte_distributor_return_pkt, _v1705, 20.0);
MAP_STATIC_SYMBOL(int rte_distributor_return_pkt(struct rte_distributor *d,
unsigned int worker_id, struct rte_mbuf **oldpkt, int num),
rte_distributor_return_pkt_v1705);
@@ -470,7 +470,7 @@ rte_distributor_process_v1705(struct rte_distributor *d,
return num_mbufs;
}
-BIND_DEFAULT_SYMBOL(rte_distributor_process, _v1705, 17.05);
+BIND_DEFAULT_SYMBOL(rte_distributor_process, _v1705, 20.0);
MAP_STATIC_SYMBOL(int rte_distributor_process(struct rte_distributor *d,
struct rte_mbuf **mbufs, unsigned int num_mbufs),
rte_distributor_process_v1705);
@@ -502,7 +502,7 @@ rte_distributor_returned_pkts_v1705(struct rte_distributor *d,
return retval;
}
-BIND_DEFAULT_SYMBOL(rte_distributor_returned_pkts, _v1705, 17.05);
+BIND_DEFAULT_SYMBOL(rte_distributor_returned_pkts, _v1705, 20.0);
MAP_STATIC_SYMBOL(int rte_distributor_returned_pkts(struct rte_distributor *d,
struct rte_mbuf **mbufs, unsigned int max_mbufs),
rte_distributor_returned_pkts_v1705);
@@ -556,7 +556,7 @@ rte_distributor_flush_v1705(struct rte_distributor *d)
return flushed;
}
-BIND_DEFAULT_SYMBOL(rte_distributor_flush, _v1705, 17.05);
+BIND_DEFAULT_SYMBOL(rte_distributor_flush, _v1705, 20.0);
MAP_STATIC_SYMBOL(int rte_distributor_flush(struct rte_distributor *d),
rte_distributor_flush_v1705);
@@ -576,7 +576,7 @@ rte_distributor_clear_returns_v1705(struct rte_distributor *d)
for (wkr = 0; wkr < d->num_workers; wkr++)
d->bufs[wkr].retptr64[0] = 0;
}
-BIND_DEFAULT_SYMBOL(rte_distributor_clear_returns, _v1705, 17.05);
+BIND_DEFAULT_SYMBOL(rte_distributor_clear_returns, _v1705, 20.0);
MAP_STATIC_SYMBOL(void rte_distributor_clear_returns(struct rte_distributor *d),
rte_distributor_clear_returns_v1705);
@@ -656,7 +656,7 @@ rte_distributor_create_v1705(const char *name,
return d;
}
-BIND_DEFAULT_SYMBOL(rte_distributor_create, _v1705, 17.05);
+BIND_DEFAULT_SYMBOL(rte_distributor_create, _v1705, 20.0);
MAP_STATIC_SYMBOL(struct rte_distributor *rte_distributor_create(
const char *name, unsigned int socket_id,
unsigned int num_workers, unsigned int alg_type),
diff --git a/lib/librte_distributor/rte_distributor_v20.c b/lib/librte_distributor/rte_distributor_v20.c
index cdc0969a8..14ee0360e 100644
--- a/lib/librte_distributor/rte_distributor_v20.c
+++ b/lib/librte_distributor/rte_distributor_v20.c
@@ -38,7 +38,6 @@ rte_distributor_request_pkt_v20(struct rte_distributor_v20 *d,
rte_pause();
buf->bufptr64 = req;
}
-VERSION_SYMBOL(rte_distributor_request_pkt, _v20, 2.0);
struct rte_mbuf *
rte_distributor_poll_pkt_v20(struct rte_distributor_v20 *d,
@@ -52,7 +51,6 @@ rte_distributor_poll_pkt_v20(struct rte_distributor_v20 *d,
int64_t ret = buf->bufptr64 >> RTE_DISTRIB_FLAG_BITS;
return (struct rte_mbuf *)((uintptr_t)ret);
}
-VERSION_SYMBOL(rte_distributor_poll_pkt, _v20, 2.0);
struct rte_mbuf *
rte_distributor_get_pkt_v20(struct rte_distributor_v20 *d,
@@ -64,7 +62,6 @@ rte_distributor_get_pkt_v20(struct rte_distributor_v20 *d,
rte_pause();
return ret;
}
-VERSION_SYMBOL(rte_distributor_get_pkt, _v20, 2.0);
int
rte_distributor_return_pkt_v20(struct rte_distributor_v20 *d,
@@ -76,7 +73,6 @@ rte_distributor_return_pkt_v20(struct rte_distributor_v20 *d,
buf->bufptr64 = req;
return 0;
}
-VERSION_SYMBOL(rte_distributor_return_pkt, _v20, 2.0);
/**** APIs called on distributor core ***/
@@ -293,7 +289,6 @@ rte_distributor_process_v20(struct rte_distributor_v20 *d,
d->returns.count = ret_count;
return num_mbufs;
}
-VERSION_SYMBOL(rte_distributor_process, _v20, 2.0);
/* return to the caller, packets returned from workers */
int
@@ -314,7 +309,6 @@ rte_distributor_returned_pkts_v20(struct rte_distributor_v20 *d,
return retval;
}
-VERSION_SYMBOL(rte_distributor_returned_pkts, _v20, 2.0);
/* return the number of packets in-flight in a distributor, i.e. packets
* being worked on or queued up in a backlog.
@@ -344,7 +338,6 @@ rte_distributor_flush_v20(struct rte_distributor_v20 *d)
return flushed;
}
-VERSION_SYMBOL(rte_distributor_flush, _v20, 2.0);
/* clears the internal returns array in the distributor */
void
@@ -355,7 +348,6 @@ rte_distributor_clear_returns_v20(struct rte_distributor_v20 *d)
memset(d->returns.mbufs, 0, sizeof(d->returns.mbufs));
#endif
}
-VERSION_SYMBOL(rte_distributor_clear_returns, _v20, 2.0);
/* creates a distributor instance */
struct rte_distributor_v20 *
@@ -399,4 +391,3 @@ rte_distributor_create_v20(const char *name,
return d;
}
-VERSION_SYMBOL(rte_distributor_create, _v20, 2.0);
diff --git a/lib/librte_distributor/rte_distributor_version.map b/lib/librte_distributor/rte_distributor_version.map
index 3a285b394..1656d129a 100644
--- a/lib/librte_distributor/rte_distributor_version.map
+++ b/lib/librte_distributor/rte_distributor_version.map
@@ -1,4 +1,4 @@
-DPDK_2.0 {
+DPDK_20.0 {
global:
rte_distributor_clear_returns;
@@ -10,20 +10,6 @@ DPDK_2.0 {
rte_distributor_request_pkt;
rte_distributor_return_pkt;
rte_distributor_returned_pkts;
-
local: *;
};
-DPDK_17.05 {
- global:
-
- rte_distributor_clear_returns;
- rte_distributor_create;
- rte_distributor_flush;
- rte_distributor_get_pkt;
- rte_distributor_poll_pkt;
- rte_distributor_process;
- rte_distributor_request_pkt;
- rte_distributor_return_pkt;
- rte_distributor_returned_pkts;
-} DPDK_2.0;
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index 7cbf82d37..343c771f7 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -1,4 +1,4 @@
-DPDK_2.0 {
+DPDK_20.0 {
global:
__rte_panic;
@@ -7,46 +7,111 @@ DPDK_2.0 {
lcore_config;
per_lcore__lcore_id;
per_lcore__rte_errno;
+ rte_bus_dump;
+ rte_bus_find;
+ rte_bus_find_by_device;
+ rte_bus_find_by_name;
+ rte_bus_get_iommu_class;
+ rte_bus_probe;
+ rte_bus_register;
+ rte_bus_scan;
+ rte_bus_unregister;
rte_calloc;
rte_calloc_socket;
rte_cpu_check_supported;
rte_cpu_get_flag_enabled;
+ rte_cpu_get_flag_name;
+ rte_cpu_is_supported;
+ rte_ctrl_thread_create;
rte_cycles_vmware_tsc_map;
rte_delay_us;
+ rte_delay_us_block;
+ rte_delay_us_callback_register;
+ rte_dev_is_probed;
+ rte_dev_probe;
+ rte_dev_remove;
+ rte_devargs_add;
+ rte_devargs_dump;
+ rte_devargs_insert;
+ rte_devargs_next;
+ rte_devargs_parse;
+ rte_devargs_parsef;
+ rte_devargs_remove;
+ rte_devargs_type_count;
rte_dump_physmem_layout;
rte_dump_registers;
rte_dump_stack;
rte_dump_tailq;
rte_eal_alarm_cancel;
rte_eal_alarm_set;
+ rte_eal_cleanup;
+ rte_eal_create_uio_dev;
rte_eal_get_configuration;
rte_eal_get_lcore_state;
rte_eal_get_physmem_size;
+ rte_eal_get_runtime_dir;
rte_eal_has_hugepages;
+ rte_eal_has_pci;
+ rte_eal_hotplug_add;
+ rte_eal_hotplug_remove;
rte_eal_hpet_init;
rte_eal_init;
rte_eal_iopl_init;
+ rte_eal_iova_mode;
rte_eal_lcore_role;
+ rte_eal_mbuf_user_pool_ops;
rte_eal_mp_remote_launch;
rte_eal_mp_wait_lcore;
+ rte_eal_primary_proc_alive;
rte_eal_process_type;
rte_eal_remote_launch;
rte_eal_tailq_lookup;
rte_eal_tailq_register;
+ rte_eal_using_phys_addrs;
+ rte_eal_vfio_intr_mode;
rte_eal_wait_lcore;
+ rte_epoll_ctl;
+ rte_epoll_wait;
rte_exit;
rte_free;
rte_get_hpet_cycles;
rte_get_hpet_hz;
rte_get_tsc_hz;
rte_hexdump;
+ rte_hypervisor_get;
+ rte_hypervisor_get_name;
+ rte_intr_allow_others;
rte_intr_callback_register;
rte_intr_callback_unregister;
+ rte_intr_cap_multiple;
rte_intr_disable;
+ rte_intr_dp_is_en;
+ rte_intr_efd_disable;
+ rte_intr_efd_enable;
rte_intr_enable;
+ rte_intr_free_epoll_fd;
+ rte_intr_rx_ctl;
+ rte_intr_tls_epfd;
+ rte_keepalive_create;
+ rte_keepalive_dispatch_pings;
+ rte_keepalive_mark_alive;
+ rte_keepalive_mark_sleep;
+ rte_keepalive_register_core;
+ rte_keepalive_register_relay_callback;
+ rte_lcore_has_role;
+ rte_lcore_index;
+ rte_lcore_to_socket_id;
rte_log;
rte_log_cur_msg_loglevel;
rte_log_cur_msg_logtype;
+ rte_log_dump;
+ rte_log_get_global_level;
+ rte_log_get_level;
+ rte_log_register;
+ rte_log_set_global_level;
+ rte_log_set_level;
+ rte_log_set_level_pattern;
+ rte_log_set_level_regexp;
rte_logs;
rte_malloc;
rte_malloc_dump_stats;
@@ -54,155 +119,38 @@ DPDK_2.0 {
rte_malloc_set_limit;
rte_malloc_socket;
rte_malloc_validate;
+ rte_malloc_virt2iova;
+ rte_mcfg_mem_read_lock;
+ rte_mcfg_mem_read_unlock;
+ rte_mcfg_mem_write_lock;
+ rte_mcfg_mem_write_unlock;
+ rte_mcfg_mempool_read_lock;
+ rte_mcfg_mempool_read_unlock;
+ rte_mcfg_mempool_write_lock;
+ rte_mcfg_mempool_write_unlock;
+ rte_mcfg_tailq_read_lock;
+ rte_mcfg_tailq_read_unlock;
+ rte_mcfg_tailq_write_lock;
+ rte_mcfg_tailq_write_unlock;
rte_mem_lock_page;
+ rte_mem_virt2iova;
rte_mem_virt2phy;
rte_memdump;
rte_memory_get_nchannel;
rte_memory_get_nrank;
rte_memzone_dump;
+ rte_memzone_free;
rte_memzone_lookup;
rte_memzone_reserve;
rte_memzone_reserve_aligned;
rte_memzone_reserve_bounded;
rte_memzone_walk;
rte_openlog_stream;
+ rte_rand;
rte_realloc;
- rte_set_application_usage_hook;
- rte_socket_id;
- rte_strerror;
- rte_strsplit;
- rte_sys_gettid;
- rte_thread_get_affinity;
- rte_thread_set_affinity;
- rte_vlog;
- rte_zmalloc;
- rte_zmalloc_socket;
-
- local: *;
-};
-
-DPDK_2.1 {
- global:
-
- rte_epoll_ctl;
- rte_epoll_wait;
- rte_intr_allow_others;
- rte_intr_dp_is_en;
- rte_intr_efd_disable;
- rte_intr_efd_enable;
- rte_intr_rx_ctl;
- rte_intr_tls_epfd;
- rte_memzone_free;
-
-} DPDK_2.0;
-
-DPDK_2.2 {
- global:
-
- rte_intr_cap_multiple;
- rte_keepalive_create;
- rte_keepalive_dispatch_pings;
- rte_keepalive_mark_alive;
- rte_keepalive_register_core;
-
-} DPDK_2.1;
-
-DPDK_16.04 {
- global:
-
- rte_cpu_get_flag_name;
- rte_eal_primary_proc_alive;
-
-} DPDK_2.2;
-
-DPDK_16.07 {
- global:
-
- rte_keepalive_mark_sleep;
- rte_keepalive_register_relay_callback;
- rte_rtm_supported;
- rte_thread_setname;
-
-} DPDK_16.04;
-
-DPDK_16.11 {
- global:
-
- rte_delay_us_block;
- rte_delay_us_callback_register;
-
-} DPDK_16.07;
-
-DPDK_17.02 {
- global:
-
- rte_bus_dump;
- rte_bus_probe;
- rte_bus_register;
- rte_bus_scan;
- rte_bus_unregister;
-
-} DPDK_16.11;
-
-DPDK_17.05 {
- global:
-
- rte_cpu_is_supported;
- rte_intr_free_epoll_fd;
- rte_log_dump;
- rte_log_get_global_level;
- rte_log_register;
- rte_log_set_global_level;
- rte_log_set_level;
- rte_log_set_level_regexp;
-
-} DPDK_17.02;
-
-DPDK_17.08 {
- global:
-
- rte_bus_find;
- rte_bus_find_by_device;
- rte_bus_find_by_name;
- rte_log_get_level;
-
-} DPDK_17.05;
-
-DPDK_17.11 {
- global:
-
- rte_eal_create_uio_dev;
- rte_bus_get_iommu_class;
- rte_eal_has_pci;
- rte_eal_iova_mode;
- rte_eal_using_phys_addrs;
- rte_eal_vfio_intr_mode;
- rte_lcore_has_role;
- rte_malloc_virt2iova;
- rte_mem_virt2iova;
- rte_vfio_enable;
- rte_vfio_is_enabled;
- rte_vfio_noiommu_is_enabled;
- rte_vfio_release_device;
- rte_vfio_setup_device;
-
-} DPDK_17.08;
-
-DPDK_18.02 {
- global:
-
- rte_hypervisor_get;
- rte_hypervisor_get_name;
- rte_vfio_clear_group;
rte_reciprocal_value;
rte_reciprocal_value_u64;
-
-} DPDK_17.11;
-
-DPDK_18.05 {
- global:
-
- rte_log_set_level_pattern;
+ rte_rtm_supported;
rte_service_attr_get;
rte_service_attr_reset_all;
rte_service_component_register;
@@ -215,6 +163,8 @@ DPDK_18.05 {
rte_service_get_count;
rte_service_get_name;
rte_service_lcore_add;
+ rte_service_lcore_attr_get;
+ rte_service_lcore_attr_reset_all;
rte_service_lcore_count;
rte_service_lcore_count_services;
rte_service_lcore_del;
@@ -224,6 +174,7 @@ DPDK_18.05 {
rte_service_lcore_stop;
rte_service_map_lcore_get;
rte_service_map_lcore_set;
+ rte_service_may_be_active;
rte_service_probe_capability;
rte_service_run_iter_on_app_lcore;
rte_service_runstate_get;
@@ -231,17 +182,23 @@ DPDK_18.05 {
rte_service_set_runstate_mapped_check;
rte_service_set_stats_enable;
rte_service_start_with_defaults;
-
-} DPDK_18.02;
-
-DPDK_18.08 {
- global:
-
- rte_eal_mbuf_user_pool_ops;
+ rte_set_application_usage_hook;
+ rte_socket_count;
+ rte_socket_id;
+ rte_socket_id_by_idx;
+ rte_srand;
+ rte_strerror;
+ rte_strscpy;
+ rte_strsplit;
+ rte_sys_gettid;
+ rte_thread_get_affinity;
+ rte_thread_set_affinity;
+ rte_thread_setname;
rte_uuid_compare;
rte_uuid_is_null;
rte_uuid_parse;
rte_uuid_unparse;
+ rte_vfio_clear_group;
rte_vfio_container_create;
rte_vfio_container_destroy;
rte_vfio_container_dma_map;
@@ -250,86 +207,49 @@ DPDK_18.08 {
rte_vfio_container_group_unbind;
rte_vfio_dma_map;
rte_vfio_dma_unmap;
+ rte_vfio_enable;
rte_vfio_get_container_fd;
rte_vfio_get_group_fd;
rte_vfio_get_group_num;
-
-} DPDK_18.05;
-
-DPDK_18.11 {
- global:
-
- rte_dev_probe;
- rte_dev_remove;
- rte_eal_get_runtime_dir;
- rte_eal_hotplug_add;
- rte_eal_hotplug_remove;
- rte_strscpy;
-
-} DPDK_18.08;
-
-DPDK_19.05 {
- global:
-
- rte_ctrl_thread_create;
- rte_dev_is_probed;
- rte_devargs_add;
- rte_devargs_dump;
- rte_devargs_insert;
- rte_devargs_next;
- rte_devargs_parse;
- rte_devargs_parsef;
- rte_devargs_remove;
- rte_devargs_type_count;
- rte_eal_cleanup;
- rte_socket_count;
- rte_socket_id_by_idx;
-
-} DPDK_18.11;
-
-DPDK_19.08 {
- global:
-
- rte_lcore_index;
- rte_lcore_to_socket_id;
- rte_mcfg_mem_read_lock;
- rte_mcfg_mem_read_unlock;
- rte_mcfg_mem_write_lock;
- rte_mcfg_mem_write_unlock;
- rte_mcfg_mempool_read_lock;
- rte_mcfg_mempool_read_unlock;
- rte_mcfg_mempool_write_lock;
- rte_mcfg_mempool_write_unlock;
- rte_mcfg_tailq_read_lock;
- rte_mcfg_tailq_read_unlock;
- rte_mcfg_tailq_write_lock;
- rte_mcfg_tailq_write_unlock;
- rte_rand;
- rte_service_lcore_attr_get;
- rte_service_lcore_attr_reset_all;
- rte_service_may_be_active;
- rte_srand;
-
-} DPDK_19.05;
+ rte_vfio_is_enabled;
+ rte_vfio_noiommu_is_enabled;
+ rte_vfio_release_device;
+ rte_vfio_setup_device;
+ rte_vlog;
+ rte_zmalloc;
+ rte_zmalloc_socket;
+ local: *;
+};
EXPERIMENTAL {
global:
- # added in 18.02
- rte_mp_action_register;
- rte_mp_action_unregister;
- rte_mp_reply;
- rte_mp_sendmsg;
-
- # added in 18.05
+ rte_class_find;
+ rte_class_find_by_name;
+ rte_class_register;
+ rte_class_unregister;
+ rte_delay_us_sleep;
+ rte_dev_dma_map;
+ rte_dev_dma_unmap;
+ rte_dev_event_callback_process;
rte_dev_event_callback_register;
rte_dev_event_callback_unregister;
rte_dev_event_monitor_start;
rte_dev_event_monitor_stop;
+ rte_dev_hotplug_handle_disable;
+ rte_dev_hotplug_handle_enable;
+ rte_dev_iterator_init;
+ rte_dev_iterator_next;
+ rte_extmem_attach;
+ rte_extmem_detach;
+ rte_extmem_register;
+ rte_extmem_unregister;
rte_fbarray_attach;
rte_fbarray_destroy;
rte_fbarray_detach;
rte_fbarray_dump_metadata;
+ rte_fbarray_find_biggest_free;
+ rte_fbarray_find_biggest_used;
rte_fbarray_find_contig_free;
rte_fbarray_find_contig_used;
rte_fbarray_find_idx;
@@ -337,49 +257,25 @@ EXPERIMENTAL {
rte_fbarray_find_next_n_free;
rte_fbarray_find_next_n_used;
rte_fbarray_find_next_used;
+ rte_fbarray_find_prev_free;
+ rte_fbarray_find_prev_n_free;
+ rte_fbarray_find_prev_n_used;
+ rte_fbarray_find_prev_used;
+ rte_fbarray_find_rev_biggest_free;
+ rte_fbarray_find_rev_biggest_used;
+ rte_fbarray_find_rev_contig_free;
+ rte_fbarray_find_rev_contig_used;
rte_fbarray_get;
rte_fbarray_init;
rte_fbarray_is_used;
rte_fbarray_set_free;
rte_fbarray_set_used;
+ rte_intr_ack;
+ rte_intr_callback_unregister_pending;
+ rte_lcore_cpuset;
+ rte_lcore_to_cpu_id;
rte_log_register_type_and_pick_level;
rte_malloc_dump_heaps;
- rte_mem_alloc_validator_register;
- rte_mem_alloc_validator_unregister;
- rte_mem_check_dma_mask;
- rte_mem_event_callback_register;
- rte_mem_event_callback_unregister;
- rte_mem_iova2virt;
- rte_mem_virt2memseg;
- rte_mem_virt2memseg_list;
- rte_memseg_contig_walk;
- rte_memseg_list_walk;
- rte_memseg_walk;
- rte_mp_request_async;
- rte_mp_request_sync;
-
- # added in 18.08
- rte_class_find;
- rte_class_find_by_name;
- rte_class_register;
- rte_class_unregister;
- rte_dev_iterator_init;
- rte_dev_iterator_next;
- rte_fbarray_find_prev_free;
- rte_fbarray_find_prev_n_free;
- rte_fbarray_find_prev_n_used;
- rte_fbarray_find_prev_used;
- rte_fbarray_find_rev_contig_free;
- rte_fbarray_find_rev_contig_used;
- rte_memseg_contig_walk_thread_unsafe;
- rte_memseg_list_walk_thread_unsafe;
- rte_memseg_walk_thread_unsafe;
-
- # added in 18.11
- rte_delay_us_sleep;
- rte_dev_event_callback_process;
- rte_dev_hotplug_handle_disable;
- rte_dev_hotplug_handle_enable;
rte_malloc_heap_create;
rte_malloc_heap_destroy;
rte_malloc_heap_get_socket;
@@ -388,35 +284,35 @@ EXPERIMENTAL {
rte_malloc_heap_memory_detach;
rte_malloc_heap_memory_remove;
rte_malloc_heap_socket_is_external;
+ rte_mcfg_timer_lock;
+ rte_mcfg_timer_unlock;
+ rte_mem_alloc_validator_register;
+ rte_mem_alloc_validator_unregister;
+ rte_mem_check_dma_mask;
rte_mem_check_dma_mask_thread_unsafe;
+ rte_mem_event_callback_register;
+ rte_mem_event_callback_unregister;
+ rte_mem_iova2virt;
rte_mem_set_dma_mask;
+ rte_mem_virt2memseg;
+ rte_mem_virt2memseg_list;
+ rte_memseg_contig_walk;
+ rte_memseg_contig_walk_thread_unsafe;
rte_memseg_get_fd;
rte_memseg_get_fd_offset;
rte_memseg_get_fd_offset_thread_unsafe;
rte_memseg_get_fd_thread_unsafe;
+ rte_memseg_list_walk;
+ rte_memseg_list_walk_thread_unsafe;
+ rte_memseg_walk;
+ rte_memseg_walk_thread_unsafe;
+ rte_mp_action_register;
+ rte_mp_action_unregister;
+ rte_mp_reply;
+ rte_mp_request_async;
+ rte_mp_request_sync;
+ rte_mp_sendmsg;
rte_option_register;
-
- # added in 19.02
- rte_extmem_attach;
- rte_extmem_detach;
- rte_extmem_register;
- rte_extmem_unregister;
-
- # added in 19.05
- rte_dev_dma_map;
- rte_dev_dma_unmap;
- rte_fbarray_find_biggest_free;
- rte_fbarray_find_biggest_used;
- rte_fbarray_find_rev_biggest_free;
- rte_fbarray_find_rev_biggest_used;
- rte_intr_callback_unregister_pending;
- rte_realloc_socket;
-
- # added in 19.08
- rte_intr_ack;
- rte_lcore_cpuset;
- rte_lcore_to_cpu_id;
- rte_mcfg_timer_lock;
- rte_mcfg_timer_unlock;
rte_rand_max;
+ rte_realloc_socket;
};
diff --git a/lib/librte_efd/rte_efd_version.map b/lib/librte_efd/rte_efd_version.map
index ae60a6417..d061f8805 100644
--- a/lib/librte_efd/rte_efd_version.map
+++ b/lib/librte_efd/rte_efd_version.map
@@ -1,4 +1,4 @@
-DPDK_17.02 {
+DPDK_20.0 {
global:
rte_efd_create;
@@ -8,6 +8,6 @@ DPDK_17.02 {
rte_efd_lookup;
rte_efd_lookup_bulk;
rte_efd_update;
-
local: *;
};
+
diff --git a/lib/librte_ethdev/rte_ethdev_version.map b/lib/librte_ethdev/rte_ethdev_version.map
index 6df42a47b..9998ab6ca 100644
--- a/lib/librte_ethdev/rte_ethdev_version.map
+++ b/lib/librte_ethdev/rte_ethdev_version.map
@@ -1,35 +1,53 @@
-DPDK_2.2 {
+DPDK_20.0 {
global:
+ _rte_eth_dev_callback_process;
+ _rte_eth_dev_reset;
+ rte_eth_add_first_rx_callback;
rte_eth_add_rx_callback;
rte_eth_add_tx_callback;
rte_eth_allmulticast_disable;
rte_eth_allmulticast_enable;
rte_eth_allmulticast_get;
+ rte_eth_dev_adjust_nb_rx_tx_desc;
rte_eth_dev_allocate;
rte_eth_dev_allocated;
+ rte_eth_dev_attach_secondary;
rte_eth_dev_callback_register;
rte_eth_dev_callback_unregister;
rte_eth_dev_close;
rte_eth_dev_configure;
rte_eth_dev_count;
+ rte_eth_dev_count_avail;
+ rte_eth_dev_count_total;
rte_eth_dev_default_mac_addr_set;
+ rte_eth_dev_filter_ctrl;
rte_eth_dev_filter_supported;
rte_eth_dev_flow_ctrl_get;
rte_eth_dev_flow_ctrl_set;
+ rte_eth_dev_fw_version_get;
rte_eth_dev_get_dcb_info;
rte_eth_dev_get_eeprom;
rte_eth_dev_get_eeprom_length;
rte_eth_dev_get_mtu;
+ rte_eth_dev_get_name_by_port;
+ rte_eth_dev_get_port_by_name;
rte_eth_dev_get_reg_info;
+ rte_eth_dev_get_sec_ctx;
+ rte_eth_dev_get_supported_ptypes;
rte_eth_dev_get_vlan_offload;
- rte_eth_devices;
rte_eth_dev_info_get;
rte_eth_dev_is_valid_port;
+ rte_eth_dev_l2_tunnel_eth_type_conf;
+ rte_eth_dev_l2_tunnel_offload_set;
+ rte_eth_dev_logtype;
rte_eth_dev_mac_addr_add;
rte_eth_dev_mac_addr_remove;
+ rte_eth_dev_pool_ops_supported;
rte_eth_dev_priority_flow_ctrl_set;
+ rte_eth_dev_probing_finish;
rte_eth_dev_release_port;
+ rte_eth_dev_reset;
rte_eth_dev_rss_hash_conf_get;
rte_eth_dev_rss_hash_update;
rte_eth_dev_rss_reta_query;
@@ -38,6 +56,7 @@ DPDK_2.2 {
rte_eth_dev_rx_intr_ctl_q;
rte_eth_dev_rx_intr_disable;
rte_eth_dev_rx_intr_enable;
+ rte_eth_dev_rx_offload_name;
rte_eth_dev_rx_queue_start;
rte_eth_dev_rx_queue_stop;
rte_eth_dev_set_eeprom;
@@ -47,18 +66,28 @@ DPDK_2.2 {
rte_eth_dev_set_mtu;
rte_eth_dev_set_rx_queue_stats_mapping;
rte_eth_dev_set_tx_queue_stats_mapping;
+ rte_eth_dev_set_vlan_ether_type;
rte_eth_dev_set_vlan_offload;
rte_eth_dev_set_vlan_pvid;
rte_eth_dev_set_vlan_strip_on_queue;
rte_eth_dev_socket_id;
rte_eth_dev_start;
rte_eth_dev_stop;
+ rte_eth_dev_tx_offload_name;
rte_eth_dev_tx_queue_start;
rte_eth_dev_tx_queue_stop;
rte_eth_dev_uc_all_hash_table_set;
rte_eth_dev_uc_hash_table_set;
+ rte_eth_dev_udp_tunnel_port_add;
+ rte_eth_dev_udp_tunnel_port_delete;
rte_eth_dev_vlan_filter;
+ rte_eth_devices;
rte_eth_dma_zone_reserve;
+ rte_eth_find_next;
+ rte_eth_find_next_owned_by;
+ rte_eth_iterator_cleanup;
+ rte_eth_iterator_init;
+ rte_eth_iterator_next;
rte_eth_led_off;
rte_eth_led_on;
rte_eth_link;
@@ -75,6 +104,7 @@ DPDK_2.2 {
rte_eth_rx_queue_info_get;
rte_eth_rx_queue_setup;
rte_eth_set_queue_rate_limit;
+ rte_eth_speed_bitflag;
rte_eth_stats;
rte_eth_stats_get;
rte_eth_stats_reset;
@@ -85,66 +115,27 @@ DPDK_2.2 {
rte_eth_timesync_read_time;
rte_eth_timesync_read_tx_timestamp;
rte_eth_timesync_write_time;
- rte_eth_tx_queue_info_get;
- rte_eth_tx_queue_setup;
- rte_eth_xstats_get;
- rte_eth_xstats_reset;
-
- local: *;
-};
-
-DPDK_16.04 {
- global:
-
- rte_eth_dev_get_supported_ptypes;
- rte_eth_dev_l2_tunnel_eth_type_conf;
- rte_eth_dev_l2_tunnel_offload_set;
- rte_eth_dev_set_vlan_ether_type;
- rte_eth_dev_udp_tunnel_port_add;
- rte_eth_dev_udp_tunnel_port_delete;
- rte_eth_speed_bitflag;
rte_eth_tx_buffer_count_callback;
rte_eth_tx_buffer_drop_callback;
rte_eth_tx_buffer_init;
rte_eth_tx_buffer_set_err_callback;
-
-} DPDK_2.2;
-
-DPDK_16.07 {
- global:
-
- rte_eth_add_first_rx_callback;
- rte_eth_dev_get_name_by_port;
- rte_eth_dev_get_port_by_name;
- rte_eth_xstats_get_names;
-
-} DPDK_16.04;
-
-DPDK_17.02 {
- global:
-
- _rte_eth_dev_reset;
- rte_eth_dev_fw_version_get;
-
-} DPDK_16.07;
-
-DPDK_17.05 {
- global:
-
- rte_eth_dev_attach_secondary;
- rte_eth_find_next;
rte_eth_tx_done_cleanup;
+ rte_eth_tx_queue_info_get;
+ rte_eth_tx_queue_setup;
+ rte_eth_xstats_get;
rte_eth_xstats_get_by_id;
rte_eth_xstats_get_id_by_name;
+ rte_eth_xstats_get_names;
rte_eth_xstats_get_names_by_id;
-
-} DPDK_17.02;
-
-DPDK_17.08 {
- global:
-
- _rte_eth_dev_callback_process;
- rte_eth_dev_adjust_nb_rx_tx_desc;
+ rte_eth_xstats_reset;
+ rte_flow_copy;
+ rte_flow_create;
+ rte_flow_destroy;
+ rte_flow_error_set;
+ rte_flow_flush;
+ rte_flow_isolate;
+ rte_flow_query;
+ rte_flow_validate;
rte_tm_capabilities_get;
rte_tm_get_number_of_leaf_nodes;
rte_tm_hierarchy_commit;
@@ -175,71 +166,31 @@ DPDK_17.08 {
rte_tm_shared_wred_context_delete;
rte_tm_wred_profile_add;
rte_tm_wred_profile_delete;
-
-} DPDK_17.05;
-
-DPDK_17.11 {
- global:
-
- rte_eth_dev_get_sec_ctx;
- rte_eth_dev_pool_ops_supported;
- rte_eth_dev_reset;
-
-} DPDK_17.08;
-
-DPDK_18.02 {
- global:
-
- rte_eth_dev_filter_ctrl;
-
-} DPDK_17.11;
-
-DPDK_18.05 {
- global:
-
- rte_eth_dev_count_avail;
- rte_eth_dev_probing_finish;
- rte_eth_find_next_owned_by;
- rte_flow_copy;
- rte_flow_create;
- rte_flow_destroy;
- rte_flow_error_set;
- rte_flow_flush;
- rte_flow_isolate;
- rte_flow_query;
- rte_flow_validate;
-
-} DPDK_18.02;
-
-DPDK_18.08 {
- global:
-
- rte_eth_dev_logtype;
-
-} DPDK_18.05;
-
-DPDK_18.11 {
- global:
-
- rte_eth_dev_rx_offload_name;
- rte_eth_dev_tx_offload_name;
- rte_eth_iterator_cleanup;
- rte_eth_iterator_init;
- rte_eth_iterator_next;
-
-} DPDK_18.08;
-
-DPDK_19.05 {
- global:
-
- rte_eth_dev_count_total;
-
-} DPDK_18.11;
+ local: *;
+};
EXPERIMENTAL {
global:
- # added in 17.11
+ rte_eth_dev_create;
+ rte_eth_dev_destroy;
+ rte_eth_dev_get_module_eeprom;
+ rte_eth_dev_get_module_info;
+ rte_eth_dev_is_removed;
+ rte_eth_dev_owner_delete;
+ rte_eth_dev_owner_get;
+ rte_eth_dev_owner_new;
+ rte_eth_dev_owner_set;
+ rte_eth_dev_owner_unset;
+ rte_eth_dev_rx_intr_ctl_q_get_fd;
+ rte_eth_devargs_parse;
+ rte_eth_find_next_of;
+ rte_eth_find_next_sibling;
+ rte_eth_read_clock;
+ rte_eth_switch_domain_alloc;
+ rte_eth_switch_domain_free;
+ rte_flow_conv;
+ rte_flow_expand_rss;
rte_mtr_capabilities_get;
rte_mtr_create;
rte_mtr_destroy;
@@ -252,35 +203,4 @@ EXPERIMENTAL {
rte_mtr_policer_actions_update;
rte_mtr_stats_read;
rte_mtr_stats_update;
-
- # added in 18.02
- rte_eth_dev_is_removed;
- rte_eth_dev_owner_delete;
- rte_eth_dev_owner_get;
- rte_eth_dev_owner_new;
- rte_eth_dev_owner_set;
- rte_eth_dev_owner_unset;
-
- # added in 18.05
- rte_eth_dev_create;
- rte_eth_dev_destroy;
- rte_eth_dev_get_module_eeprom;
- rte_eth_dev_get_module_info;
- rte_eth_devargs_parse;
- rte_eth_switch_domain_alloc;
- rte_eth_switch_domain_free;
-
- # added in 18.08
- rte_flow_expand_rss;
-
- # added in 18.11
- rte_eth_dev_rx_intr_ctl_q_get_fd;
- rte_flow_conv;
-
- # added in 19.05
- rte_eth_find_next_of;
- rte_eth_find_next_sibling;
-
- # added in 19.08
- rte_eth_read_clock;
};
diff --git a/lib/librte_eventdev/rte_eventdev_version.map b/lib/librte_eventdev/rte_eventdev_version.map
index 76b3021d3..d354c1e49 100644
--- a/lib/librte_eventdev/rte_eventdev_version.map
+++ b/lib/librte_eventdev/rte_eventdev_version.map
@@ -1,61 +1,38 @@
-DPDK_17.05 {
+DPDK_20.0 {
global:
- rte_eventdevs;
-
+ rte_event_crypto_adapter_caps_get;
+ rte_event_crypto_adapter_create;
+ rte_event_crypto_adapter_create_ext;
+ rte_event_crypto_adapter_event_port_get;
+ rte_event_crypto_adapter_free;
+ rte_event_crypto_adapter_queue_pair_add;
+ rte_event_crypto_adapter_queue_pair_del;
+ rte_event_crypto_adapter_service_id_get;
+ rte_event_crypto_adapter_start;
+ rte_event_crypto_adapter_stats_get;
+ rte_event_crypto_adapter_stats_reset;
+ rte_event_crypto_adapter_stop;
+ rte_event_dequeue_timeout_ticks;
+ rte_event_dev_attr_get;
+ rte_event_dev_close;
+ rte_event_dev_configure;
rte_event_dev_count;
+ rte_event_dev_dump;
rte_event_dev_get_dev_id;
- rte_event_dev_socket_id;
rte_event_dev_info_get;
- rte_event_dev_configure;
+ rte_event_dev_selftest;
+ rte_event_dev_service_id_get;
+ rte_event_dev_socket_id;
rte_event_dev_start;
rte_event_dev_stop;
- rte_event_dev_close;
- rte_event_dev_dump;
+ rte_event_dev_stop_flush_callback_register;
rte_event_dev_xstats_by_name_get;
rte_event_dev_xstats_get;
rte_event_dev_xstats_names_get;
rte_event_dev_xstats_reset;
-
- rte_event_port_default_conf_get;
- rte_event_port_setup;
- rte_event_port_link;
- rte_event_port_unlink;
- rte_event_port_links_get;
-
- rte_event_queue_default_conf_get;
- rte_event_queue_setup;
-
- rte_event_dequeue_timeout_ticks;
-
- rte_event_pmd_allocate;
- rte_event_pmd_release;
- rte_event_pmd_vdev_init;
- rte_event_pmd_vdev_uninit;
- rte_event_pmd_pci_probe;
- rte_event_pmd_pci_remove;
-
- local: *;
-};
-
-DPDK_17.08 {
- global:
-
- rte_event_ring_create;
- rte_event_ring_free;
- rte_event_ring_init;
- rte_event_ring_lookup;
-} DPDK_17.05;
-
-DPDK_17.11 {
- global:
-
- rte_event_dev_attr_get;
- rte_event_dev_service_id_get;
- rte_event_port_attr_get;
- rte_event_queue_attr_get;
-
rte_event_eth_rx_adapter_caps_get;
+ rte_event_eth_rx_adapter_cb_register;
rte_event_eth_rx_adapter_create;
rte_event_eth_rx_adapter_create_ext;
rte_event_eth_rx_adapter_free;
@@ -63,38 +40,9 @@ DPDK_17.11 {
rte_event_eth_rx_adapter_queue_del;
rte_event_eth_rx_adapter_service_id_get;
rte_event_eth_rx_adapter_start;
+ rte_event_eth_rx_adapter_stats_get;
rte_event_eth_rx_adapter_stats_reset;
rte_event_eth_rx_adapter_stop;
-} DPDK_17.08;
-
-DPDK_18.02 {
- global:
-
- rte_event_dev_selftest;
-} DPDK_17.11;
-
-DPDK_18.05 {
- global:
-
- rte_event_dev_stop_flush_callback_register;
-} DPDK_18.02;
-
-DPDK_19.05 {
- global:
-
- rte_event_crypto_adapter_caps_get;
- rte_event_crypto_adapter_create;
- rte_event_crypto_adapter_create_ext;
- rte_event_crypto_adapter_event_port_get;
- rte_event_crypto_adapter_free;
- rte_event_crypto_adapter_queue_pair_add;
- rte_event_crypto_adapter_queue_pair_del;
- rte_event_crypto_adapter_service_id_get;
- rte_event_crypto_adapter_start;
- rte_event_crypto_adapter_stats_get;
- rte_event_crypto_adapter_stats_reset;
- rte_event_crypto_adapter_stop;
- rte_event_port_unlinks_in_progress;
rte_event_eth_tx_adapter_caps_get;
rte_event_eth_tx_adapter_create;
rte_event_eth_tx_adapter_create_ext;
@@ -107,6 +55,26 @@ DPDK_19.05 {
rte_event_eth_tx_adapter_stats_get;
rte_event_eth_tx_adapter_stats_reset;
rte_event_eth_tx_adapter_stop;
+ rte_event_pmd_allocate;
+ rte_event_pmd_pci_probe;
+ rte_event_pmd_pci_remove;
+ rte_event_pmd_release;
+ rte_event_pmd_vdev_init;
+ rte_event_pmd_vdev_uninit;
+ rte_event_port_attr_get;
+ rte_event_port_default_conf_get;
+ rte_event_port_link;
+ rte_event_port_links_get;
+ rte_event_port_setup;
+ rte_event_port_unlink;
+ rte_event_port_unlinks_in_progress;
+ rte_event_queue_attr_get;
+ rte_event_queue_default_conf_get;
+ rte_event_queue_setup;
+ rte_event_ring_create;
+ rte_event_ring_free;
+ rte_event_ring_init;
+ rte_event_ring_lookup;
rte_event_timer_adapter_caps_get;
rte_event_timer_adapter_create;
rte_event_timer_adapter_create_ext;
@@ -121,11 +89,7 @@ DPDK_19.05 {
rte_event_timer_arm_burst;
rte_event_timer_arm_tmo_tick_burst;
rte_event_timer_cancel_burst;
-} DPDK_18.05;
-
-DPDK_19.08 {
- global:
+ rte_eventdevs;
+ local: *;
+};
- rte_event_eth_rx_adapter_cb_register;
- rte_event_eth_rx_adapter_stats_get;
-} DPDK_19.05;
diff --git a/lib/librte_flow_classify/rte_flow_classify_version.map b/lib/librte_flow_classify/rte_flow_classify_version.map
index 49bc25c6a..9bbdc8b6e 100644
--- a/lib/librte_flow_classify/rte_flow_classify_version.map
+++ b/lib/librte_flow_classify/rte_flow_classify_version.map
@@ -8,6 +8,5 @@ EXPERIMENTAL {
rte_flow_classify_table_entry_add;
rte_flow_classify_table_entry_delete;
rte_flow_classify_validate;
-
local: *;
};
diff --git a/lib/librte_gro/rte_gro_version.map b/lib/librte_gro/rte_gro_version.map
index 1606b6dc7..a10aa0768 100644
--- a/lib/librte_gro/rte_gro_version.map
+++ b/lib/librte_gro/rte_gro_version.map
@@ -1,4 +1,4 @@
-DPDK_17.08 {
+DPDK_20.0 {
global:
rte_gro_ctx_create;
@@ -7,6 +7,6 @@ DPDK_17.08 {
rte_gro_reassemble;
rte_gro_reassemble_burst;
rte_gro_timeout_flush;
-
local: *;
};
+
diff --git a/lib/librte_gso/rte_gso_version.map b/lib/librte_gso/rte_gso_version.map
index e1fd453ed..56f2223f0 100644
--- a/lib/librte_gso/rte_gso_version.map
+++ b/lib/librte_gso/rte_gso_version.map
@@ -1,7 +1,7 @@
-DPDK_17.11 {
+DPDK_20.0 {
global:
rte_gso_segment;
-
local: *;
};
+
diff --git a/lib/librte_hash/rte_hash_version.map b/lib/librte_hash/rte_hash_version.map
index 734ae28b0..682188284 100644
--- a/lib/librte_hash/rte_hash_version.map
+++ b/lib/librte_hash/rte_hash_version.map
@@ -1,62 +1,35 @@
-DPDK_2.0 {
+DPDK_20.0 {
global:
rte_fbk_hash_create;
rte_fbk_hash_find_existing;
rte_fbk_hash_free;
rte_hash_add_key;
+ rte_hash_add_key_data;
rte_hash_add_key_with_hash;
+ rte_hash_add_key_with_hash_data;
+ rte_hash_count;
rte_hash_create;
rte_hash_del_key;
rte_hash_del_key_with_hash;
rte_hash_find_existing;
rte_hash_free;
+ rte_hash_get_key_with_position;
rte_hash_hash;
+ rte_hash_iterate;
rte_hash_lookup;
rte_hash_lookup_bulk;
- rte_hash_lookup_with_hash;
-
- local: *;
-};
-
-DPDK_2.1 {
- global:
-
- rte_hash_add_key_data;
- rte_hash_add_key_with_hash_data;
- rte_hash_iterate;
rte_hash_lookup_bulk_data;
rte_hash_lookup_data;
+ rte_hash_lookup_with_hash;
rte_hash_lookup_with_hash_data;
rte_hash_reset;
-
-} DPDK_2.0;
-
-DPDK_2.2 {
- global:
-
rte_hash_set_cmp_func;
-
-} DPDK_2.1;
-
-DPDK_16.07 {
- global:
-
- rte_hash_get_key_with_position;
-
-} DPDK_2.2;
-
-
-DPDK_18.08 {
- global:
-
- rte_hash_count;
-
-} DPDK_16.07;
+ local: *;
+};
EXPERIMENTAL {
global:
rte_hash_free_key_with_position;
-
};
diff --git a/lib/librte_ip_frag/rte_ip_frag_version.map b/lib/librte_ip_frag/rte_ip_frag_version.map
index a193007c6..fe3ac7b8a 100644
--- a/lib/librte_ip_frag/rte_ip_frag_version.map
+++ b/lib/librte_ip_frag/rte_ip_frag_version.map
@@ -1,24 +1,17 @@
-DPDK_2.0 {
+DPDK_20.0 {
global:
rte_ip_frag_free_death_row;
rte_ip_frag_table_create;
+ rte_ip_frag_table_destroy;
rte_ip_frag_table_statistics_dump;
rte_ipv4_frag_reassemble_packet;
rte_ipv4_fragment_packet;
rte_ipv6_frag_reassemble_packet;
rte_ipv6_fragment_packet;
-
local: *;
};
-DPDK_17.08 {
- global:
-
- rte_ip_frag_table_destroy;
-
-} DPDK_2.0;
-
EXPERIMENTAL {
global:
diff --git a/lib/librte_ipsec/rte_ipsec_version.map b/lib/librte_ipsec/rte_ipsec_version.map
index ee9f1961b..caf763ab9 100644
--- a/lib/librte_ipsec/rte_ipsec_version.map
+++ b/lib/librte_ipsec/rte_ipsec_version.map
@@ -10,6 +10,5 @@ EXPERIMENTAL {
rte_ipsec_sa_type;
rte_ipsec_ses_from_crypto;
rte_ipsec_session_prepare;
-
local: *;
};
diff --git a/lib/librte_jobstats/rte_jobstats_version.map b/lib/librte_jobstats/rte_jobstats_version.map
index f89441438..1ee93ffef 100644
--- a/lib/librte_jobstats/rte_jobstats_version.map
+++ b/lib/librte_jobstats/rte_jobstats_version.map
@@ -1,6 +1,7 @@
-DPDK_2.0 {
+DPDK_20.0 {
global:
+ rte_jobstats_abort;
rte_jobstats_context_finish;
rte_jobstats_context_init;
rte_jobstats_context_reset;
@@ -14,13 +15,6 @@ DPDK_2.0 {
rte_jobstats_set_target;
rte_jobstats_set_update_period_function;
rte_jobstats_start;
-
local: *;
};
-DPDK_16.04 {
- global:
-
- rte_jobstats_abort;
-
-} DPDK_2.0;
diff --git a/lib/librte_kni/rte_kni_version.map b/lib/librte_kni/rte_kni_version.map
index c877dc6aa..1de824375 100644
--- a/lib/librte_kni/rte_kni_version.map
+++ b/lib/librte_kni/rte_kni_version.map
@@ -1,4 +1,4 @@
-DPDK_2.0 {
+DPDK_20.0 {
global:
rte_kni_alloc;
@@ -12,7 +12,6 @@ DPDK_2.0 {
rte_kni_rx_burst;
rte_kni_tx_burst;
rte_kni_unregister_handlers;
-
local: *;
};
diff --git a/lib/librte_kvargs/rte_kvargs_version.map b/lib/librte_kvargs/rte_kvargs_version.map
index 8f4b4e3f8..ea1db117e 100644
--- a/lib/librte_kvargs/rte_kvargs_version.map
+++ b/lib/librte_kvargs/rte_kvargs_version.map
@@ -1,11 +1,10 @@
-DPDK_2.0 {
+DPDK_20.0 {
global:
rte_kvargs_count;
rte_kvargs_free;
rte_kvargs_parse;
rte_kvargs_process;
-
local: *;
};
@@ -14,5 +13,4 @@ EXPERIMENTAL {
rte_kvargs_parse_delim;
rte_kvargs_strcmp;
-
-} DPDK_2.0;
+};
diff --git a/lib/librte_latencystats/rte_latencystats_version.map b/lib/librte_latencystats/rte_latencystats_version.map
index ac8403e82..a2d6d6722 100644
--- a/lib/librte_latencystats/rte_latencystats_version.map
+++ b/lib/librte_latencystats/rte_latencystats_version.map
@@ -1,4 +1,4 @@
-DPDK_17.05 {
+DPDK_20.0 {
global:
rte_latencystats_get;
@@ -6,6 +6,6 @@ DPDK_17.05 {
rte_latencystats_init;
rte_latencystats_uninit;
rte_latencystats_update;
-
local: *;
};
+
diff --git a/lib/librte_lpm/rte_lpm.c b/lib/librte_lpm/rte_lpm.c
index 3a929a1b1..ce4681b79 100644
--- a/lib/librte_lpm/rte_lpm.c
+++ b/lib/librte_lpm/rte_lpm.c
@@ -113,7 +113,6 @@ rte_lpm_find_existing_v20(const char *name)
return l;
}
-VERSION_SYMBOL(rte_lpm_find_existing, _v20, 2.0);
struct rte_lpm *
rte_lpm_find_existing_v1604(const char *name)
@@ -139,7 +138,7 @@ rte_lpm_find_existing_v1604(const char *name)
return l;
}
-BIND_DEFAULT_SYMBOL(rte_lpm_find_existing, _v1604, 16.04);
+BIND_DEFAULT_SYMBOL(rte_lpm_find_existing, _v1604, 20.0);
MAP_STATIC_SYMBOL(struct rte_lpm *rte_lpm_find_existing(const char *name),
rte_lpm_find_existing_v1604);
@@ -217,7 +216,6 @@ rte_lpm_create_v20(const char *name, int socket_id, int max_rules,
return lpm;
}
-VERSION_SYMBOL(rte_lpm_create, _v20, 2.0);
struct rte_lpm *
rte_lpm_create_v1604(const char *name, int socket_id,
@@ -320,7 +318,7 @@ rte_lpm_create_v1604(const char *name, int socket_id,
return lpm;
}
-BIND_DEFAULT_SYMBOL(rte_lpm_create, _v1604, 16.04);
+BIND_DEFAULT_SYMBOL(rte_lpm_create, _v1604, 20.0);
MAP_STATIC_SYMBOL(
struct rte_lpm *rte_lpm_create(const char *name, int socket_id,
const struct rte_lpm_config *config), rte_lpm_create_v1604);
@@ -355,7 +353,6 @@ rte_lpm_free_v20(struct rte_lpm_v20 *lpm)
rte_free(lpm);
rte_free(te);
}
-VERSION_SYMBOL(rte_lpm_free, _v20, 2.0);
void
rte_lpm_free_v1604(struct rte_lpm *lpm)
@@ -386,7 +383,7 @@ rte_lpm_free_v1604(struct rte_lpm *lpm)
rte_free(lpm);
rte_free(te);
}
-BIND_DEFAULT_SYMBOL(rte_lpm_free, _v1604, 16.04);
+BIND_DEFAULT_SYMBOL(rte_lpm_free, _v1604, 20.0);
MAP_STATIC_SYMBOL(void rte_lpm_free(struct rte_lpm *lpm),
rte_lpm_free_v1604);
@@ -1215,7 +1212,6 @@ rte_lpm_add_v20(struct rte_lpm_v20 *lpm, uint32_t ip, uint8_t depth,
return 0;
}
-VERSION_SYMBOL(rte_lpm_add, _v20, 2.0);
int
rte_lpm_add_v1604(struct rte_lpm *lpm, uint32_t ip, uint8_t depth,
@@ -1256,7 +1252,7 @@ rte_lpm_add_v1604(struct rte_lpm *lpm, uint32_t ip, uint8_t depth,
return 0;
}
-BIND_DEFAULT_SYMBOL(rte_lpm_add, _v1604, 16.04);
+BIND_DEFAULT_SYMBOL(rte_lpm_add, _v1604, 20.0);
MAP_STATIC_SYMBOL(int rte_lpm_add(struct rte_lpm *lpm, uint32_t ip,
uint8_t depth, uint32_t next_hop), rte_lpm_add_v1604);
@@ -1288,7 +1284,6 @@ uint8_t *next_hop)
/* If rule is not found return 0. */
return 0;
}
-VERSION_SYMBOL(rte_lpm_is_rule_present, _v20, 2.0);
int
rte_lpm_is_rule_present_v1604(struct rte_lpm *lpm, uint32_t ip, uint8_t depth,
@@ -1315,7 +1310,7 @@ uint32_t *next_hop)
/* If rule is not found return 0. */
return 0;
}
-BIND_DEFAULT_SYMBOL(rte_lpm_is_rule_present, _v1604, 16.04);
+BIND_DEFAULT_SYMBOL(rte_lpm_is_rule_present, _v1604, 20.0);
MAP_STATIC_SYMBOL(int rte_lpm_is_rule_present(struct rte_lpm *lpm, uint32_t ip,
uint8_t depth, uint32_t *next_hop), rte_lpm_is_rule_present_v1604);
@@ -1895,7 +1890,6 @@ rte_lpm_delete_v20(struct rte_lpm_v20 *lpm, uint32_t ip, uint8_t depth)
sub_rule_depth);
}
}
-VERSION_SYMBOL(rte_lpm_delete, _v20, 2.0);
int
rte_lpm_delete_v1604(struct rte_lpm *lpm, uint32_t ip, uint8_t depth)
@@ -1949,7 +1943,7 @@ rte_lpm_delete_v1604(struct rte_lpm *lpm, uint32_t ip, uint8_t depth)
sub_rule_depth);
}
}
-BIND_DEFAULT_SYMBOL(rte_lpm_delete, _v1604, 16.04);
+BIND_DEFAULT_SYMBOL(rte_lpm_delete, _v1604, 20.0);
MAP_STATIC_SYMBOL(int rte_lpm_delete(struct rte_lpm *lpm, uint32_t ip,
uint8_t depth), rte_lpm_delete_v1604);
@@ -1971,7 +1965,6 @@ rte_lpm_delete_all_v20(struct rte_lpm_v20 *lpm)
/* Delete all rules form the rules table. */
memset(lpm->rules_tbl, 0, sizeof(lpm->rules_tbl[0]) * lpm->max_rules);
}
-VERSION_SYMBOL(rte_lpm_delete_all, _v20, 2.0);
void
rte_lpm_delete_all_v1604(struct rte_lpm *lpm)
@@ -1989,6 +1982,6 @@ rte_lpm_delete_all_v1604(struct rte_lpm *lpm)
/* Delete all rules form the rules table. */
memset(lpm->rules_tbl, 0, sizeof(lpm->rules_tbl[0]) * lpm->max_rules);
}
-BIND_DEFAULT_SYMBOL(rte_lpm_delete_all, _v1604, 16.04);
+BIND_DEFAULT_SYMBOL(rte_lpm_delete_all, _v1604, 20.0);
MAP_STATIC_SYMBOL(void rte_lpm_delete_all(struct rte_lpm *lpm),
rte_lpm_delete_all_v1604);
diff --git a/lib/librte_lpm/rte_lpm6.c b/lib/librte_lpm/rte_lpm6.c
index 9b8aeb972..44828f72c 100644
--- a/lib/librte_lpm/rte_lpm6.c
+++ b/lib/librte_lpm/rte_lpm6.c
@@ -817,7 +817,6 @@ rte_lpm6_add_v20(struct rte_lpm6 *lpm, uint8_t *ip, uint8_t depth,
{
return rte_lpm6_add_v1705(lpm, ip, depth, next_hop);
}
-VERSION_SYMBOL(rte_lpm6_add, _v20, 2.0);
/*
@@ -913,7 +912,7 @@ rte_lpm6_add_v1705(struct rte_lpm6 *lpm, uint8_t *ip, uint8_t depth,
return status;
}
-BIND_DEFAULT_SYMBOL(rte_lpm6_add, _v1705, 17.05);
+BIND_DEFAULT_SYMBOL(rte_lpm6_add, _v1705, 20.0);
MAP_STATIC_SYMBOL(int rte_lpm6_add(struct rte_lpm6 *lpm, uint8_t *ip,
uint8_t depth, uint32_t next_hop),
rte_lpm6_add_v1705);
@@ -970,7 +969,6 @@ rte_lpm6_lookup_v20(const struct rte_lpm6 *lpm, uint8_t *ip, uint8_t *next_hop)
return status;
}
-VERSION_SYMBOL(rte_lpm6_lookup, _v20, 2.0);
int
rte_lpm6_lookup_v1705(const struct rte_lpm6 *lpm, uint8_t *ip,
@@ -1000,7 +998,7 @@ rte_lpm6_lookup_v1705(const struct rte_lpm6 *lpm, uint8_t *ip,
return status;
}
-BIND_DEFAULT_SYMBOL(rte_lpm6_lookup, _v1705, 17.05);
+BIND_DEFAULT_SYMBOL(rte_lpm6_lookup, _v1705, 20.0);
MAP_STATIC_SYMBOL(int rte_lpm6_lookup(const struct rte_lpm6 *lpm, uint8_t *ip,
uint32_t *next_hop), rte_lpm6_lookup_v1705);
@@ -1046,7 +1044,6 @@ rte_lpm6_lookup_bulk_func_v20(const struct rte_lpm6 *lpm,
return 0;
}
-VERSION_SYMBOL(rte_lpm6_lookup_bulk_func, _v20, 2.0);
int
rte_lpm6_lookup_bulk_func_v1705(const struct rte_lpm6 *lpm,
@@ -1089,7 +1086,7 @@ rte_lpm6_lookup_bulk_func_v1705(const struct rte_lpm6 *lpm,
return 0;
}
-BIND_DEFAULT_SYMBOL(rte_lpm6_lookup_bulk_func, _v1705, 17.05);
+BIND_DEFAULT_SYMBOL(rte_lpm6_lookup_bulk_func, _v1705, 20.0);
MAP_STATIC_SYMBOL(int rte_lpm6_lookup_bulk_func(const struct rte_lpm6 *lpm,
uint8_t ips[][RTE_LPM6_IPV6_ADDR_SIZE],
int32_t *next_hops, unsigned int n),
@@ -1116,7 +1113,6 @@ rte_lpm6_is_rule_present_v20(struct rte_lpm6 *lpm, uint8_t *ip, uint8_t depth,
return status;
}
-VERSION_SYMBOL(rte_lpm6_is_rule_present, _v20, 2.0);
int
rte_lpm6_is_rule_present_v1705(struct rte_lpm6 *lpm, uint8_t *ip, uint8_t depth,
@@ -1135,7 +1131,7 @@ rte_lpm6_is_rule_present_v1705(struct rte_lpm6 *lpm, uint8_t *ip, uint8_t depth,
return rule_find(lpm, masked_ip, depth, next_hop);
}
-BIND_DEFAULT_SYMBOL(rte_lpm6_is_rule_present, _v1705, 17.05);
+BIND_DEFAULT_SYMBOL(rte_lpm6_is_rule_present, _v1705, 20.0);
MAP_STATIC_SYMBOL(int rte_lpm6_is_rule_present(struct rte_lpm6 *lpm,
uint8_t *ip, uint8_t depth, uint32_t *next_hop),
rte_lpm6_is_rule_present_v1705);
diff --git a/lib/librte_lpm/rte_lpm_version.map b/lib/librte_lpm/rte_lpm_version.map
index 90beac853..e57e6e7c3 100644
--- a/lib/librte_lpm/rte_lpm_version.map
+++ b/lib/librte_lpm/rte_lpm_version.map
@@ -1,13 +1,6 @@
-DPDK_2.0 {
+DPDK_20.0 {
global:
- rte_lpm_add;
- rte_lpm_create;
- rte_lpm_delete;
- rte_lpm_delete_all;
- rte_lpm_find_existing;
- rte_lpm_free;
- rte_lpm_is_rule_present;
rte_lpm6_add;
rte_lpm6_create;
rte_lpm6_delete;
@@ -18,29 +11,13 @@ DPDK_2.0 {
rte_lpm6_is_rule_present;
rte_lpm6_lookup;
rte_lpm6_lookup_bulk_func;
-
- local: *;
-};
-
-DPDK_16.04 {
- global:
-
rte_lpm_add;
- rte_lpm_find_existing;
rte_lpm_create;
- rte_lpm_free;
- rte_lpm_is_rule_present;
rte_lpm_delete;
rte_lpm_delete_all;
+ rte_lpm_find_existing;
+ rte_lpm_free;
+ rte_lpm_is_rule_present;
+ local: *;
+};
-} DPDK_2.0;
-
-DPDK_17.05 {
- global:
-
- rte_lpm6_add;
- rte_lpm6_is_rule_present;
- rte_lpm6_lookup;
- rte_lpm6_lookup_bulk_func;
-
-} DPDK_16.04;
diff --git a/lib/librte_mbuf/rte_mbuf_version.map b/lib/librte_mbuf/rte_mbuf_version.map
index 2662a37bf..998d3a141 100644
--- a/lib/librte_mbuf/rte_mbuf_version.map
+++ b/lib/librte_mbuf/rte_mbuf_version.map
@@ -1,24 +1,4 @@
-DPDK_2.0 {
- global:
-
- rte_get_rx_ol_flag_name;
- rte_get_tx_ol_flag_name;
- rte_mbuf_sanity_check;
- rte_pktmbuf_dump;
- rte_pktmbuf_init;
- rte_pktmbuf_pool_init;
-
- local: *;
-};
-
-DPDK_2.1 {
- global:
-
- rte_pktmbuf_pool_create;
-
-} DPDK_2.0;
-
-DPDK_16.11 {
+DPDK_20.0 {
global:
__rte_pktmbuf_read;
@@ -31,23 +11,25 @@ DPDK_16.11 {
rte_get_ptype_name;
rte_get_ptype_tunnel_name;
rte_get_rx_ol_flag_list;
+ rte_get_rx_ol_flag_name;
rte_get_tx_ol_flag_list;
-
-} DPDK_2.1;
-
-DPDK_18.08 {
- global:
-
+ rte_get_tx_ol_flag_name;
rte_mbuf_best_mempool_ops;
rte_mbuf_platform_mempool_ops;
+ rte_mbuf_sanity_check;
rte_mbuf_set_platform_mempool_ops;
rte_mbuf_set_user_mempool_ops;
rte_mbuf_user_mempool_ops;
+ rte_pktmbuf_dump;
+ rte_pktmbuf_init;
+ rte_pktmbuf_pool_create;
rte_pktmbuf_pool_create_by_ops;
-} DPDK_16.11;
+ rte_pktmbuf_pool_init;
+ local: *;
+};
EXPERIMENTAL {
global:
rte_mbuf_check;
-} DPDK_18.08;
+};
diff --git a/lib/librte_member/rte_member_version.map b/lib/librte_member/rte_member_version.map
index 019e4cd96..e21bf8510 100644
--- a/lib/librte_member/rte_member_version.map
+++ b/lib/librte_member/rte_member_version.map
@@ -1,4 +1,4 @@
-DPDK_17.11 {
+DPDK_20.0 {
global:
rte_member_add;
@@ -11,6 +11,6 @@ DPDK_17.11 {
rte_member_lookup_multi;
rte_member_lookup_multi_bulk;
rte_member_reset;
-
local: *;
};
+
diff --git a/lib/librte_mempool/rte_mempool_version.map b/lib/librte_mempool/rte_mempool_version.map
index 17cbca460..27ab9183d 100644
--- a/lib/librte_mempool/rte_mempool_version.map
+++ b/lib/librte_mempool/rte_mempool_version.map
@@ -1,57 +1,38 @@
-DPDK_2.0 {
+DPDK_20.0 {
global:
rte_mempool_audit;
- rte_mempool_calc_obj_size;
- rte_mempool_create;
- rte_mempool_dump;
- rte_mempool_list_dump;
- rte_mempool_lookup;
- rte_mempool_walk;
-
- local: *;
-};
-
-DPDK_16.07 {
- global:
-
rte_mempool_avail_count;
rte_mempool_cache_create;
rte_mempool_cache_flush;
rte_mempool_cache_free;
+ rte_mempool_calc_obj_size;
rte_mempool_check_cookies;
+ rte_mempool_contig_blocks_check_cookies;
+ rte_mempool_create;
rte_mempool_create_empty;
rte_mempool_default_cache;
+ rte_mempool_dump;
rte_mempool_free;
rte_mempool_generic_get;
rte_mempool_generic_put;
rte_mempool_in_use_count;
+ rte_mempool_list_dump;
+ rte_mempool_lookup;
rte_mempool_mem_iter;
rte_mempool_obj_iter;
+ rte_mempool_op_calc_mem_size_default;
+ rte_mempool_op_populate_default;
rte_mempool_ops_table;
rte_mempool_populate_anon;
rte_mempool_populate_default;
+ rte_mempool_populate_iova;
rte_mempool_populate_virt;
rte_mempool_register_ops;
rte_mempool_set_ops_byname;
-
-} DPDK_2.0;
-
-DPDK_17.11 {
- global:
-
- rte_mempool_populate_iova;
-
-} DPDK_16.07;
-
-DPDK_18.05 {
- global:
-
- rte_mempool_contig_blocks_check_cookies;
- rte_mempool_op_calc_mem_size_default;
- rte_mempool_op_populate_default;
-
-} DPDK_17.11;
+ rte_mempool_walk;
+ local: *;
+};
EXPERIMENTAL {
global:
diff --git a/lib/librte_meter/rte_meter_version.map b/lib/librte_meter/rte_meter_version.map
index 4b460d580..633326a62 100644
--- a/lib/librte_meter/rte_meter_version.map
+++ b/lib/librte_meter/rte_meter_version.map
@@ -1,21 +1,15 @@
-DPDK_2.0 {
+DPDK_20.0 {
global:
rte_meter_srtcm_color_aware_check;
rte_meter_srtcm_color_blind_check;
rte_meter_srtcm_config;
+ rte_meter_srtcm_profile_config;
rte_meter_trtcm_color_aware_check;
rte_meter_trtcm_color_blind_check;
rte_meter_trtcm_config;
-
- local: *;
-};
-
-DPDK_18.08 {
- global:
-
- rte_meter_srtcm_profile_config;
rte_meter_trtcm_profile_config;
+ local: *;
};
EXPERIMENTAL {
diff --git a/lib/librte_metrics/rte_metrics_version.map b/lib/librte_metrics/rte_metrics_version.map
index 6ac99a44a..7c77cc14d 100644
--- a/lib/librte_metrics/rte_metrics_version.map
+++ b/lib/librte_metrics/rte_metrics_version.map
@@ -1,4 +1,4 @@
-DPDK_17.05 {
+DPDK_20.0 {
global:
rte_metrics_get_names;
@@ -8,7 +8,6 @@ DPDK_17.05 {
rte_metrics_reg_names;
rte_metrics_update_value;
rte_metrics_update_values;
-
local: *;
};
diff --git a/lib/librte_net/rte_net_version.map b/lib/librte_net/rte_net_version.map
index fffc4a372..f5cccd646 100644
--- a/lib/librte_net/rte_net_version.map
+++ b/lib/librte_net/rte_net_version.map
@@ -1,30 +1,18 @@
-DPDK_16.11 {
- global:
- rte_net_get_ptype;
-
- local: *;
-};
-
-DPDK_17.05 {
- global:
-
- rte_net_crc_calc;
- rte_net_crc_set_alg;
-
-} DPDK_16.11;
-
-DPDK_19.08 {
+DPDK_20.0 {
global:
rte_eth_random_addr;
rte_ether_format_addr;
-
-} DPDK_17.05;
+ rte_net_crc_calc;
+ rte_net_crc_set_alg;
+ rte_net_get_ptype;
+ local: *;
+};
EXPERIMENTAL {
global:
+ rte_ether_unformat_addr;
rte_net_make_rarp_packet;
rte_net_skip_ip6_ext;
- rte_ether_unformat_addr;
};
diff --git a/lib/librte_pci/rte_pci_version.map b/lib/librte_pci/rte_pci_version.map
index c0280277b..6fc4a9d7e 100644
--- a/lib/librte_pci/rte_pci_version.map
+++ b/lib/librte_pci/rte_pci_version.map
@@ -1,4 +1,4 @@
-DPDK_17.11 {
+DPDK_20.0 {
global:
eal_parse_pci_BDF;
@@ -9,6 +9,6 @@ DPDK_17.11 {
rte_pci_addr_cmp;
rte_pci_addr_parse;
rte_pci_device_name;
-
local: *;
};
+
diff --git a/lib/librte_pdump/rte_pdump_version.map b/lib/librte_pdump/rte_pdump_version.map
index 3e744f301..aaa8db524 100644
--- a/lib/librte_pdump/rte_pdump_version.map
+++ b/lib/librte_pdump/rte_pdump_version.map
@@ -1,4 +1,4 @@
-DPDK_16.07 {
+DPDK_20.0 {
global:
rte_pdump_disable;
@@ -7,6 +7,6 @@ DPDK_16.07 {
rte_pdump_enable_by_deviceid;
rte_pdump_init;
rte_pdump_uninit;
-
local: *;
};
+
diff --git a/lib/librte_pipeline/rte_pipeline_version.map b/lib/librte_pipeline/rte_pipeline_version.map
index 420f065d6..b5fb09853 100644
--- a/lib/librte_pipeline/rte_pipeline_version.map
+++ b/lib/librte_pipeline/rte_pipeline_version.map
@@ -1,6 +1,8 @@
-DPDK_2.0 {
+DPDK_20.0 {
global:
+ rte_pipeline_ah_packet_drop;
+ rte_pipeline_ah_packet_hijack;
rte_pipeline_check;
rte_pipeline_create;
rte_pipeline_flush;
@@ -9,43 +11,22 @@ DPDK_2.0 {
rte_pipeline_port_in_create;
rte_pipeline_port_in_disable;
rte_pipeline_port_in_enable;
+ rte_pipeline_port_in_stats_read;
rte_pipeline_port_out_create;
rte_pipeline_port_out_packet_insert;
+ rte_pipeline_port_out_stats_read;
rte_pipeline_run;
rte_pipeline_table_create;
rte_pipeline_table_default_entry_add;
rte_pipeline_table_default_entry_delete;
rte_pipeline_table_entry_add;
+ rte_pipeline_table_entry_add_bulk;
rte_pipeline_table_entry_delete;
-
+ rte_pipeline_table_entry_delete_bulk;
+ rte_pipeline_table_stats_read;
local: *;
};
-DPDK_2.1 {
- global:
-
- rte_pipeline_port_in_stats_read;
- rte_pipeline_port_out_stats_read;
- rte_pipeline_table_stats_read;
-
-} DPDK_2.0;
-
-DPDK_2.2 {
- global:
-
- rte_pipeline_table_entry_add_bulk;
- rte_pipeline_table_entry_delete_bulk;
-
-} DPDK_2.1;
-
-DPDK_16.04 {
- global:
-
- rte_pipeline_ah_packet_hijack;
- rte_pipeline_ah_packet_drop;
-
-} DPDK_2.2;
-
EXPERIMENTAL {
global:
@@ -59,6 +40,7 @@ EXPERIMENTAL {
rte_port_in_action_profile_freeze;
rte_table_action_apply;
rte_table_action_create;
+ rte_table_action_crypto_sym_session_get;
rte_table_action_dscp_table_update;
rte_table_action_free;
rte_table_action_meter_profile_add;
@@ -68,9 +50,8 @@ EXPERIMENTAL {
rte_table_action_profile_create;
rte_table_action_profile_free;
rte_table_action_profile_freeze;
- rte_table_action_table_params_get;
rte_table_action_stats_read;
+ rte_table_action_table_params_get;
rte_table_action_time_read;
rte_table_action_ttl_read;
- rte_table_action_crypto_sym_session_get;
};
diff --git a/lib/librte_port/rte_port_version.map b/lib/librte_port/rte_port_version.map
index 609bcec3f..2037c3ed0 100644
--- a/lib/librte_port/rte_port_version.map
+++ b/lib/librte_port/rte_port_version.map
@@ -1,62 +1,32 @@
-DPDK_2.0 {
+DPDK_20.0 {
global:
rte_port_ethdev_reader_ops;
+ rte_port_ethdev_writer_nodrop_ops;
rte_port_ethdev_writer_ops;
+ rte_port_fd_reader_ops;
+ rte_port_fd_writer_nodrop_ops;
+ rte_port_fd_writer_ops;
+ rte_port_kni_reader_ops;
+ rte_port_kni_writer_nodrop_ops;
+ rte_port_kni_writer_ops;
+ rte_port_ring_multi_reader_ops;
+ rte_port_ring_multi_writer_nodrop_ops;
+ rte_port_ring_multi_writer_ops;
rte_port_ring_reader_ipv4_frag_ops;
+ rte_port_ring_reader_ipv6_frag_ops;
rte_port_ring_reader_ops;
rte_port_ring_writer_ipv4_ras_ops;
+ rte_port_ring_writer_ipv6_ras_ops;
+ rte_port_ring_writer_nodrop_ops;
rte_port_ring_writer_ops;
rte_port_sched_reader_ops;
rte_port_sched_writer_ops;
rte_port_sink_ops;
rte_port_source_ops;
-
- local: *;
-};
-
-DPDK_2.1 {
- global:
-
- rte_port_ethdev_writer_nodrop_ops;
- rte_port_ring_reader_ipv6_frag_ops;
- rte_port_ring_writer_ipv6_ras_ops;
- rte_port_ring_writer_nodrop_ops;
-
-} DPDK_2.0;
-
-DPDK_2.2 {
- global:
-
- rte_port_ring_multi_reader_ops;
- rte_port_ring_multi_writer_ops;
- rte_port_ring_multi_writer_nodrop_ops;
-
-} DPDK_2.1;
-
-DPDK_16.07 {
- global:
-
- rte_port_kni_reader_ops;
- rte_port_kni_writer_ops;
- rte_port_kni_writer_nodrop_ops;
-
-} DPDK_2.2;
-
-DPDK_16.11 {
- global:
-
- rte_port_fd_reader_ops;
- rte_port_fd_writer_ops;
- rte_port_fd_writer_nodrop_ops;
-
-} DPDK_16.07;
-
-DPDK_18.11 {
- global:
-
rte_port_sym_crypto_reader_ops;
- rte_port_sym_crypto_writer_ops;
rte_port_sym_crypto_writer_nodrop_ops;
+ rte_port_sym_crypto_writer_ops;
+ local: *;
+};
-} DPDK_16.11;
diff --git a/lib/librte_power/rte_power_version.map b/lib/librte_power/rte_power_version.map
index 042917360..18e77802b 100644
--- a/lib/librte_power/rte_power_version.map
+++ b/lib/librte_power/rte_power_version.map
@@ -1,39 +1,26 @@
-DPDK_2.0 {
+DPDK_20.0 {
global:
rte_power_exit;
+ rte_power_freq_disable_turbo;
rte_power_freq_down;
+ rte_power_freq_enable_turbo;
rte_power_freq_max;
rte_power_freq_min;
rte_power_freq_up;
rte_power_freqs;
+ rte_power_get_capabilities;
rte_power_get_env;
rte_power_get_freq;
+ rte_power_guest_channel_send_msg;
rte_power_init;
rte_power_set_env;
rte_power_set_freq;
+ rte_power_turbo_status;
rte_power_unset_env;
-
local: *;
};
-DPDK_17.11 {
- global:
-
- rte_power_guest_channel_send_msg;
- rte_power_freq_disable_turbo;
- rte_power_freq_enable_turbo;
- rte_power_turbo_status;
-
-} DPDK_2.0;
-
-DPDK_18.08 {
- global:
-
- rte_power_get_capabilities;
-
-} DPDK_17.11;
-
EXPERIMENTAL {
global:
diff --git a/lib/librte_rawdev/rte_rawdev_version.map b/lib/librte_rawdev/rte_rawdev_version.map
index b61dbff11..fe9de5dfe 100644
--- a/lib/librte_rawdev/rte_rawdev_version.map
+++ b/lib/librte_rawdev/rte_rawdev_version.map
@@ -1,4 +1,4 @@
-DPDK_18.08 {
+DPDK_20.0 {
global:
rte_rawdev_close;
@@ -17,8 +17,8 @@ DPDK_18.08 {
rte_rawdev_pmd_release;
rte_rawdev_queue_conf_get;
rte_rawdev_queue_count;
- rte_rawdev_queue_setup;
rte_rawdev_queue_release;
+ rte_rawdev_queue_setup;
rte_rawdev_reset;
rte_rawdev_selftest;
rte_rawdev_set_attr;
@@ -30,6 +30,6 @@ DPDK_18.08 {
rte_rawdev_xstats_names_get;
rte_rawdev_xstats_reset;
rte_rawdevs;
-
local: *;
};
+
diff --git a/lib/librte_rcu/rte_rcu_version.map b/lib/librte_rcu/rte_rcu_version.map
index f8b9ef2ab..cd5aedaa9 100644
--- a/lib/librte_rcu/rte_rcu_version.map
+++ b/lib/librte_rcu/rte_rcu_version.map
@@ -8,6 +8,5 @@ EXPERIMENTAL {
rte_rcu_qsbr_synchronize;
rte_rcu_qsbr_thread_register;
rte_rcu_qsbr_thread_unregister;
-
local: *;
};
diff --git a/lib/librte_reorder/rte_reorder_version.map b/lib/librte_reorder/rte_reorder_version.map
index 0a8a54de8..d7a2afd18 100644
--- a/lib/librte_reorder/rte_reorder_version.map
+++ b/lib/librte_reorder/rte_reorder_version.map
@@ -1,13 +1,13 @@
-DPDK_2.0 {
+DPDK_20.0 {
global:
rte_reorder_create;
- rte_reorder_init;
+ rte_reorder_drain;
rte_reorder_find_existing;
- rte_reorder_reset;
rte_reorder_free;
+ rte_reorder_init;
rte_reorder_insert;
- rte_reorder_drain;
-
+ rte_reorder_reset;
local: *;
};
+
diff --git a/lib/librte_ring/rte_ring_version.map b/lib/librte_ring/rte_ring_version.map
index 510c1386e..38a095619 100644
--- a/lib/librte_ring/rte_ring_version.map
+++ b/lib/librte_ring/rte_ring_version.map
@@ -1,26 +1,18 @@
-DPDK_2.0 {
+DPDK_20.0 {
global:
rte_ring_create;
rte_ring_dump;
+ rte_ring_free;
rte_ring_get_memsize;
rte_ring_init;
rte_ring_list_dump;
rte_ring_lookup;
-
local: *;
};
-DPDK_2.2 {
- global:
-
- rte_ring_free;
-
-} DPDK_2.0;
-
EXPERIMENTAL {
global:
rte_ring_reset;
-
};
diff --git a/lib/librte_sched/rte_sched_version.map b/lib/librte_sched/rte_sched_version.map
index 729588794..7f3fe2d9a 100644
--- a/lib/librte_sched/rte_sched_version.map
+++ b/lib/librte_sched/rte_sched_version.map
@@ -1,4 +1,4 @@
-DPDK_2.0 {
+DPDK_20.0 {
global:
rte_approx;
@@ -14,22 +14,15 @@ DPDK_2.0 {
rte_sched_port_enqueue;
rte_sched_port_free;
rte_sched_port_get_memory_footprint;
+ rte_sched_port_pkt_read_color;
+ rte_sched_port_pkt_read_tree_path;
+ rte_sched_port_pkt_write;
rte_sched_queue_read_stats;
rte_sched_subport_config;
rte_sched_subport_read_stats;
-
local: *;
};
-DPDK_2.1 {
- global:
-
- rte_sched_port_pkt_write;
- rte_sched_port_pkt_read_tree_path;
- rte_sched_port_pkt_read_color;
-
-} DPDK_2.0;
-
EXPERIMENTAL {
global:
diff --git a/lib/librte_security/rte_security_version.map b/lib/librte_security/rte_security_version.map
index 53267bf3c..5e21cbfe2 100644
--- a/lib/librte_security/rte_security_version.map
+++ b/lib/librte_security/rte_security_version.map
@@ -1,4 +1,4 @@
-DPDK_18.11 {
+DPDK_20.0 {
global:
rte_security_attach_session;
@@ -8,7 +8,6 @@ DPDK_18.11 {
rte_security_session_destroy;
rte_security_session_get_size;
rte_security_set_pkt_metadata;
-
local: *;
};
diff --git a/lib/librte_stack/rte_stack_version.map b/lib/librte_stack/rte_stack_version.map
index 6662679c3..b646cf7d2 100644
--- a/lib/librte_stack/rte_stack_version.map
+++ b/lib/librte_stack/rte_stack_version.map
@@ -4,6 +4,5 @@ EXPERIMENTAL {
rte_stack_create;
rte_stack_free;
rte_stack_lookup;
-
local: *;
};
diff --git a/lib/librte_table/rte_table_version.map b/lib/librte_table/rte_table_version.map
index 6237252be..5f2cdd13d 100644
--- a/lib/librte_table/rte_table_version.map
+++ b/lib/librte_table/rte_table_version.map
@@ -1,4 +1,4 @@
-DPDK_17.11 {
+DPDK_20.0 {
global:
rte_table_acl_ops;
@@ -15,6 +15,6 @@ DPDK_17.11 {
rte_table_lpm_ipv6_ops;
rte_table_lpm_ops;
rte_table_stub_ops;
-
local: *;
};
+
diff --git a/lib/librte_telemetry/rte_telemetry_version.map b/lib/librte_telemetry/rte_telemetry_version.map
index fa62d7718..cbbf41075 100644
--- a/lib/librte_telemetry/rte_telemetry_version.map
+++ b/lib/librte_telemetry/rte_telemetry_version.map
@@ -5,6 +5,5 @@ EXPERIMENTAL {
rte_telemetry_init;
rte_telemetry_parse;
rte_telemetry_selftest;
-
local: *;
};
diff --git a/lib/librte_timer/rte_timer.c b/lib/librte_timer/rte_timer.c
index bdcf05d06..34c864b60 100644
--- a/lib/librte_timer/rte_timer.c
+++ b/lib/librte_timer/rte_timer.c
@@ -145,7 +145,6 @@ rte_timer_subsystem_init_v20(void)
priv_timer[lcore_id].prev_lcore = lcore_id;
}
}
-VERSION_SYMBOL(rte_timer_subsystem_init, _v20, 2.0);
/* Init the timer library. Allocate an array of timer data structs in shared
* memory, and allocate the zeroth entry for use with original timer
@@ -211,7 +210,7 @@ rte_timer_subsystem_init_v1905(void)
}
MAP_STATIC_SYMBOL(int rte_timer_subsystem_init(void),
rte_timer_subsystem_init_v1905);
-BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05);
+BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 20.0);
void
rte_timer_subsystem_finalize(void)
@@ -572,7 +571,6 @@ rte_timer_reset_v20(struct rte_timer *tim, uint64_t ticks,
return __rte_timer_reset(tim, cur_time + ticks, period, tim_lcore,
fct, arg, 0, &default_timer_data);
}
-VERSION_SYMBOL(rte_timer_reset, _v20, 2.0);
int
rte_timer_reset_v1905(struct rte_timer *tim, uint64_t ticks,
@@ -587,7 +585,7 @@ MAP_STATIC_SYMBOL(int rte_timer_reset(struct rte_timer *tim, uint64_t ticks,
unsigned int tim_lcore,
rte_timer_cb_t fct, void *arg),
rte_timer_reset_v1905);
-BIND_DEFAULT_SYMBOL(rte_timer_reset, _v1905, 19.05);
+BIND_DEFAULT_SYMBOL(rte_timer_reset, _v1905, 20.0);
int
rte_timer_alt_reset(uint32_t timer_data_id, struct rte_timer *tim,
@@ -662,7 +660,6 @@ rte_timer_stop_v20(struct rte_timer *tim)
{
return __rte_timer_stop(tim, 0, &default_timer_data);
}
-VERSION_SYMBOL(rte_timer_stop, _v20, 2.0);
int
rte_timer_stop_v1905(struct rte_timer *tim)
@@ -671,7 +668,7 @@ rte_timer_stop_v1905(struct rte_timer *tim)
}
MAP_STATIC_SYMBOL(int rte_timer_stop(struct rte_timer *tim),
rte_timer_stop_v1905);
-BIND_DEFAULT_SYMBOL(rte_timer_stop, _v1905, 19.05);
+BIND_DEFAULT_SYMBOL(rte_timer_stop, _v1905, 20.0);
int
rte_timer_alt_stop(uint32_t timer_data_id, struct rte_timer *tim)
@@ -822,7 +819,6 @@ rte_timer_manage_v20(void)
{
__rte_timer_manage(&default_timer_data);
}
-VERSION_SYMBOL(rte_timer_manage, _v20, 2.0);
int
rte_timer_manage_v1905(void)
@@ -836,7 +832,7 @@ rte_timer_manage_v1905(void)
return 0;
}
MAP_STATIC_SYMBOL(int rte_timer_manage(void), rte_timer_manage_v1905);
-BIND_DEFAULT_SYMBOL(rte_timer_manage, _v1905, 19.05);
+BIND_DEFAULT_SYMBOL(rte_timer_manage, _v1905, 20.0);
int
rte_timer_alt_manage(uint32_t timer_data_id,
@@ -1079,7 +1075,6 @@ rte_timer_dump_stats_v20(FILE *f)
{
__rte_timer_dump_stats(&default_timer_data, f);
}
-VERSION_SYMBOL(rte_timer_dump_stats, _v20, 2.0);
int
rte_timer_dump_stats_v1905(FILE *f)
@@ -1088,7 +1083,7 @@ rte_timer_dump_stats_v1905(FILE *f)
}
MAP_STATIC_SYMBOL(int rte_timer_dump_stats(FILE *f),
rte_timer_dump_stats_v1905);
-BIND_DEFAULT_SYMBOL(rte_timer_dump_stats, _v1905, 19.05);
+BIND_DEFAULT_SYMBOL(rte_timer_dump_stats, _v1905, 20.0);
int
rte_timer_alt_dump_stats(uint32_t timer_data_id __rte_unused, FILE *f)
diff --git a/lib/librte_timer/rte_timer_version.map b/lib/librte_timer/rte_timer_version.map
index 72f75c818..1303571fe 100644
--- a/lib/librte_timer/rte_timer_version.map
+++ b/lib/librte_timer/rte_timer_version.map
@@ -1,4 +1,4 @@
-DPDK_2.0 {
+DPDK_20.0 {
global:
rte_timer_dump_stats;
@@ -10,20 +10,9 @@ DPDK_2.0 {
rte_timer_stop;
rte_timer_stop_sync;
rte_timer_subsystem_init;
-
local: *;
};
-DPDK_19.05 {
- global:
-
- rte_timer_dump_stats;
- rte_timer_manage;
- rte_timer_reset;
- rte_timer_stop;
- rte_timer_subsystem_init;
-} DPDK_2.0;
-
EXPERIMENTAL {
global:
diff --git a/lib/librte_vhost/rte_vhost_version.map b/lib/librte_vhost/rte_vhost_version.map
index 5f1d4a75c..7d6178a26 100644
--- a/lib/librte_vhost/rte_vhost_version.map
+++ b/lib/librte_vhost/rte_vhost_version.map
@@ -1,90 +1,59 @@
-DPDK_2.0 {
+DPDK_20.0 {
global:
+ rte_vhost_avail_entries;
rte_vhost_dequeue_burst;
rte_vhost_driver_callback_register;
- rte_vhost_driver_register;
- rte_vhost_enable_guest_notification;
- rte_vhost_enqueue_burst;
-
- local: *;
-};
-
-DPDK_2.1 {
- global:
-
- rte_vhost_driver_unregister;
-
-} DPDK_2.0;
-
-DPDK_16.07 {
- global:
-
- rte_vhost_avail_entries;
- rte_vhost_get_ifname;
- rte_vhost_get_numa_node;
- rte_vhost_get_queue_num;
-
-} DPDK_2.1;
-
-DPDK_17.05 {
- global:
-
rte_vhost_driver_disable_features;
rte_vhost_driver_enable_features;
rte_vhost_driver_get_features;
+ rte_vhost_driver_register;
rte_vhost_driver_set_features;
rte_vhost_driver_start;
+ rte_vhost_driver_unregister;
+ rte_vhost_enable_guest_notification;
+ rte_vhost_enqueue_burst;
+ rte_vhost_get_ifname;
rte_vhost_get_mem_table;
rte_vhost_get_mtu;
rte_vhost_get_negotiated_features;
+ rte_vhost_get_numa_node;
+ rte_vhost_get_queue_num;
rte_vhost_get_vhost_vring;
rte_vhost_get_vring_num;
rte_vhost_gpa_to_vva;
rte_vhost_log_used_vring;
rte_vhost_log_write;
-
-} DPDK_16.07;
-
-DPDK_17.08 {
- global:
-
rte_vhost_rx_queue_count;
-
-} DPDK_17.05;
-
-DPDK_18.02 {
- global:
-
rte_vhost_vring_call;
-
-} DPDK_17.08;
+ local: *;
+};
EXPERIMENTAL {
global:
- rte_vdpa_register_device;
- rte_vdpa_unregister_device;
rte_vdpa_find_device_id;
rte_vdpa_get_device;
rte_vdpa_get_device_num;
+ rte_vdpa_register_device;
+ rte_vdpa_relay_vring_used;
+ rte_vdpa_unregister_device;
+ rte_vhost_crypto_create;
+ rte_vhost_crypto_fetch_requests;
+ rte_vhost_crypto_finalize_requests;
+ rte_vhost_crypto_free;
+ rte_vhost_crypto_set_zero_copy;
rte_vhost_driver_attach_vdpa_device;
rte_vhost_driver_detach_vdpa_device;
- rte_vhost_driver_get_vdpa_device_id;
- rte_vhost_get_vdpa_device_id;
rte_vhost_driver_get_protocol_features;
rte_vhost_driver_get_queue_num;
+ rte_vhost_driver_get_vdpa_device_id;
+ rte_vhost_driver_set_protocol_features;
+ rte_vhost_extern_callback_register;
rte_vhost_get_log_base;
+ rte_vhost_get_vdpa_device_id;
rte_vhost_get_vring_base;
+ rte_vhost_host_notifier_ctrl;
rte_vhost_set_vring_base;
- rte_vhost_crypto_create;
- rte_vhost_crypto_free;
- rte_vhost_crypto_fetch_requests;
- rte_vhost_crypto_finalize_requests;
- rte_vhost_crypto_set_zero_copy;
rte_vhost_va_from_guest_pa;
- rte_vhost_host_notifier_ctrl;
- rte_vdpa_relay_vring_used;
- rte_vhost_extern_callback_register;
- rte_vhost_driver_set_protocol_features;
};
--
2.22.0.windows.1
^ permalink raw reply [relevance 1%]
* [dpdk-dev] [PATCH 3/8] buildtools: add ABI versioning check script
2019-09-30 9:21 8% [dpdk-dev] [PATCH 1/8] config: change ABI versioning for global Marcin Baran
2019-09-30 9:21 10% ` [dpdk-dev] [PATCH 2/8] buildtools: scripts for updating symbols abi version Marcin Baran
@ 2019-09-30 9:21 23% ` Marcin Baran
2019-09-30 9:21 1% ` [dpdk-dev] [PATCH 4/8] build: change ABI version to 20.0 Marcin Baran
2019-09-30 9:21 2% ` [dpdk-dev] [PATCH 5/8] lib: remove dead code from timer Marcin Baran
3 siblings, 0 replies; 200+ results
From: Marcin Baran @ 2019-09-30 9:21 UTC (permalink / raw)
To: dev, bruce.richardson, ray.kinsella; +Cc: Marcin Baran, Pawel Modrak
The script 'check-abi-version.sh' should be used
to check whether built libraries are versioned
with correct ABI number (provided ABI, provided
ABI+1 or EXPERIMENTAL).
Signed-off-by: Marcin Baran <marcinx.baran@intel.com>
Signed-off-by: Pawel Modrak <pawelx.modrak@intel.com>
---
buildtools/check-abi-version.sh | 46 +++++++++++++++++++++++++++++++++
buildtools/update_abi.sh | 41 +++++++++++++++++++++++++++++
2 files changed, 87 insertions(+)
create mode 100755 buildtools/check-abi-version.sh
create mode 100755 buildtools/update_abi.sh
diff --git a/buildtools/check-abi-version.sh b/buildtools/check-abi-version.sh
new file mode 100755
index 000000000..ead25c080
--- /dev/null
+++ b/buildtools/check-abi-version.sh
@@ -0,0 +1,46 @@
+#!/bin/sh
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2019 Intel Corporation
+
+# Check whether library symbols have correct
+# version (provided ABI number or provided ABI
+# number + 1 or EXPERIMENTAL).
+# Args:
+# $1: path of the library .so file
+# $2: ABI major version number to check
+# (defaults to ABI_VERSION file value)
+
+if [ -z "$1" ]; then
+ echo "Script checks whether library symbols have"
+ echo "correct version (ABI_VER/ABI_VER+1/EXPERIMENTAL)"
+ echo "Usage:"
+ echo " $0 SO_FILE_PATH [ABI_VER]"
+ exit 1
+fi
+
+LIB="$1"
+DEFAULT_ABI=$(cat "$(dirname \
+ $(readlink -f $0))/../config/ABI_VERSION" | \
+ cut -d'.' -f 1)
+ABIVER="DPDK_${2-$DEFAULT_ABI}"
+NEXT_ABIVER="DPDK_$((${2-$DEFAULT_ABI}+1))"
+
+ret=0
+
+for SYM in `objdump -TC --section=.text ${LIB} | \
+ grep -e "DPDK\|EXPERIMENTAL" | \
+ awk '{print $(NF-1) "-" $NF}'`
+do
+ version=$(echo $SYM | cut -d'-' -f 1)
+ symbol=$(echo $SYM | cut -d'-' -f 2)
+ case $version in (*"$ABIVER"*|*"$NEXT_ABIVER"*|"EXPERIMENTAL")
+ ;;
+ (*)
+ echo "Warning: symbol $symbol ($version) should be annotated " \
+ "as ABI version $ABIVER / $NEXT_ABIVER, or EXPERIMENTAL."
+ ret=1
+ ;;
+ esac
+done
+
+exit $ret
diff --git a/buildtools/update_abi.sh b/buildtools/update_abi.sh
new file mode 100755
index 000000000..43ee79a92
--- /dev/null
+++ b/buildtools/update_abi.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2019 Intel Corporation
+
+abi_version=""
+abi_version_file="./config/ABI_VERSION"
+update_path="lib drivers"
+
+if [ -z "$1" ]
+then
+ echo "\$provide ABI version"
+fi
+
+abi_version=$1
+abi_version_with_prefix="DPDK_$abi_version"
+
+if [ -n "$2" ]
+then
+ abi_version_file=$2
+fi
+
+if [ -n "$3" ]
+then
+ update_path=${@:3}
+fi
+
+echo "New ABI version:" $abi_version
+echo "ABI_VERSION path:" $abi_version_file
+echo "Path to update:" $update_path
+
+echo $abi_version > $abi_version_file
+
+grep --binary-files=without-match --recursive --files-with-matches \
+--max-count=1 --include \*.c 'BIND_DEFAULT_SYMBOL\|VERSION_SYMBOL' \
+$update_path | xargs --max-lines=1 --verbose -I {} \
+./buildtools/update_default_symbol_abi.py {} \
+$abi_version_with_prefix
+
+find $update_path -name \*version.map -exec \
+./buildtools/update_version_map_abi.py {} \
+$abi_version_with_prefix \; -print
--
2.22.0.windows.1
^ permalink raw reply [relevance 23%]
* [dpdk-dev] [PATCH 2/8] buildtools: scripts for updating symbols abi version
2019-09-30 9:21 8% [dpdk-dev] [PATCH 1/8] config: change ABI versioning for global Marcin Baran
@ 2019-09-30 9:21 10% ` Marcin Baran
2019-09-30 9:21 23% ` [dpdk-dev] [PATCH 3/8] buildtools: add ABI versioning check script Marcin Baran
` (2 subsequent siblings)
3 siblings, 0 replies; 200+ results
From: Marcin Baran @ 2019-09-30 9:21 UTC (permalink / raw)
To: dev, bruce.richardson, ray.kinsella; +Cc: Pawel Modrak
From: Pawel Modrak <pawelx.modrak@intel.com>
The scripts updates version number for binding symbols
and version map files.
Signed-off-by: Pawel Modrak <pawelx.modrak@intel.com>
---
buildtools/update_default_symbol_abi.py | 57 ++++++++++++
buildtools/update_version_map_abi.py | 110 ++++++++++++++++++++++++
2 files changed, 167 insertions(+)
create mode 100755 buildtools/update_default_symbol_abi.py
create mode 100755 buildtools/update_version_map_abi.py
diff --git a/buildtools/update_default_symbol_abi.py b/buildtools/update_default_symbol_abi.py
new file mode 100755
index 000000000..6fe60d8cd
--- /dev/null
+++ b/buildtools/update_default_symbol_abi.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2019 Intel Corporation
+
+from __future__ import print_function
+import sys
+import argparse
+import re
+
+def setup_options():
+ arg_parser = argparse.ArgumentParser(description='Update default bind symbol abi version.')
+ arg_parser.add_argument('map_file', metavar='map_file', type=str, nargs=1,
+ help='path to source file (pattern: *.c)')
+ arg_parser.add_argument('dst_abi_version', metavar='dst_abi_version', type=str, nargs=1,
+ help='target ABI version (pattern: DPDK_*)')
+ arg_parser.add_argument('-v', '--verbose', action='store_true', help='print changes')
+
+ return arg_parser.parse_args()
+
+def replace_abi(f_in, abi_version_number, verbose):
+ source_file_content = []
+
+ for ln_no, ln in enumerate(f_in, 1):
+ if re.search("^BIND_DEFAULT_SYMBOL\(.*", ln):
+ source_file_content += [re.sub(", [0-9]{1,2}\.[0-9]{1,2}\);$", ", " + abi_version_number + ");", ln)]
+ if verbose:
+ print(f_in.name + ":" + str(ln_no) + ": " + ln[:-1] + " -> " + source_file_content[-1][:-1])
+ elif re.search("^VERSION_SYMBOL\(.*", ln):
+ if verbose:
+ print(f_in.name + ":" + str(ln_no) + ": " + ln[:-1] + " -> " + "[DELETED]")
+ else:
+ source_file_content += [ln]
+
+ return source_file_content
+
+def main(args):
+ params = setup_options()
+
+ if not params.map_file[0].endswith('.c') or \
+ not params.dst_abi_version[0].startswith('DPDK_'):
+ print('Wrong pattern for input files!\n')
+ arg_parser.print_help()
+ return 1
+
+ abi_version_number = params.dst_abi_version[0][5:]
+ source_file_content = []
+
+ with open(params.map_file[0]) as f_in:
+ source_file_content = replace_abi(f_in, abi_version_number, params.verbose)
+
+ with open(params.map_file[0], 'w') as f_out:
+ f_out.writelines(source_file_content)
+
+ return 0
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv))
diff --git a/buildtools/update_version_map_abi.py b/buildtools/update_version_map_abi.py
new file mode 100755
index 000000000..5a840e766
--- /dev/null
+++ b/buildtools/update_version_map_abi.py
@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2019 Intel Corporation
+
+from __future__ import print_function
+import sys
+import argparse
+
+def setup_options():
+ arg_parser = argparse.ArgumentParser(description='Merge versions in linker version script.')
+ arg_parser.add_argument("map_file", metavar='map_file', type=str, nargs=1,
+ help='path to linker version script file (pattern: *version.map)')
+ arg_parser.add_argument("dst_abi_version", metavar='dst_abi_version', type=str, nargs=1,
+ help='target ABI version (pattern: DPDK_*)')
+ arg_parser.add_argument("src_abi_version", metavar='src_abi_version', default="DPDK_", type=str, nargs='?',
+ help='source ABI version (pattern: DPDK_*, default: DPDK_)')
+ arg_parser.add_argument('-v', '--verbose', action='store_true', help='print changes')
+
+ return arg_parser.parse_args()
+
+def is_function_line(ln):
+ return ln.startswith('\t') and ln.endswith(';\n') and ":" not in ln
+
+def is_dpdk_version_start_line(ln, src_abi_version):
+ return ln.startswith(src_abi_version) and ln.endswith('{\n') and ":" not in ln
+
+def is_experimental_version_start_line(ln):
+ return ln.startswith('EXPERIMENTAL {') and ln.endswith('\n') and ":" not in ln
+
+def is_version_end_line(ln):
+ return ln.startswith('}') and ln.endswith(';\n') and ":" not in ln
+
+def is_global_line(ln):
+ return ln.startswith('\tglobal:') and ln.endswith('\n')
+
+def is_local_line(ln):
+ return ln.startswith('\tlocal: ') and ln.endswith(';\n')
+
+def store_functions(f_in):
+ functions = []
+ local_line = []
+
+ for line in f_in:
+ if is_version_end_line(line): break
+ elif is_local_line(line):
+ local_line = [line]
+ elif is_function_line(line): functions += [line]
+
+ return functions, local_line
+
+def parse_linker_version_map_file(f_in, src_abi_version):
+ dpdk_functions = []
+ experimental_functions = []
+ dpdk_local_line = []
+ experimental_local_line = []
+
+ for line in f_in:
+ if is_dpdk_version_start_line(line, src_abi_version):
+ dpdk_functions_tmp, dpdk_local_line_tmp = store_functions(f_in)
+ dpdk_functions += dpdk_functions_tmp
+ dpdk_local_line = dpdk_local_line_tmp if len(dpdk_local_line_tmp) > 0 else dpdk_local_line
+ elif is_experimental_version_start_line(line):
+ experimental_functions_tmp, experimental_local_line_tmp = store_functions(f_in)
+ experimental_functions += experimental_functions_tmp
+ experimental_local_line += experimental_local_line_tmp
+
+ return list(set(dpdk_functions)), list(set(experimental_functions)), list(set(dpdk_local_line)), list(set(experimental_local_line))
+
+def main(args):
+ params = setup_options()
+
+ if not params.map_file[0].endswith('version.map') or \
+ not params.dst_abi_version[0].startswith('DPDK_'):
+ print('Wrong pattern for input files!\n')
+ arg_parser.print_help()
+ return 1
+
+ dpdk_functions = []
+ experimental_functions = []
+ dpdk_local_line = []
+ experimental_local_line = []
+
+ with open(params.map_file[0]) as f_in:
+ dpdk_functions, experimental_functions, dpdk_local_line, experimental_local_line = parse_linker_version_map_file(f_in, params.src_abi_version)
+
+ dpdk_functions.sort()
+ experimental_functions.sort()
+
+ with open(params.map_file[0], 'w') as f_out:
+ if len(dpdk_functions) > 0:
+ dpdk_functions = params.dst_abi_version + [" {\n"] + ["\tglobal:\n\n"] + dpdk_functions + dpdk_local_line + ["};\n\n"]
+ if params.verbose:
+ print(*dpdk_functions)
+ f_out.writelines(dpdk_functions)
+ elif len(dpdk_local_line) > 0:
+ dpdk_functions = params.dst_abi_version + [" {\n"] + ["\n\tlocal: *;\n};\n"]
+ if params.verbose:
+ print(*dpdk_functions)
+ f_out.writelines(dpdk_functions)
+
+ if len(experimental_functions) > 0:
+ experimental_functions = ["EXPERIMENTAL" + " {\n"] + ["\tglobal:\n\n"] + experimental_functions + experimental_local_line + ["};\n"]
+ if params.verbose:
+ print(*experimental_functions)
+ f_out.writelines(experimental_functions)
+
+ return 0
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv))
--
2.22.0.windows.1
^ permalink raw reply [relevance 10%]
* [dpdk-dev] [PATCH 1/8] config: change ABI versioning for global
@ 2019-09-30 9:21 8% Marcin Baran
2019-09-30 9:21 10% ` [dpdk-dev] [PATCH 2/8] buildtools: scripts for updating symbols abi version Marcin Baran
` (3 more replies)
0 siblings, 4 replies; 200+ results
From: Marcin Baran @ 2019-09-30 9:21 UTC (permalink / raw)
To: dev, bruce.richardson, ray.kinsella; +Cc: Marcin Baran, Pawel Modrak
The libraries should be maintained using global
ABI versioning. The changes includes adding global
ABI version support for both makefile and meson
build system. Experimental libraries should be
marked as 0.
Signed-off-by: Marcin Baran <marcinx.baran@intel.com>
Signed-off-by: Pawel Modrak <pawelx.modrak@intel.com>
---
buildtools/meson.build | 2 ++
config/ABI_VERSION | 1 +
config/meson.build | 3 ++-
drivers/meson.build | 20 ++++++++++++++------
lib/meson.build | 18 +++++++++++++-----
meson_options.txt | 2 --
mk/rte.lib.mk | 19 +++++++++++--------
7 files changed, 43 insertions(+), 22 deletions(-)
create mode 100644 config/ABI_VERSION
diff --git a/buildtools/meson.build b/buildtools/meson.build
index 32c79c130..78ce69977 100644
--- a/buildtools/meson.build
+++ b/buildtools/meson.build
@@ -12,3 +12,5 @@ if python3.found()
else
map_to_def_cmd = ['meson', 'runpython', files('map_to_def.py')]
endif
+
+is_experimental_cmd = [find_program('grep', 'findstr'), '^DPDK_']
diff --git a/config/ABI_VERSION b/config/ABI_VERSION
new file mode 100644
index 000000000..9a7c1e503
--- /dev/null
+++ b/config/ABI_VERSION
@@ -0,0 +1 @@
+20.0
diff --git a/config/meson.build b/config/meson.build
index 2bafea530..16ad8968d 100644
--- a/config/meson.build
+++ b/config/meson.build
@@ -17,7 +17,8 @@ endforeach
# set the major version, which might be used by drivers and libraries
# depending on the configuration options
pver = meson.project_version().split('.')
-major_version = '@0@.@1@'.format(pver.get(0), pver.get(1))
+major_version = run_command(find_program('cat', 'more'),
+ files('ABI_VERSION')).stdout().strip()
# extract all version information into the build configuration
dpdk_conf.set('RTE_VER_YEAR', pver.get(0).to_int())
diff --git a/drivers/meson.build b/drivers/meson.build
index 2ed2e9541..5c5fe87c7 100644
--- a/drivers/meson.build
+++ b/drivers/meson.build
@@ -110,9 +110,20 @@ foreach class:dpdk_driver_classes
output: out_filename,
depends: [pmdinfogen, tmp_lib])
- if get_option('per_library_versions')
- lib_version = '@0@.1'.format(version)
- so_version = '@0@'.format(version)
+ version_map = '@0@/@1@/@2@_version.map'.format(
+ meson.current_source_dir(),
+ drv_path, lib_name)
+
+ if is_windows
+ version_map = '\\'.join(version_map.split('/'))
+ endif
+
+ is_experimental = run_command(is_experimental_cmd,
+ files(version_map)).returncode()
+
+ if is_experimental != 0
+ lib_version = '0.1'
+ so_version = '0'
else
lib_version = major_version
so_version = major_version
@@ -128,9 +139,6 @@ foreach class:dpdk_driver_classes
install: true)
# now build the shared driver
- version_map = '@0@/@1@/@2@_version.map'.format(
- meson.current_source_dir(),
- drv_path, lib_name)
shared_lib = shared_library(lib_name,
sources,
objects: objs,
diff --git a/lib/meson.build b/lib/meson.build
index e5ff83893..3892c16e8 100644
--- a/lib/meson.build
+++ b/lib/meson.build
@@ -97,9 +97,19 @@ foreach l:libraries
cflags += '-DALLOW_EXPERIMENTAL_API'
endif
- if get_option('per_library_versions')
- lib_version = '@0@.1'.format(version)
- so_version = '@0@'.format(version)
+ version_map = '@0@/@1@/rte_@2@_version.map'.format(
+ meson.current_source_dir(), dir_name, name)
+
+ if is_windows
+ version_map = '\\'.join(version_map.split('/'))
+ endif
+
+ is_experimental = run_command(is_experimental_cmd,
+ files(version_map)).returncode()
+
+ if is_experimental != 0
+ lib_version = '0.1'
+ so_version = '0'
else
lib_version = major_version
so_version = major_version
@@ -120,8 +130,6 @@ foreach l:libraries
# then use pre-build objects to build shared lib
sources = []
objs += static_lib.extract_all_objects(recursive: false)
- version_map = '@0@/@1@/rte_@2@_version.map'.format(
- meson.current_source_dir(), dir_name, name)
implib = dir_name + '.dll.a'
def_file = custom_target(name + '_def',
diff --git a/meson_options.txt b/meson_options.txt
index 448f3e63d..000e38fd9 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -28,8 +28,6 @@ option('max_lcores', type: 'integer', value: 128,
description: 'maximum number of cores/threads supported by EAL')
option('max_numa_nodes', type: 'integer', value: 4,
description: 'maximum number of NUMA nodes supported by EAL')
-option('per_library_versions', type: 'boolean', value: true,
- description: 'true: each lib gets its own version number, false: DPDK version used for each lib')
option('tests', type: 'boolean', value: true,
description: 'build unit tests')
option('use_hpet', type: 'boolean', value: false,
diff --git a/mk/rte.lib.mk b/mk/rte.lib.mk
index 4df8849a0..f84161c6d 100644
--- a/mk/rte.lib.mk
+++ b/mk/rte.lib.mk
@@ -11,20 +11,23 @@ EXTLIB_BUILD ?= n
# VPATH contains at least SRCDIR
VPATH += $(SRCDIR)
-ifneq ($(CONFIG_RTE_MAJOR_ABI),)
-ifneq ($(LIBABIVER),)
-LIBABIVER := $(CONFIG_RTE_MAJOR_ABI)
+ifeq ($(OS), Windows_NT)
+search_cmd = findstr
+print_cmd = more
+else
+search_cmd = grep
+print_cmd = cat
endif
+
+ifneq ($(shell $(search_cmd) "^DPDK_" $(SRCDIR)/$(EXPORT_MAP)),)
+LIBABIVER := $(shell $(print_cmd) $(RTE_SRCDIR)/config/ABI_VERSION)
+else
+LIBABIVER := 0
endif
ifeq ($(CONFIG_RTE_BUILD_SHARED_LIB),y)
LIB := $(patsubst %.a,%.so.$(LIBABIVER),$(LIB))
ifeq ($(EXTLIB_BUILD),n)
-ifeq ($(CONFIG_RTE_MAJOR_ABI),)
-ifeq ($(CONFIG_RTE_NEXT_ABI),y)
-LIB := $(LIB).1
-endif
-endif
CPU_LDFLAGS += --version-script=$(SRCDIR)/$(EXPORT_MAP)
endif
endif
--
2.22.0.windows.1
^ permalink raw reply [relevance 8%]
* [dpdk-dev] [PATCH 0/5] mbuf related patches
@ 2019-09-28 0:37 3% Stephen Hemminger
0 siblings, 0 replies; 200+ results
From: Stephen Hemminger @ 2019-09-28 0:37 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger
This patch set is all about improving the mbuf related cloning
and copying. They are motivated by seeing issues with mbuf copying
in rte_pdump and realized this a wider and more general problem.
The pdump copy could not handle different size pools and did
not handle meta data, etc.
They cause no functional or ABI changes. The only visible part
to older code is converting a couple of inlines to real functions.
This kind of change confuses checkpatch which thinks these new
functions should be marked experimental when they must not be.
Stephen Hemminger (5):
mbuf: don't generate invalid mbuf in clone test
mbuf: delinline rte_pktmbuf_linearize
mbuf: deinline rte_pktmbuf_clone
mbuf: add a pktmbuf copy routine
mbuf: add pktmbuf copy test
app/test/test_mbuf.c | 129 +++++++++++++++++++++++
lib/librte_mbuf/rte_mbuf.c | 149 +++++++++++++++++++++++++++
lib/librte_mbuf/rte_mbuf.h | 102 ++++++------------
lib/librte_mbuf/rte_mbuf_version.map | 8 ++
4 files changed, 315 insertions(+), 73 deletions(-)
--
2.20.1
^ permalink raw reply [relevance 3%]
* [dpdk-dev] [PATCH v2 2/2] build: support building ABI versioned files twice
2019-09-27 20:59 3% ` [dpdk-dev] [PATCH v2 0/2] Improve function versioning meson support Bruce Richardson
2019-09-27 20:59 10% ` [dpdk-dev] [PATCH v2 1/2] eal: split compat header file Bruce Richardson
@ 2019-09-27 20:59 15% ` Bruce Richardson
1 sibling, 0 replies; 200+ results
From: Bruce Richardson @ 2019-09-27 20:59 UTC (permalink / raw)
To: dev
Cc: Andrzej Ostruszka, Thomas Monjalon, Ray Kinsella, Neil Horman,
bluca, Bruce Richardson
Any file with ABI versioned functions needs different macros for shared and
static builds, so we need to accomodate that. Rather than building
everything twice, we just flag to the build system which libraries need
that handling, by setting use_function_versioning in the meson.build files.
To ensure we don't get silent errors at build time due to this meson flag
being missed, we add an explicit error to the function versioning header
file if a known C macro is not defined. Since "make" builds always only
build one of shared or static libraries, this define can be always set, and
so is added to the common_base file. For meson, the build flag - and
therefore the C define - is set for the three libraries that need the
function versioning: "distributor", "lpm" and "timer".
Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
---
config/common_base | 1 +
config/rte_config.h | 3 ---
doc/guides/contributing/coding_style.rst | 7 +++++++
lib/librte_distributor/meson.build | 1 +
.../common/include/rte_function_versioning.h | 4 ++++
lib/librte_lpm/meson.build | 1 +
lib/librte_timer/meson.build | 1 +
lib/meson.build | 16 +++++++++++++---
8 files changed, 28 insertions(+), 6 deletions(-)
diff --git a/config/common_base b/config/common_base
index 8ef75c203..655258a97 100644
--- a/config/common_base
+++ b/config/common_base
@@ -111,6 +111,7 @@ CONFIG_RTE_MAX_VFIO_CONTAINERS=64
CONFIG_RTE_MALLOC_DEBUG=n
CONFIG_RTE_EAL_NUMA_AWARE_HUGEPAGES=n
CONFIG_RTE_USE_LIBBSD=n
+CONFIG_RTE_USE_FUNCTION_VERSIONING=y
#
# Recognize/ignore the AVX/AVX512 CPU flags for performance/power testing.
diff --git a/config/rte_config.h b/config/rte_config.h
index 0bbbe274f..b63a2fdea 100644
--- a/config/rte_config.h
+++ b/config/rte_config.h
@@ -31,9 +31,6 @@
/****** library defines ********/
-/* compat defines */
-#define RTE_BUILD_SHARED_LIB
-
/* EAL defines */
#define RTE_MAX_HEAPS 32
#define RTE_MAX_MEMSEG_LISTS 128
diff --git a/doc/guides/contributing/coding_style.rst b/doc/guides/contributing/coding_style.rst
index 449b33494..e95a1a2be 100644
--- a/doc/guides/contributing/coding_style.rst
+++ b/doc/guides/contributing/coding_style.rst
@@ -948,6 +948,13 @@ reason
built. For missing dependencies this should be of the form
``'missing dependency, "libname"'``.
+use_function_versioning
+ **Default Value = false**.
+ Specifies if the library in question has ABI versioned functions. If it
+ has, this value should be set to ensure that the C files are compiled
+ twice with suitable parameters for each of shared or static library
+ builds.
+
version
**Default Value = 1**.
Specifies the ABI version of the library, and is used as the major
diff --git a/lib/librte_distributor/meson.build b/lib/librte_distributor/meson.build
index dba7e3b2a..5149f9bf5 100644
--- a/lib/librte_distributor/meson.build
+++ b/lib/librte_distributor/meson.build
@@ -9,3 +9,4 @@ else
endif
headers = files('rte_distributor.h')
deps += ['mbuf']
+use_function_versioning = true
diff --git a/lib/librte_eal/common/include/rte_function_versioning.h b/lib/librte_eal/common/include/rte_function_versioning.h
index ce963d4b1..55e88ffae 100644
--- a/lib/librte_eal/common/include/rte_function_versioning.h
+++ b/lib/librte_eal/common/include/rte_function_versioning.h
@@ -7,6 +7,10 @@
#define _RTE_FUNCTION_VERSIONING_H_
#include <rte_common.h>
+#ifndef RTE_USE_FUNCTION_VERSIONING
+#error Use of function versioning disabled, is "use_function_versioning=true" in meson.build?
+#endif
+
#ifdef RTE_BUILD_SHARED_LIB
/*
diff --git a/lib/librte_lpm/meson.build b/lib/librte_lpm/meson.build
index a5176d8ae..4e3920660 100644
--- a/lib/librte_lpm/meson.build
+++ b/lib/librte_lpm/meson.build
@@ -8,3 +8,4 @@ headers = files('rte_lpm.h', 'rte_lpm6.h')
# without worrying about which architecture we actually need
headers += files('rte_lpm_altivec.h', 'rte_lpm_neon.h', 'rte_lpm_sse.h')
deps += ['hash']
+use_function_versioning = true
diff --git a/lib/librte_timer/meson.build b/lib/librte_timer/meson.build
index d3b828ce9..b7edfe2e7 100644
--- a/lib/librte_timer/meson.build
+++ b/lib/librte_timer/meson.build
@@ -4,3 +4,4 @@
sources = files('rte_timer.c')
headers = files('rte_timer.h')
allow_experimental_apis = true
+use_function_versioning = true
diff --git a/lib/meson.build b/lib/meson.build
index e5ff83893..1b0ed767c 100644
--- a/lib/meson.build
+++ b/lib/meson.build
@@ -47,6 +47,7 @@ foreach l:libraries
name = l
version = 1
allow_experimental_apis = false
+ use_function_versioning = false
sources = []
headers = []
includes = []
@@ -96,6 +97,9 @@ foreach l:libraries
if allow_experimental_apis
cflags += '-DALLOW_EXPERIMENTAL_API'
endif
+ if use_function_versioning
+ cflags += '-DRTE_USE_FUNCTION_VERSIONING'
+ endif
if get_option('per_library_versions')
lib_version = '@0@.1'.format(version)
@@ -117,9 +121,15 @@ foreach l:libraries
include_directories: includes,
dependencies: static_deps)
- # then use pre-build objects to build shared lib
- sources = []
- objs += static_lib.extract_all_objects(recursive: false)
+ if not use_function_versioning
+ # use pre-build objects to build shared lib
+ sources = []
+ objs += static_lib.extract_all_objects(recursive: false)
+ else
+ # for compat we need to rebuild with
+ # RTE_BUILD_SHARED_LIB defined
+ cflags += '-DRTE_BUILD_SHARED_LIB'
+ endif
version_map = '@0@/@1@/rte_@2@_version.map'.format(
meson.current_source_dir(), dir_name, name)
implib = dir_name + '.dll.a'
--
2.21.0
^ permalink raw reply [relevance 15%]
* [dpdk-dev] [PATCH v2 1/2] eal: split compat header file
2019-09-27 20:59 3% ` [dpdk-dev] [PATCH v2 0/2] Improve function versioning meson support Bruce Richardson
@ 2019-09-27 20:59 10% ` Bruce Richardson
2019-09-27 20:59 15% ` [dpdk-dev] [PATCH v2 2/2] build: support building ABI versioned files twice Bruce Richardson
1 sibling, 0 replies; 200+ results
From: Bruce Richardson @ 2019-09-27 20:59 UTC (permalink / raw)
To: dev
Cc: Andrzej Ostruszka, Thomas Monjalon, Ray Kinsella, Neil Horman,
bluca, Bruce Richardson
The compat.h header file provided macros for two purposes:
1. it provided the macros for marking functions as rte_experimental
2. it provided the macros for doing function versioning
Although these were in the same file, #1 is something that is for use by
public header files, which #2 is for internal use only. Therefore, we can
split these into two headers, keeping #1 in rte_compat.h and #2 in a new
file rte_function_versioning.h. For "make" builds, since internal objects
pick up the headers from the "include/" folder, we need to add the new
header to the installation list, but for "meson" builds it does not need to
be installed as it's not for public use.
The rework also serves to allow the use of the function versioning macros
to files that actually need them, so the use of experimental functions does
not need including of the versioning code.
Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
---
V2: added in missing rte_compat.h file
---
doc/api/doxy-api-index.md | 3 +-
doc/guides/contributing/versioning.rst | 4 +-
lib/librte_distributor/rte_distributor.c | 2 +-
lib/librte_distributor/rte_distributor_v20.c | 2 +-
lib/librte_eal/common/Makefile | 1 +
lib/librte_eal/common/include/rte_compat.h | 70 ----------------
.../common/include/rte_function_versioning.h | 79 +++++++++++++++++++
lib/librte_lpm/rte_lpm.c | 1 +
lib/librte_lpm/rte_lpm.h | 1 -
lib/librte_lpm/rte_lpm6.c | 1 +
lib/librte_timer/rte_timer.c | 2 +-
11 files changed, 89 insertions(+), 77 deletions(-)
create mode 100644 lib/librte_eal/common/include/rte_function_versioning.h
diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
index 6c2d888ee..9acf36ba1 100644
--- a/doc/api/doxy-api-index.md
+++ b/doc/api/doxy-api-index.md
@@ -171,5 +171,6 @@ The public API headers are grouped by topics:
- **misc**:
[EAL config] (@ref rte_eal.h),
[common] (@ref rte_common.h),
- [ABI compat] (@ref rte_compat.h),
+ [experimental APIs] (@ref rte_compat.h),
+ [ABI versioning] (@ref rte_function_versioning.h),
[version] (@ref rte_version.h)
diff --git a/doc/guides/contributing/versioning.rst b/doc/guides/contributing/versioning.rst
index 3ab2c4346..64984c54e 100644
--- a/doc/guides/contributing/versioning.rst
+++ b/doc/guides/contributing/versioning.rst
@@ -206,7 +206,7 @@ functionality or behavior. When that occurs, it is desirable to allow for
backward compatibility for a time with older binaries that are dynamically
linked to the DPDK.
-To support backward compatibility the ``rte_compat.h``
+To support backward compatibility the ``rte_function_versioning.h``
header file provides macros to use when updating exported functions. These
macros are used in conjunction with the ``rte_<library>_version.map`` file for
a given library to allow multiple versions of a symbol to exist in a shared
@@ -362,7 +362,7 @@ the function, we add this line of code
VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
-Remembering to also add the rte_compat.h header to the requisite c file where
+Remembering to also add the rte_function_versioning.h header to the requisite c file where
these changes are being made. The above macro instructs the linker to create a
new symbol ``rte_acl_create@DPDK_2.0``, which matches the symbol created in older
builds, but now points to the above newly named function. We have now mapped
diff --git a/lib/librte_distributor/rte_distributor.c b/lib/librte_distributor/rte_distributor.c
index 21eb1fb0a..6d1e971a9 100644
--- a/lib/librte_distributor/rte_distributor.c
+++ b/lib/librte_distributor/rte_distributor.c
@@ -8,7 +8,7 @@
#include <rte_mbuf.h>
#include <rte_memory.h>
#include <rte_cycles.h>
-#include <rte_compat.h>
+#include <rte_function_versioning.h>
#include <rte_memzone.h>
#include <rte_errno.h>
#include <rte_string_fns.h>
diff --git a/lib/librte_distributor/rte_distributor_v20.c b/lib/librte_distributor/rte_distributor_v20.c
index cdc0969a8..64c611fa9 100644
--- a/lib/librte_distributor/rte_distributor_v20.c
+++ b/lib/librte_distributor/rte_distributor_v20.c
@@ -9,7 +9,7 @@
#include <rte_memory.h>
#include <rte_memzone.h>
#include <rte_errno.h>
-#include <rte_compat.h>
+#include <rte_function_versioning.h>
#include <rte_string_fns.h>
#include <rte_eal_memconfig.h>
#include <rte_pause.h>
diff --git a/lib/librte_eal/common/Makefile b/lib/librte_eal/common/Makefile
index a00d4fcad..d70f84fd7 100644
--- a/lib/librte_eal/common/Makefile
+++ b/lib/librte_eal/common/Makefile
@@ -4,6 +4,7 @@
include $(RTE_SDK)/mk/rte.vars.mk
INC := rte_branch_prediction.h rte_common.h rte_compat.h
+INC += rte_function_versioning.h
INC += rte_debug.h rte_eal.h rte_eal_interrupts.h
INC += rte_errno.h rte_launch.h rte_lcore.h
INC += rte_log.h rte_memory.h rte_memzone.h
diff --git a/lib/librte_eal/common/include/rte_compat.h b/lib/librte_eal/common/include/rte_compat.h
index 92ff28faf..3eb33784b 100644
--- a/lib/librte_eal/common/include/rte_compat.h
+++ b/lib/librte_eal/common/include/rte_compat.h
@@ -5,76 +5,6 @@
#ifndef _RTE_COMPAT_H_
#define _RTE_COMPAT_H_
-#include <rte_common.h>
-
-#ifdef RTE_BUILD_SHARED_LIB
-
-/*
- * Provides backwards compatibility when updating exported functions.
- * When a symol is exported from a library to provide an API, it also provides a
- * calling convention (ABI) that is embodied in its name, return type,
- * arguments, etc. On occasion that function may need to change to accommodate
- * new functionality, behavior, etc. When that occurs, it is desirable to
- * allow for backwards compatibility for a time with older binaries that are
- * dynamically linked to the dpdk. To support that, the __vsym and
- * VERSION_SYMBOL macros are created. They, in conjunction with the
- * <library>_version.map file for a given library allow for multiple versions of
- * a symbol to exist in a shared library so that older binaries need not be
- * immediately recompiled.
- *
- * Refer to the guidelines document in the docs subdirectory for details on the
- * use of these macros
- */
-
-/*
- * Macro Parameters:
- * b - function base name
- * e - function version extension, to be concatenated with base name
- * n - function symbol version string to be applied
- * f - function prototype
- * p - full function symbol name
- */
-
-/*
- * VERSION_SYMBOL
- * Creates a symbol version table entry binding symbol <b>@DPDK_<n> to the internal
- * function name <b>_<e>
- */
-#define VERSION_SYMBOL(b, e, n) __asm__(".symver " RTE_STR(b) RTE_STR(e) ", " RTE_STR(b) "@DPDK_" RTE_STR(n))
-
-/*
- * BIND_DEFAULT_SYMBOL
- * Creates a symbol version entry instructing the linker to bind references to
- * symbol <b> to the internal symbol <b>_<e>
- */
-#define BIND_DEFAULT_SYMBOL(b, e, n) __asm__(".symver " RTE_STR(b) RTE_STR(e) ", " RTE_STR(b) "@@DPDK_" RTE_STR(n))
-#define __vsym __attribute__((used))
-
-/*
- * MAP_STATIC_SYMBOL
- * If a function has been bifurcated into multiple versions, none of which
- * are defined as the exported symbol name in the map file, this macro can be
- * used to alias a specific version of the symbol to its exported name. For
- * example, if you have 2 versions of a function foo_v1 and foo_v2, where the
- * former is mapped to foo@DPDK_1 and the latter is mapped to foo@DPDK_2 when
- * building a shared library, this macro can be used to map either foo_v1 or
- * foo_v2 to the symbol foo when building a static library, e.g.:
- * MAP_STATIC_SYMBOL(void foo(), foo_v2);
- */
-#define MAP_STATIC_SYMBOL(f, p)
-
-#else
-/*
- * No symbol versioning in use
- */
-#define VERSION_SYMBOL(b, e, n)
-#define __vsym
-#define BIND_DEFAULT_SYMBOL(b, e, n)
-#define MAP_STATIC_SYMBOL(f, p) f __attribute__((alias(RTE_STR(p))))
-/*
- * RTE_BUILD_SHARED_LIB=n
- */
-#endif
#ifndef ALLOW_EXPERIMENTAL_API
diff --git a/lib/librte_eal/common/include/rte_function_versioning.h b/lib/librte_eal/common/include/rte_function_versioning.h
new file mode 100644
index 000000000..ce963d4b1
--- /dev/null
+++ b/lib/librte_eal/common/include/rte_function_versioning.h
@@ -0,0 +1,79 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2015 Neil Horman <nhorman@tuxdriver.com>.
+ * All rights reserved.
+ */
+
+#ifndef _RTE_FUNCTION_VERSIONING_H_
+#define _RTE_FUNCTION_VERSIONING_H_
+#include <rte_common.h>
+
+#ifdef RTE_BUILD_SHARED_LIB
+
+/*
+ * Provides backwards compatibility when updating exported functions.
+ * When a symol is exported from a library to provide an API, it also provides a
+ * calling convention (ABI) that is embodied in its name, return type,
+ * arguments, etc. On occasion that function may need to change to accommodate
+ * new functionality, behavior, etc. When that occurs, it is desirable to
+ * allow for backwards compatibility for a time with older binaries that are
+ * dynamically linked to the dpdk. To support that, the __vsym and
+ * VERSION_SYMBOL macros are created. They, in conjunction with the
+ * <library>_version.map file for a given library allow for multiple versions of
+ * a symbol to exist in a shared library so that older binaries need not be
+ * immediately recompiled.
+ *
+ * Refer to the guidelines document in the docs subdirectory for details on the
+ * use of these macros
+ */
+
+/*
+ * Macro Parameters:
+ * b - function base name
+ * e - function version extension, to be concatenated with base name
+ * n - function symbol version string to be applied
+ * f - function prototype
+ * p - full function symbol name
+ */
+
+/*
+ * VERSION_SYMBOL
+ * Creates a symbol version table entry binding symbol <b>@DPDK_<n> to the internal
+ * function name <b>_<e>
+ */
+#define VERSION_SYMBOL(b, e, n) __asm__(".symver " RTE_STR(b) RTE_STR(e) ", " RTE_STR(b) "@DPDK_" RTE_STR(n))
+
+/*
+ * BIND_DEFAULT_SYMBOL
+ * Creates a symbol version entry instructing the linker to bind references to
+ * symbol <b> to the internal symbol <b>_<e>
+ */
+#define BIND_DEFAULT_SYMBOL(b, e, n) __asm__(".symver " RTE_STR(b) RTE_STR(e) ", " RTE_STR(b) "@@DPDK_" RTE_STR(n))
+#define __vsym __attribute__((used))
+
+/*
+ * MAP_STATIC_SYMBOL
+ * If a function has been bifurcated into multiple versions, none of which
+ * are defined as the exported symbol name in the map file, this macro can be
+ * used to alias a specific version of the symbol to its exported name. For
+ * example, if you have 2 versions of a function foo_v1 and foo_v2, where the
+ * former is mapped to foo@DPDK_1 and the latter is mapped to foo@DPDK_2 when
+ * building a shared library, this macro can be used to map either foo_v1 or
+ * foo_v2 to the symbol foo when building a static library, e.g.:
+ * MAP_STATIC_SYMBOL(void foo(), foo_v2);
+ */
+#define MAP_STATIC_SYMBOL(f, p)
+
+#else
+/*
+ * No symbol versioning in use
+ */
+#define VERSION_SYMBOL(b, e, n)
+#define __vsym
+#define BIND_DEFAULT_SYMBOL(b, e, n)
+#define MAP_STATIC_SYMBOL(f, p) f __attribute__((alias(RTE_STR(p))))
+/*
+ * RTE_BUILD_SHARED_LIB=n
+ */
+#endif
+
+#endif /* _RTE_FUNCTION_VERSIONING_H_ */
diff --git a/lib/librte_lpm/rte_lpm.c b/lib/librte_lpm/rte_lpm.c
index 3a929a1b1..c96395e26 100644
--- a/lib/librte_lpm/rte_lpm.c
+++ b/lib/librte_lpm/rte_lpm.c
@@ -22,6 +22,7 @@
#include <rte_rwlock.h>
#include <rte_spinlock.h>
#include <rte_tailq.h>
+#include <rte_function_versioning.h>
#include "rte_lpm.h"
diff --git a/lib/librte_lpm/rte_lpm.h b/lib/librte_lpm/rte_lpm.h
index 906ec4483..26303e628 100644
--- a/lib/librte_lpm/rte_lpm.h
+++ b/lib/librte_lpm/rte_lpm.h
@@ -20,7 +20,6 @@
#include <rte_memory.h>
#include <rte_common.h>
#include <rte_vect.h>
-#include <rte_compat.h>
#ifdef __cplusplus
extern "C" {
diff --git a/lib/librte_lpm/rte_lpm6.c b/lib/librte_lpm/rte_lpm6.c
index 9b8aeb972..e20f82460 100644
--- a/lib/librte_lpm/rte_lpm6.c
+++ b/lib/librte_lpm/rte_lpm6.c
@@ -25,6 +25,7 @@
#include <assert.h>
#include <rte_jhash.h>
#include <rte_tailq.h>
+#include <rte_function_versioning.h>
#include "rte_lpm6.h"
diff --git a/lib/librte_timer/rte_timer.c b/lib/librte_timer/rte_timer.c
index bdcf05d06..3834c9473 100644
--- a/lib/librte_timer/rte_timer.c
+++ b/lib/librte_timer/rte_timer.c
@@ -25,8 +25,8 @@
#include <rte_pause.h>
#include <rte_memzone.h>
#include <rte_malloc.h>
-#include <rte_compat.h>
#include <rte_errno.h>
+#include <rte_function_versioning.h>
#include "rte_timer.h"
--
2.21.0
^ permalink raw reply [relevance 10%]
* [dpdk-dev] [PATCH v2 0/2] Improve function versioning meson support
2019-09-27 19:49 3% [dpdk-dev] [PATCH 0/2] Improve function versioning meson support Bruce Richardson
2019-09-27 19:49 11% ` [dpdk-dev] [PATCH 1/2] eal: split compat header file Bruce Richardson
2019-09-27 19:49 15% ` [dpdk-dev] [PATCH 2/2] build: support building ABI versioned files twice Bruce Richardson
@ 2019-09-27 20:59 3% ` Bruce Richardson
2019-09-27 20:59 10% ` [dpdk-dev] [PATCH v2 1/2] eal: split compat header file Bruce Richardson
2019-09-27 20:59 15% ` [dpdk-dev] [PATCH v2 2/2] build: support building ABI versioned files twice Bruce Richardson
2 siblings, 2 replies; 200+ results
From: Bruce Richardson @ 2019-09-27 20:59 UTC (permalink / raw)
To: dev
Cc: Andrzej Ostruszka, Thomas Monjalon, Ray Kinsella, Neil Horman,
bluca, Bruce Richardson
Adding support for LTO has exposed some issues with how the functions
versioning was supported by meson, which was always set to build both
shared and static libraries.
For plain C code, so long as the -fPIC compiler flag was passed, the
output is identical whether or not the code is to be included in a
static library or a dynamic one. Unfortunately, when using function
versioning that no longer held as different macros were used for the
versioned functions depending on which type of build it was. This means
that any files that use versioning need to be built twice, with
different defines in each case.
While the trivial solution here is just to rebuild everything twice,
that involves a lot of unnecessary work when building DPDK. A better
option is to identify those files or components which need multiple
builds and rebuild only those. To do this, we add a new meson.build
setting for libraries "use_function_versioning" and when that is set, we
rebuild all source files twice, initially for static library and then
with -DRTE_BUILD_SHARED_LIB for the shared library.
If the flag is not set, then the static versioning setting only is used,
which could lead to the build succeeding but later causing problems. To
avoid that, we add a new define which must be set when the versioning
header is included. This addition while solving 1 problem raises 2
other, more minor problems:
* what to do with make builds? since make only builds one library type,
we can just always define the new value.
* what about files that include rte_compat.h for the macro for
"experimental"? To solve this, we can split compat.h in two, since the
versioning macro should be internal only to DPDK (as no public header
should expose anything but the latest APIs), while the experimental
macros are primarily for public use.
V2: added in file that missed a "git add" when doing V1
Bruce Richardson (2):
eal: split compat header file
build: support building ABI versioned files twice
config/common_base | 1 +
config/rte_config.h | 3 -
doc/api/doxy-api-index.md | 3 +-
doc/guides/contributing/coding_style.rst | 7 ++
doc/guides/contributing/versioning.rst | 4 +-
lib/librte_distributor/meson.build | 1 +
lib/librte_distributor/rte_distributor.c | 2 +-
lib/librte_distributor/rte_distributor_v20.c | 2 +-
lib/librte_eal/common/Makefile | 1 +
lib/librte_eal/common/include/rte_compat.h | 70 ----------------
.../common/include/rte_function_versioning.h | 83 +++++++++++++++++++
lib/librte_lpm/meson.build | 1 +
lib/librte_lpm/rte_lpm.c | 1 +
lib/librte_lpm/rte_lpm.h | 1 -
lib/librte_lpm/rte_lpm6.c | 1 +
lib/librte_timer/meson.build | 1 +
lib/librte_timer/rte_timer.c | 2 +-
lib/meson.build | 16 +++-
18 files changed, 117 insertions(+), 83 deletions(-)
create mode 100644 lib/librte_eal/common/include/rte_function_versioning.h
--
2.21.0
^ permalink raw reply [relevance 3%]
* [dpdk-dev] [PATCH 2/2] build: support building ABI versioned files twice
2019-09-27 19:49 3% [dpdk-dev] [PATCH 0/2] Improve function versioning meson support Bruce Richardson
2019-09-27 19:49 11% ` [dpdk-dev] [PATCH 1/2] eal: split compat header file Bruce Richardson
@ 2019-09-27 19:49 15% ` Bruce Richardson
2019-09-27 20:59 3% ` [dpdk-dev] [PATCH v2 0/2] Improve function versioning meson support Bruce Richardson
2 siblings, 0 replies; 200+ results
From: Bruce Richardson @ 2019-09-27 19:49 UTC (permalink / raw)
To: dev
Cc: Andrzej Ostruszka, Thomas Monjalon, Ray Kinsella, Neil Horman,
Bruce Richardson
Any file with ABI versioned functions needs different macros for shared and
static builds, so we need to accomodate that. Rather than building
everything twice, we just flag to the build system which libraries need
that handling, by setting use_function_versioning in the meson.build files.
To ensure we don't get silent errors at build time due to this meson flag
being missed, we add an explicit error to the function versioning header
file if a known C macro is not defined. Since "make" builds always only
build one of shared or static libraries, this define can be always set, and
so is added to the common_base file. For meson, the build flag - and
therefore the C define - is set for the three libraries that need the
function versioning: "distributor", "lpm" and "timer".
Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
---
config/common_base | 1 +
config/rte_config.h | 3 ---
doc/guides/contributing/coding_style.rst | 7 +++++++
lib/librte_distributor/meson.build | 1 +
.../common/include/rte_function_versioning.h | 4 ++++
lib/librte_lpm/meson.build | 1 +
lib/librte_timer/meson.build | 1 +
lib/meson.build | 16 +++++++++++++---
8 files changed, 28 insertions(+), 6 deletions(-)
diff --git a/config/common_base b/config/common_base
index 8ef75c203..655258a97 100644
--- a/config/common_base
+++ b/config/common_base
@@ -111,6 +111,7 @@ CONFIG_RTE_MAX_VFIO_CONTAINERS=64
CONFIG_RTE_MALLOC_DEBUG=n
CONFIG_RTE_EAL_NUMA_AWARE_HUGEPAGES=n
CONFIG_RTE_USE_LIBBSD=n
+CONFIG_RTE_USE_FUNCTION_VERSIONING=y
#
# Recognize/ignore the AVX/AVX512 CPU flags for performance/power testing.
diff --git a/config/rte_config.h b/config/rte_config.h
index 0bbbe274f..b63a2fdea 100644
--- a/config/rte_config.h
+++ b/config/rte_config.h
@@ -31,9 +31,6 @@
/****** library defines ********/
-/* compat defines */
-#define RTE_BUILD_SHARED_LIB
-
/* EAL defines */
#define RTE_MAX_HEAPS 32
#define RTE_MAX_MEMSEG_LISTS 128
diff --git a/doc/guides/contributing/coding_style.rst b/doc/guides/contributing/coding_style.rst
index 449b33494..e95a1a2be 100644
--- a/doc/guides/contributing/coding_style.rst
+++ b/doc/guides/contributing/coding_style.rst
@@ -948,6 +948,13 @@ reason
built. For missing dependencies this should be of the form
``'missing dependency, "libname"'``.
+use_function_versioning
+ **Default Value = false**.
+ Specifies if the library in question has ABI versioned functions. If it
+ has, this value should be set to ensure that the C files are compiled
+ twice with suitable parameters for each of shared or static library
+ builds.
+
version
**Default Value = 1**.
Specifies the ABI version of the library, and is used as the major
diff --git a/lib/librte_distributor/meson.build b/lib/librte_distributor/meson.build
index dba7e3b2a..5149f9bf5 100644
--- a/lib/librte_distributor/meson.build
+++ b/lib/librte_distributor/meson.build
@@ -9,3 +9,4 @@ else
endif
headers = files('rte_distributor.h')
deps += ['mbuf']
+use_function_versioning = true
diff --git a/lib/librte_eal/common/include/rte_function_versioning.h b/lib/librte_eal/common/include/rte_function_versioning.h
index ce963d4b1..55e88ffae 100644
--- a/lib/librte_eal/common/include/rte_function_versioning.h
+++ b/lib/librte_eal/common/include/rte_function_versioning.h
@@ -7,6 +7,10 @@
#define _RTE_FUNCTION_VERSIONING_H_
#include <rte_common.h>
+#ifndef RTE_USE_FUNCTION_VERSIONING
+#error Use of function versioning disabled, is "use_function_versioning=true" in meson.build?
+#endif
+
#ifdef RTE_BUILD_SHARED_LIB
/*
diff --git a/lib/librte_lpm/meson.build b/lib/librte_lpm/meson.build
index a5176d8ae..4e3920660 100644
--- a/lib/librte_lpm/meson.build
+++ b/lib/librte_lpm/meson.build
@@ -8,3 +8,4 @@ headers = files('rte_lpm.h', 'rte_lpm6.h')
# without worrying about which architecture we actually need
headers += files('rte_lpm_altivec.h', 'rte_lpm_neon.h', 'rte_lpm_sse.h')
deps += ['hash']
+use_function_versioning = true
diff --git a/lib/librte_timer/meson.build b/lib/librte_timer/meson.build
index d3b828ce9..b7edfe2e7 100644
--- a/lib/librte_timer/meson.build
+++ b/lib/librte_timer/meson.build
@@ -4,3 +4,4 @@
sources = files('rte_timer.c')
headers = files('rte_timer.h')
allow_experimental_apis = true
+use_function_versioning = true
diff --git a/lib/meson.build b/lib/meson.build
index e5ff83893..1b0ed767c 100644
--- a/lib/meson.build
+++ b/lib/meson.build
@@ -47,6 +47,7 @@ foreach l:libraries
name = l
version = 1
allow_experimental_apis = false
+ use_function_versioning = false
sources = []
headers = []
includes = []
@@ -96,6 +97,9 @@ foreach l:libraries
if allow_experimental_apis
cflags += '-DALLOW_EXPERIMENTAL_API'
endif
+ if use_function_versioning
+ cflags += '-DRTE_USE_FUNCTION_VERSIONING'
+ endif
if get_option('per_library_versions')
lib_version = '@0@.1'.format(version)
@@ -117,9 +121,15 @@ foreach l:libraries
include_directories: includes,
dependencies: static_deps)
- # then use pre-build objects to build shared lib
- sources = []
- objs += static_lib.extract_all_objects(recursive: false)
+ if not use_function_versioning
+ # use pre-build objects to build shared lib
+ sources = []
+ objs += static_lib.extract_all_objects(recursive: false)
+ else
+ # for compat we need to rebuild with
+ # RTE_BUILD_SHARED_LIB defined
+ cflags += '-DRTE_BUILD_SHARED_LIB'
+ endif
version_map = '@0@/@1@/rte_@2@_version.map'.format(
meson.current_source_dir(), dir_name, name)
implib = dir_name + '.dll.a'
--
2.21.0
^ permalink raw reply [relevance 15%]
* [dpdk-dev] [PATCH 1/2] eal: split compat header file
2019-09-27 19:49 3% [dpdk-dev] [PATCH 0/2] Improve function versioning meson support Bruce Richardson
@ 2019-09-27 19:49 11% ` Bruce Richardson
2019-09-27 19:49 15% ` [dpdk-dev] [PATCH 2/2] build: support building ABI versioned files twice Bruce Richardson
2019-09-27 20:59 3% ` [dpdk-dev] [PATCH v2 0/2] Improve function versioning meson support Bruce Richardson
2 siblings, 0 replies; 200+ results
From: Bruce Richardson @ 2019-09-27 19:49 UTC (permalink / raw)
To: dev
Cc: Andrzej Ostruszka, Thomas Monjalon, Ray Kinsella, Neil Horman,
Bruce Richardson
The compat.h header file provided macros for two purposes:
1. it provided the macros for marking functions as rte_experimental
2. it provided the macros for doing function versioning
Although these were in the same file, #1 is something that is for use by
public header files, which #2 is for internal use only. Therefore, we can
split these into two headers, keeping #1 in rte_compat.h and #2 in a new
file rte_function_versioning.h. For "make" builds, since internal objects
pick up the headers from the "include/" folder, we need to add the new
header to the installation list, but for "meson" builds it does not need to
be installed as it's not for public use.
The rework also serves to allow the use of the function versioning macros
to files that actually need them, so the use of experimental functions does
not need including of the versioning code.
Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
---
doc/api/doxy-api-index.md | 3 ++-
doc/guides/contributing/versioning.rst | 4 ++--
lib/librte_distributor/rte_distributor.c | 2 +-
lib/librte_distributor/rte_distributor_v20.c | 2 +-
lib/librte_eal/common/Makefile | 1 +
...rte_compat.h => rte_function_versioning.h} | 19 +++----------------
lib/librte_lpm/rte_lpm.c | 1 +
lib/librte_lpm/rte_lpm.h | 1 -
lib/librte_lpm/rte_lpm6.c | 1 +
lib/librte_timer/rte_timer.c | 2 +-
10 files changed, 13 insertions(+), 23 deletions(-)
rename lib/librte_eal/common/include/{rte_compat.h => rte_function_versioning.h} (89%)
diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
index 6c2d888ee..9acf36ba1 100644
--- a/doc/api/doxy-api-index.md
+++ b/doc/api/doxy-api-index.md
@@ -171,5 +171,6 @@ The public API headers are grouped by topics:
- **misc**:
[EAL config] (@ref rte_eal.h),
[common] (@ref rte_common.h),
- [ABI compat] (@ref rte_compat.h),
+ [experimental APIs] (@ref rte_compat.h),
+ [ABI versioning] (@ref rte_function_versioning.h),
[version] (@ref rte_version.h)
diff --git a/doc/guides/contributing/versioning.rst b/doc/guides/contributing/versioning.rst
index 3ab2c4346..64984c54e 100644
--- a/doc/guides/contributing/versioning.rst
+++ b/doc/guides/contributing/versioning.rst
@@ -206,7 +206,7 @@ functionality or behavior. When that occurs, it is desirable to allow for
backward compatibility for a time with older binaries that are dynamically
linked to the DPDK.
-To support backward compatibility the ``rte_compat.h``
+To support backward compatibility the ``rte_function_versioning.h``
header file provides macros to use when updating exported functions. These
macros are used in conjunction with the ``rte_<library>_version.map`` file for
a given library to allow multiple versions of a symbol to exist in a shared
@@ -362,7 +362,7 @@ the function, we add this line of code
VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
-Remembering to also add the rte_compat.h header to the requisite c file where
+Remembering to also add the rte_function_versioning.h header to the requisite c file where
these changes are being made. The above macro instructs the linker to create a
new symbol ``rte_acl_create@DPDK_2.0``, which matches the symbol created in older
builds, but now points to the above newly named function. We have now mapped
diff --git a/lib/librte_distributor/rte_distributor.c b/lib/librte_distributor/rte_distributor.c
index 21eb1fb0a..6d1e971a9 100644
--- a/lib/librte_distributor/rte_distributor.c
+++ b/lib/librte_distributor/rte_distributor.c
@@ -8,7 +8,7 @@
#include <rte_mbuf.h>
#include <rte_memory.h>
#include <rte_cycles.h>
-#include <rte_compat.h>
+#include <rte_function_versioning.h>
#include <rte_memzone.h>
#include <rte_errno.h>
#include <rte_string_fns.h>
diff --git a/lib/librte_distributor/rte_distributor_v20.c b/lib/librte_distributor/rte_distributor_v20.c
index cdc0969a8..64c611fa9 100644
--- a/lib/librte_distributor/rte_distributor_v20.c
+++ b/lib/librte_distributor/rte_distributor_v20.c
@@ -9,7 +9,7 @@
#include <rte_memory.h>
#include <rte_memzone.h>
#include <rte_errno.h>
-#include <rte_compat.h>
+#include <rte_function_versioning.h>
#include <rte_string_fns.h>
#include <rte_eal_memconfig.h>
#include <rte_pause.h>
diff --git a/lib/librte_eal/common/Makefile b/lib/librte_eal/common/Makefile
index a00d4fcad..d70f84fd7 100644
--- a/lib/librte_eal/common/Makefile
+++ b/lib/librte_eal/common/Makefile
@@ -4,6 +4,7 @@
include $(RTE_SDK)/mk/rte.vars.mk
INC := rte_branch_prediction.h rte_common.h rte_compat.h
+INC += rte_function_versioning.h
INC += rte_debug.h rte_eal.h rte_eal_interrupts.h
INC += rte_errno.h rte_launch.h rte_lcore.h
INC += rte_log.h rte_memory.h rte_memzone.h
diff --git a/lib/librte_eal/common/include/rte_compat.h b/lib/librte_eal/common/include/rte_function_versioning.h
similarity index 89%
rename from lib/librte_eal/common/include/rte_compat.h
rename to lib/librte_eal/common/include/rte_function_versioning.h
index 92ff28faf..ce963d4b1 100644
--- a/lib/librte_eal/common/include/rte_compat.h
+++ b/lib/librte_eal/common/include/rte_function_versioning.h
@@ -3,8 +3,8 @@
* All rights reserved.
*/
-#ifndef _RTE_COMPAT_H_
-#define _RTE_COMPAT_H_
+#ifndef _RTE_FUNCTION_VERSIONING_H_
+#define _RTE_FUNCTION_VERSIONING_H_
#include <rte_common.h>
#ifdef RTE_BUILD_SHARED_LIB
@@ -76,17 +76,4 @@
*/
#endif
-#ifndef ALLOW_EXPERIMENTAL_API
-
-#define __rte_experimental \
-__attribute__((deprecated("Symbol is not yet part of stable ABI"), \
-section(".text.experimental")))
-
-#else
-
-#define __rte_experimental \
-__attribute__((section(".text.experimental")))
-
-#endif
-
-#endif /* _RTE_COMPAT_H_ */
+#endif /* _RTE_FUNCTION_VERSIONING_H_ */
diff --git a/lib/librte_lpm/rte_lpm.c b/lib/librte_lpm/rte_lpm.c
index 3a929a1b1..c96395e26 100644
--- a/lib/librte_lpm/rte_lpm.c
+++ b/lib/librte_lpm/rte_lpm.c
@@ -22,6 +22,7 @@
#include <rte_rwlock.h>
#include <rte_spinlock.h>
#include <rte_tailq.h>
+#include <rte_function_versioning.h>
#include "rte_lpm.h"
diff --git a/lib/librte_lpm/rte_lpm.h b/lib/librte_lpm/rte_lpm.h
index 906ec4483..26303e628 100644
--- a/lib/librte_lpm/rte_lpm.h
+++ b/lib/librte_lpm/rte_lpm.h
@@ -20,7 +20,6 @@
#include <rte_memory.h>
#include <rte_common.h>
#include <rte_vect.h>
-#include <rte_compat.h>
#ifdef __cplusplus
extern "C" {
diff --git a/lib/librte_lpm/rte_lpm6.c b/lib/librte_lpm/rte_lpm6.c
index 9b8aeb972..e20f82460 100644
--- a/lib/librte_lpm/rte_lpm6.c
+++ b/lib/librte_lpm/rte_lpm6.c
@@ -25,6 +25,7 @@
#include <assert.h>
#include <rte_jhash.h>
#include <rte_tailq.h>
+#include <rte_function_versioning.h>
#include "rte_lpm6.h"
diff --git a/lib/librte_timer/rte_timer.c b/lib/librte_timer/rte_timer.c
index bdcf05d06..3834c9473 100644
--- a/lib/librte_timer/rte_timer.c
+++ b/lib/librte_timer/rte_timer.c
@@ -25,8 +25,8 @@
#include <rte_pause.h>
#include <rte_memzone.h>
#include <rte_malloc.h>
-#include <rte_compat.h>
#include <rte_errno.h>
+#include <rte_function_versioning.h>
#include "rte_timer.h"
--
2.21.0
^ permalink raw reply [relevance 11%]
* [dpdk-dev] [PATCH 0/2] Improve function versioning meson support
@ 2019-09-27 19:49 3% Bruce Richardson
2019-09-27 19:49 11% ` [dpdk-dev] [PATCH 1/2] eal: split compat header file Bruce Richardson
` (2 more replies)
0 siblings, 3 replies; 200+ results
From: Bruce Richardson @ 2019-09-27 19:49 UTC (permalink / raw)
To: dev
Cc: Andrzej Ostruszka, Thomas Monjalon, Ray Kinsella, Neil Horman,
Bruce Richardson
Adding support for LTO has exposed some issues with how the functions
versioning was supported by meson, which was always set to build both
shared and static libraries.
For plain C code, so long as the -fPIC compiler flag was passed, the
output is identical whether or not the code is to be included in a
static library or a dynamic one. Unfortunately, when using function
versioning that no longer held as different macros were used for the
versioned functions depending on which type of build it was. This means
that any files that use versioning need to be built twice, with
different defines in each case.
While the trivial solution here is just to rebuild everything twice,
that involves a lot of unnecessary work when building DPDK. A better
option is to identify those files or components which need multiple
builds and rebuild only those. To do this, we add a new meson.build
setting for libraries "use_function_versioning" and when that is set, we
rebuild all source files twice, initially for static library and then
with -DRTE_BUILD_SHARED_LIB for the shared library.
If the flag is not set, then the static versioning setting only is used,
which could lead to the build succeeding but later causing problems. To
avoid that, we add a new define which must be set when the versioning
header is included. This addition while solving 1 problem raises 2
other, more minor problems:
* what to do with make builds? since make only builds one library type,
we can just always define the new value.
* what about files that include rte_compat.h for the macro for
"experimental"? To solve this, we can split compat.h in two, since the
versioning macro should be internal only to DPDK (as no public header
should expose anything but the latest APIs), while the experimental
macros are primarily for public use.
Bruce Richardson (2):
eal: split compat header file
build: support building ABI versioned files twice
config/common_base | 1 +
config/rte_config.h | 3 ---
doc/api/doxy-api-index.md | 3 ++-
doc/guides/contributing/coding_style.rst | 7 ++++++
doc/guides/contributing/versioning.rst | 4 ++--
lib/librte_distributor/meson.build | 1 +
lib/librte_distributor/rte_distributor.c | 2 +-
lib/librte_distributor/rte_distributor_v20.c | 2 +-
lib/librte_eal/common/Makefile | 1 +
...rte_compat.h => rte_function_versioning.h} | 23 ++++++-------------
lib/librte_lpm/meson.build | 1 +
lib/librte_lpm/rte_lpm.c | 1 +
lib/librte_lpm/rte_lpm.h | 1 -
lib/librte_lpm/rte_lpm6.c | 1 +
lib/librte_timer/meson.build | 1 +
lib/librte_timer/rte_timer.c | 2 +-
lib/meson.build | 16 ++++++++++---
17 files changed, 41 insertions(+), 29 deletions(-)
rename lib/librte_eal/common/include/{rte_compat.h => rte_function_versioning.h} (89%)
--
2.21.0
^ permalink raw reply [relevance 3%]
* [dpdk-dev] [PATCH v6 4/4] doc: add maintainer for abi policy
2019-09-27 16:54 10% [dpdk-dev] [PATCH v6 0/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
` (2 preceding siblings ...)
2019-09-27 16:54 30% ` [dpdk-dev] [PATCH v6 3/4] doc: updates to versioning guide for " Ray Kinsella
@ 2019-09-27 16:54 13% ` Ray Kinsella
3 siblings, 0 replies; 200+ results
From: Ray Kinsella @ 2019-09-27 16:54 UTC (permalink / raw)
To: dev
Cc: mdr, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, nhorman,
maxime.coquelin, john.mcnamara, marko.kovacevic, hemant.agrawal,
ktraynor, aconole
Add an entry to the maintainer file for the abi policy.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
MAINTAINERS | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/MAINTAINERS b/MAINTAINERS
index b3d9aad..d231f03 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -80,6 +80,10 @@ M: Marko Kovacevic <marko.kovacevic@intel.com>
F: README
F: doc/
+ABI Policy
+M: Ray Kinsella <mdr@ashroe.eu>
+F: doc/guides/contributing/abi_*.rst
+
Developers and Maintainers Tools
M: Thomas Monjalon <thomas@monjalon.net>
F: MAINTAINERS
--
2.7.4
^ permalink raw reply [relevance 13%]
* [dpdk-dev] [PATCH v6 3/4] doc: updates to versioning guide for abi versions
2019-09-27 16:54 10% [dpdk-dev] [PATCH v6 0/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
2019-09-27 16:54 13% ` [dpdk-dev] [PATCH v6 1/4] doc: separate versioning.rst into version and policy Ray Kinsella
2019-09-27 16:54 12% ` [dpdk-dev] [PATCH v6 2/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
@ 2019-09-27 16:54 30% ` Ray Kinsella
2019-09-27 16:54 13% ` [dpdk-dev] [PATCH v6 4/4] doc: add maintainer for abi policy Ray Kinsella
3 siblings, 0 replies; 200+ results
From: Ray Kinsella @ 2019-09-27 16:54 UTC (permalink / raw)
To: dev
Cc: mdr, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, nhorman,
maxime.coquelin, john.mcnamara, marko.kovacevic, hemant.agrawal,
ktraynor, aconole
Updates to the ABI versioning guide, to account for the changes to the DPDK
ABI/API policy. Fixes for references to abi versioning and policy guides.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
doc/guides/contributing/abi_versioning.rst | 248 +++++++++++++++++++----------
doc/guides/contributing/patches.rst | 6 +-
doc/guides/rel_notes/deprecation.rst | 2 +-
3 files changed, 172 insertions(+), 84 deletions(-)
diff --git a/doc/guides/contributing/abi_versioning.rst b/doc/guides/contributing/abi_versioning.rst
index 53e6ac0..76c63c8 100644
--- a/doc/guides/contributing/abi_versioning.rst
+++ b/doc/guides/contributing/abi_versioning.rst
@@ -1,44 +1,134 @@
.. SPDX-License-Identifier: BSD-3-Clause
Copyright 2018 The DPDK contributors
-.. library_versioning:
+.. _abi_versioning:
-Library versioning
+ABI Versioning
+==============
+
+This document details the mechanics of ABI version management in DPDK.
+
+.. _what_is_soname:
+
+What is a library's soname?
+---------------------------
+
+System libraries usually adopt the familiar major and minor version naming
+convention, where major versions (e.g. ``librte_eal 20.x, 21.x``) are presumed
+to be ABI incompatible with each other and minor versions (e.g. ``librte_eal
+20.1, 20.2``) are presumed to be ABI compatible. A library's `soname
+<https://en.wikipedia.org/wiki/Soname>`_. is typically used to provide backward
+compatibility information about a given library, describing the lowest common
+denominator ABI supported by the library. The soname or logical name for the
+library, is typically comprised of the library's name and major version e.g.
+``librte_eal.so.20``.
+
+During an application's build process, a library's soname is noted as a runtime
+dependency of the application. This information is then used by the `dynamic
+linker <https://en.wikipedia.org/wiki/Dynamic_linker>`_ when resolving the
+applications dependencies at runtime, to load a library supporting the correct
+ABI version. The library loaded at runtime therefore, may be a minor revision
+supporting the same major ABI version (e.g. ``librte_eal.20.2``), as the library
+used to link the application (e.g ``librte_eal.20.0``).
+
+.. _major_abi_versions:
+
+Major ABI versions
------------------
-Downstreams might want to provide different DPDK releases at the same time to
-support multiple consumers of DPDK linked against older and newer sonames.
+An ABI version change to a given library, especially in core libraries such as
+``librte_mbuf``, may cause an implicit ripple effect on the ABI of it's
+consuming libraries, causing ABI breakages. There may however be no explicit
+reason to bump a dependent library's ABI version, as there may have been no
+obvious change to the dependent library's API, even though the library's ABI
+compatibility will have been broken.
+
+This interdependence of DPDK libraries, means that ABI versioning of libraries
+is more manageable at a project level, with all project libraries sharing a
+**single ABI version**. In addition, the need to maintain a stable ABI for some
+number of releases as described in the section :ref:`abi_policy`, means
+that ABI version increments need to carefully planned and managed at a project
+level.
+
+Major ABI versions are therefore declared typically aligned with an LTS release
+and is then supported some number of subsequent releases, shared across all
+libraries. This means that a single project level ABI version, reflected in all
+individual library's soname, library filenames and associated version maps
+persists over multiple releases.
+
+.. code-block:: none
+
+ $ head ./lib/librte_acl/rte_acl_version.map
+ DPDK_20 {
+ global:
+ ...
-Also due to the interdependencies that DPDK libraries can have applications
-might end up with an executable space in which multiple versions of a library
-are mapped by ld.so.
+ $ head ./lib/librte_eal/rte_eal_version.map
+ DPDK_20 {
+ global:
+ ...
-Think of LibA that got an ABI bump and LibB that did not get an ABI bump but is
-depending on LibA.
+When an ABI change is made between major ABI versions to a given library, a new
+section is added to that library's version map describing the impending new ABI
+version, as described in the section :ref:`example_abi_macro_usage`. The
+library's soname and filename however do not change, e.g. ``libacl.so.20``, as
+ABI compatibility with the last major ABI version continues to be preserved for
+that library.
-.. note::
+.. code-block:: none
- Application
- \-> LibA.old
- \-> LibB.new -> LibA.new
+ $ head ./lib/librte_acl/rte_acl_version.map
+ DPDK_20 {
+ global:
+ ...
-That is a conflict which can be avoided by setting ``CONFIG_RTE_MAJOR_ABI``.
-If set, the value of ``CONFIG_RTE_MAJOR_ABI`` overwrites all - otherwise per
-library - versions defined in the libraries ``LIBABIVER``.
-An example might be ``CONFIG_RTE_MAJOR_ABI=16.11`` which will make all libraries
-``librte<?>.so.16.11`` instead of ``librte<?>.so.<LIBABIVER>``.
+ DPDK_21 {
+ global:
+
+ } DPDK_20;
+ ...
+ $ head ./lib/librte_eal/rte_eal_version.map
+ DPDK_20 {
+ global:
+ ...
+
+However when a new ABI version is declared, for example DPDK ``21``, old
+depreciated functions may be safely removed at this point and the entire old
+major ABI version removed, see the section :ref:`deprecating_entire_abi` on
+how this may be done.
+
+.. code-block:: none
+
+ $ head ./lib/librte_acl/rte_acl_version.map
+ DPDK_21 {
+ global:
+ ...
+
+ $ head ./lib/librte_eal/rte_eal_version.map
+ DPDK_21 {
+ global:
+ ...
+
+At the same time, the major ABI version is changed atomically across all
+libraries by incrementing the major version in individual library's soname, e.g.
+``libacl.so.21``. This is done by bumping the LIBABIVER number in the libraries
+Makefile to indicate to dynamic linking applications that this is a later, and
+possibly incompatible library version:
+
+.. code-block:: c
+
+ -LIBABIVER := 20
+ +LIBABIVER := 21
-ABI versioning
---------------
Versioning Macros
-~~~~~~~~~~~~~~~~~
+-----------------
When a symbol is exported from a library to provide an API, it also provides a
calling convention (ABI) that is embodied in its name, return type and
arguments. Occasionally that function may need to change to accommodate new
-functionality or behavior. When that occurs, it is desirable to allow for
+functionality or behavior. When that occurs, it is may be required to allow for
backward compatibility for a time with older binaries that are dynamically
linked to the DPDK.
@@ -61,8 +151,10 @@ The macros exported are:
fully qualified function ``p``, so that if a symbol becomes versioned, it
can still be mapped back to the public symbol name.
+.. _example_abi_macro_usage:
+
Examples of ABI Macro use
-^^^^^^^^^^^^^^^^^^^^^^^^^
+~~~~~~~~~~~~~~~~~~~~~~~~~
Updating a public API
_____________________
@@ -106,16 +198,16 @@ maintain previous ABI versions that are accessible only to previously compiled
binaries
The addition of a parameter to the function is ABI breaking as the function is
-public, and existing application may use it in its current form. However, the
+public, and existing application may use it in its current form. However, the
compatibility macros in DPDK allow a developer to use symbol versioning so that
multiple functions can be mapped to the same public symbol based on when an
-application was linked to it. To see how this is done, we start with the
-requisite libraries version map file. Initially the version map file for the
-acl library looks like this
+application was linked to it. To see how this is done, we start with the
+requisite libraries version map file. Initially the version map file for the acl
+library looks like this
.. code-block:: none
- DPDK_2.0 {
+ DPDK_20 {
global:
rte_acl_add_rules;
@@ -141,7 +233,7 @@ This file needs to be modified as follows
.. code-block:: none
- DPDK_2.0 {
+ DPDK_20 {
global:
rte_acl_add_rules;
@@ -163,16 +255,16 @@ This file needs to be modified as follows
local: *;
};
- DPDK_2.1 {
+ DPDK_21 {
global:
rte_acl_create;
- } DPDK_2.0;
+ } DPDK_20;
The addition of the new block tells the linker that a new version node is
-available (DPDK_2.1), which contains the symbol rte_acl_create, and inherits the
-symbols from the DPDK_2.0 node. This list is directly translated into a list of
-exported symbols when DPDK is compiled as a shared library
+available (DPDK_21), which contains the symbol rte_acl_create, and inherits
+the symbols from the DPDK_20 node. This list is directly translated into a
+list of exported symbols when DPDK is compiled as a shared library
Next, we need to specify in the code which function map to the rte_acl_create
symbol at which versions. First, at the site of the initial symbol definition,
@@ -191,22 +283,22 @@ with the public symbol name
Note that the base name of the symbol was kept intact, as this is conducive to
the macros used for versioning symbols. That is our next step, mapping this new
-symbol name to the initial symbol name at version node 2.0. Immediately after
+symbol name to the initial symbol name at version node 20. Immediately after
the function, we add this line of code
.. code-block:: c
- VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
+ VERSION_SYMBOL(rte_acl_create, _v20, 20);
Remembering to also add the rte_compat.h header to the requisite c file where
-these changes are being made. The above macro instructs the linker to create a
-new symbol ``rte_acl_create@DPDK_2.0``, which matches the symbol created in older
-builds, but now points to the above newly named function. We have now mapped
-the original rte_acl_create symbol to the original function (but with a new
-name)
+these changes are being made. The above macro instructs the linker to create a
+new symbol ``rte_acl_create@DPDK_20``, which matches the symbol created in
+older builds, but now points to the above newly named function. We have now
+mapped the original rte_acl_create symbol to the original function (but with a
+new name)
-Next, we need to create the 2.1 version of the symbol. We create a new function
-name, with a different suffix, and implement it appropriately
+Next, we need to create the 21 version of the symbol. We create a new function
+name, with a different suffix, and implement it appropriately
.. code-block:: c
@@ -220,12 +312,12 @@ name, with a different suffix, and implement it appropriately
return ctx;
}
-This code serves as our new API call. Its the same as our old call, but adds
-the new parameter in place. Next we need to map this function to the symbol
-``rte_acl_create@DPDK_2.1``. To do this, we modify the public prototype of the call
-in the header file, adding the macro there to inform all including applications,
-that on re-link, the default rte_acl_create symbol should point to this
-function. Note that we could do this by simply naming the function above
+This code serves as our new API call. Its the same as our old call, but adds the
+new parameter in place. Next we need to map this function to the symbol
+``rte_acl_create@DPDK_21``. To do this, we modify the public prototype of the
+call in the header file, adding the macro there to inform all including
+applications, that on re-link, the default rte_acl_create symbol should point to
+this function. Note that we could do this by simply naming the function above
rte_acl_create, and the linker would chose the most recent version tag to apply
in the version script, but we can also do this in the header file
@@ -233,11 +325,11 @@ in the version script, but we can also do this in the header file
struct rte_acl_ctx *
-rte_acl_create(const struct rte_acl_param *param);
- +rte_acl_create(const struct rte_acl_param *param, int debug);
- +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
+ +rte_acl_create_v21(const struct rte_acl_param *param, int debug);
+ +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 21);
The BIND_DEFAULT_SYMBOL macro explicitly tells applications that include this
-header, to link to the rte_acl_create_v21 function and apply the DPDK_2.1
+header, to link to the rte_acl_create_v21 function and apply the DPDK_21
version node to it. This method is more explicit and flexible than just
re-implementing the exact symbol name, and allows for other features (such as
linking to the old symbol version by default, when the new ABI is to be opt-in
@@ -257,6 +349,7 @@ assumption is that the most recent version of the symbol is the one you want to
map. So, back in the C file where, immediately after ``rte_acl_create_v21`` is
defined, we add this
+
.. code-block:: c
struct rte_acl_ctx *
@@ -270,21 +363,22 @@ That tells the compiler that, when building a static library, any calls to the
symbol ``rte_acl_create`` should be linked to ``rte_acl_create_v21``
That's it, on the next shared library rebuild, there will be two versions of
-rte_acl_create, an old DPDK_2.0 version, used by previously built applications,
-and a new DPDK_2.1 version, used by future built applications.
+rte_acl_create, an old DPDK_20 version, used by previously built applications,
+and a new DPDK_21 version, used by future built applications.
Deprecating part of a public API
________________________________
-Lets assume that you've done the above update, and after a few releases have
-passed you decide you would like to retire the old version of the function.
-After having gone through the ABI deprecation announcement process, removal is
-easy. Start by removing the symbol from the requisite version map file:
+Lets assume that you've done the above update, and in preparation for the next
+major ABI version you decide you would like to retire the old version of the
+function. After having gone through the ABI deprecation announcement process,
+removal is easy. Start by removing the symbol from the requisite version map
+file:
.. code-block:: none
- DPDK_2.0 {
+ DPDK_20 {
global:
rte_acl_add_rules;
@@ -306,48 +400,42 @@ easy. Start by removing the symbol from the requisite version map file:
local: *;
};
- DPDK_2.1 {
+ DPDK_21 {
global:
rte_acl_create;
- } DPDK_2.0;
+ } DPDK_20;
Next remove the corresponding versioned export.
.. code-block:: c
- -VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
+ -VERSION_SYMBOL(rte_acl_create, _v20, 20);
Note that the internal function definition could also be removed, but its used
-in our example by the newer version _v21, so we leave it in place. This is a
-coding style choice.
-
-Lastly, we need to bump the LIBABIVER number for this library in the Makefile to
-indicate to applications doing dynamic linking that this is a later, and
-possibly incompatible library version:
-
-.. code-block:: c
+in our example by the newer version v21, so we leave it in place and declare it
+as static. This is a coding style choice.
- -LIBABIVER := 1
- +LIBABIVER := 2
+.. _deprecating_entire_abi:
Deprecating an entire ABI version
_________________________________
-While removing a symbol from and ABI may be useful, it is often more practical
-to remove an entire version node at once. If a version node completely
-specifies an API, then removing part of it, typically makes it incomplete. In
-those cases it is better to remove the entire node
+While removing a symbol from an ABI may be useful, it is more practical to
+remove an entire version node at once, as is typically done at the declaration
+of a major ABI version. If a version node completely specifies an API, then
+removing part of it, typically makes it incomplete. In those cases it is better
+to remove the entire node.
To do this, start by modifying the version map file, such that all symbols from
-the node to be removed are merged into the next node in the map
+the node to be removed are merged into the next node in the map.
In the case of our map above, it would transform to look as follows
.. code-block:: none
- DPDK_2.1 {
+ DPDK_21 {
global:
rte_acl_add_rules;
@@ -375,8 +463,8 @@ symbols.
.. code-block:: c
- -BIND_DEFAULT_SYMBOL(rte_acl_create, _v20, 2.0);
- +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
+ -BIND_DEFAULT_SYMBOL(rte_acl_create, _v20, 20);
+ +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 21);
Lastly, any VERSION_SYMBOL macros that point to the old version node should be
removed, taking care to keep, where need old code in place to support newer
diff --git a/doc/guides/contributing/patches.rst b/doc/guides/contributing/patches.rst
index 9e1013b..d63c9f5 100644
--- a/doc/guides/contributing/patches.rst
+++ b/doc/guides/contributing/patches.rst
@@ -156,9 +156,9 @@ Make your planned changes in the cloned ``dpdk`` repo. Here are some guidelines
* For other PMDs and more info, refer to the ``MAINTAINERS`` file.
-* New external functions should be added to the local ``version.map`` file.
- See the :doc:`Guidelines for ABI policy and versioning </contributing/versioning>`.
- New external functions should also be added in alphabetical order.
+* New external functions should be added to the local ``version.map`` file. See
+ the :ref:`ABI policy <abi_policy>` and :ref:`ABI versioning <abi_versioning>`
+ guides. New external functions should also be added in alphabetical order.
* Important changes will require an addition to the release notes in ``doc/guides/rel_notes/``.
See the :ref:`Release Notes section of the Documentation Guidelines <doc_guidelines>` for details.
diff --git a/doc/guides/rel_notes/deprecation.rst b/doc/guides/rel_notes/deprecation.rst
index 0ee8533..2e163a5 100644
--- a/doc/guides/rel_notes/deprecation.rst
+++ b/doc/guides/rel_notes/deprecation.rst
@@ -4,7 +4,7 @@
ABI and API Deprecation
=======================
-See the :doc:`guidelines document for details of the ABI policy </contributing/versioning>`.
+See the :ref:`guidelines document for details of the ABI policy <abi_policy>`.
API and ABI deprecation notices are to be posted here.
--
2.7.4
^ permalink raw reply [relevance 30%]
* [dpdk-dev] [PATCH v6 2/4] doc: changes to abi policy introducing major abi versions
2019-09-27 16:54 10% [dpdk-dev] [PATCH v6 0/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
2019-09-27 16:54 13% ` [dpdk-dev] [PATCH v6 1/4] doc: separate versioning.rst into version and policy Ray Kinsella
@ 2019-09-27 16:54 12% ` Ray Kinsella
2019-09-27 16:54 30% ` [dpdk-dev] [PATCH v6 3/4] doc: updates to versioning guide for " Ray Kinsella
2019-09-27 16:54 13% ` [dpdk-dev] [PATCH v6 4/4] doc: add maintainer for abi policy Ray Kinsella
3 siblings, 0 replies; 200+ results
From: Ray Kinsella @ 2019-09-27 16:54 UTC (permalink / raw)
To: dev
Cc: mdr, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, nhorman,
maxime.coquelin, john.mcnamara, marko.kovacevic, hemant.agrawal,
ktraynor, aconole
This policy change introduces major ABI versions, these are
declared every year, typically aligned with the LTS release
and are supported by subsequent releases in the following year.
This change is intended to improve ABI stabilty for those projects
consuming DPDK.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
doc/guides/contributing/abi_policy.rst | 321 +++++++++++++++------
.../contributing/img/abi_stability_policy.png | Bin 0 -> 61277 bytes
doc/guides/contributing/img/what_is_an_abi.png | Bin 0 -> 151683 bytes
doc/guides/contributing/stable.rst | 12 +-
4 files changed, 241 insertions(+), 92 deletions(-)
create mode 100644 doc/guides/contributing/img/abi_stability_policy.png
create mode 100644 doc/guides/contributing/img/what_is_an_abi.png
diff --git a/doc/guides/contributing/abi_policy.rst b/doc/guides/contributing/abi_policy.rst
index 55bacb4..8862d24 100644
--- a/doc/guides/contributing/abi_policy.rst
+++ b/doc/guides/contributing/abi_policy.rst
@@ -1,33 +1,46 @@
.. SPDX-License-Identifier: BSD-3-Clause
- Copyright 2018 The DPDK contributors
+ Copyright 2019 The DPDK contributors
-.. abi_api_policy:
+.. _abi_policy:
-DPDK ABI/API policy
-===================
+ABI Policy
+==========
Description
-----------
-This document details some methods for handling ABI management in the DPDK.
+This document details the management policy that ensures the long-term stability
+of the DPDK ABI and API.
General Guidelines
------------------
-#. Whenever possible, ABI should be preserved
-#. ABI/API may be changed with a deprecation process
-#. The modification of symbols can generally be managed with versioning
-#. Libraries or APIs marked in ``experimental`` state may change without constraint
-#. New APIs will be marked as ``experimental`` for at least one release to allow
- any issues found by users of the new API to be fixed quickly
-#. The addition of symbols is generally not problematic
-#. The removal of symbols generally is an ABI break and requires bumping of the
- LIBABIVER macro
-#. Updates to the minimum hardware requirements, which drop support for hardware which
- was previously supported, should be treated as an ABI change.
-
-What is an ABI
-~~~~~~~~~~~~~~
+#. Major ABI versions are declared every **year** and are then supported for one
+ year, typically aligned with the :ref:`LTS release <stable_lts_releases>`.
+#. The ABI version is managed at a project level in DPDK, with the ABI version
+ reflected in all :ref:`library's soname <what_is_soname>`.
+#. The ABI should be preserved and not changed lightly. ABI changes must follow
+ the outlined :ref:`deprecation process <abi_changes>`.
+#. The addition of symbols is generally not problematic. The modification of
+ symbols is managed with :ref:`ABI Versioning <abi_versioning>`.
+#. The removal of symbols is considered an :ref:`ABI breakage <abi_breakages>`,
+ once approved these will form part of the next ABI version.
+#. Libraries or APIs marked as :ref:`Experimental <experimental_apis>` are not
+ considered part of an ABI version and may change without constraint.
+#. Updates to the :ref:`minimum hardware requirements <hw_rqmts>`, which drop
+ support for hardware which was previously supported, should be treated as an
+ ABI change.
+
+.. note::
+
+ In 2019, the DPDK community stated it's intention to move to ABI stable
+ releases, over a number of release cycles. Beginning with maintaining ABI
+ stability through one year of DPDK releases starting from DPDK 19.11. This
+ policy will be reviewed in 2020, with intention of lengthening the stability
+ period.
+
+What is an ABI?
+~~~~~~~~~~~~~~~
An ABI (Application Binary Interface) is the set of runtime interfaces exposed
by a library. It is similar to an API (Application Programming Interface) but
@@ -39,30 +52,80 @@ Therefore, in the case of dynamic linking, it is critical that an ABI is
preserved, or (when modified), done in such a way that the application is unable
to behave improperly or in an unexpected fashion.
+.. _figure_what_is_an_abi:
+
+.. figure:: img/what_is_an_abi.*
+
+*Figure 1. Illustration of DPDK API and ABI .*
-ABI/API Deprecation
--------------------
+
+What is an ABI version?
+~~~~~~~~~~~~~~~~~~~~~~~
+
+An ABI version is an instance of a library's ABI at a specific release. Certain
+releases are considered by the community to be milestone releases, the yearly
+LTS for example. Supporting those milestone release's ABI for some number of
+subsequent releases is desirable to facilitate application upgrade. Those ABI
+version's aligned with milestones release are therefore called 'ABI major
+versions' and are supported for some number of releases.
+
+More details on major ABI version can be found in the :ref:`ABI versioning
+<major_abi_versions>` guide.
The DPDK ABI policy
-~~~~~~~~~~~~~~~~~~~
+-------------------
+
+A major ABI version is declared every year, aligned with that year's LTS
+release, e.g. v19.11. This ABI version is then supported for one year by all
+subsequent releases within that time period, until the next LTS release, e.g.
+v20.11.
+
+At the declaration of a major ABI version, major version numbers encoded in
+libraries soname's are bumped to indicate the new version, with the minor
+version reset to ``0``. An example would be ``librte_eal.so.20.3`` would become
+``librte_eal.so.21.0``.
-ABI versions are set at the time of major release labeling, and the ABI may
-change multiple times, without warning, between the last release label and the
-HEAD label of the git tree.
+The ABI may then change multiple times, without warning, between the last major
+ABI version increment and the HEAD label of the git tree, with the condition
+that ABI compatibility with the major ABI version is preserved and therefore
+soname's do not change.
-ABI versions, once released, are available until such time as their
-deprecation has been noted in the Release Notes for at least one major release
-cycle. For example consider the case where the ABI for DPDK 2.0 has been
-shipped and then a decision is made to modify it during the development of
-DPDK 2.1. The decision will be recorded in the Release Notes for the DPDK 2.1
-release and the modification will be made available in the DPDK 2.2 release.
+Minor versions are incremented to indicate the release of a new ABI compatible
+DPDK release, typically the DPDK quarterly releases. An example of this, might
+be that ``librte_eal.so.20.1`` would indicate the first ABI compatible DPDK
+release, following the declaration of the new major ABI version ``20``.
-ABI versions may be deprecated in whole or in part as needed by a given
-update.
+ABI versions, are supported by each release until such time as the next major
+ABI version is declared. At that time, the deprecation of the previous major ABI
+version will be noted in the Release Notes with guidance on individual symbol
+depreciation and upgrade notes provided.
-Some ABI changes may be too significant to reasonably maintain multiple
-versions. In those cases ABI's may be updated without backward compatibility
-being provided. The requirements for doing so are:
+.. _figure_abi_stability_policy:
+
+.. figure:: img/abi_stability_policy.*
+
+*Figure 2. Mapping of new ABI versions and ABI version compatibility to DPDK
+releases.*
+
+.. _abi_changes:
+
+ABI Changes
+~~~~~~~~~~~
+
+The ABI may still change after the declaration of a major ABI version, that is
+new APIs may be still added or existing APIs may be modified.
+
+.. Warning::
+
+ Note that, this policy details the method by which the ABI may be changed,
+ with due regard to preserving compatibility and observing depreciation
+ notices. This process however should not be undertaken lightly, as a general
+ rule ABI stability is extremely important for downstream consumers of DPDK.
+ The ABI should only be changed for significant reasons, such as performance
+ enhancements. ABI breakages due to changes such as reorganizing public
+ structure fields for aesthetic or readability purposes should be avoided.
+
+The requirements for changing the ABI are:
#. At least 3 acknowledgments of the need to do so must be made on the
dpdk.org mailing list.
@@ -71,34 +134,119 @@ being provided. The requirements for doing so are:
no maintainer is available for the component, the tree/sub-tree maintainer
for that component must acknowledge the ABI change instead.
+ - The acknowledgment of a member of the technical board, as a delegate of the
+ `technical board <https://core.dpdk.org/techboard/>`_ acknowledging the
+ need for the ABI change, is also mandatory.
+
- It is also recommended that acknowledgments from different "areas of
interest" be sought for each deprecation, for example: from NIC vendors,
CPU vendors, end-users, etc.
-#. The changes (including an alternative map file) can be included with
- deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option,
- to provide more details about oncoming changes.
- ``RTE_NEXT_ABI`` wrapper will be removed when it become the default ABI.
- More preferred way to provide this information is sending the feature
- as a separate patch and reference it in deprecation notice.
+#. Backward compatibility with the major ABI version must be maintained through
+ :ref:`abi_versioning`, with :ref:`forward-only <forward-only>` compatibility
+ offered for any ABI changes that are indicated to be part of the next ABI
+ version.
-#. A full deprecation cycle, as explained above, must be made to offer
- downstream consumers sufficient warning of the change.
+ - In situations where backward compatibility is not possible, read the
+ section on :ref:`abi_breakages`.
-Note that the above process for ABI deprecation should not be undertaken
-lightly. ABI stability is extremely important for downstream consumers of the
-DPDK, especially when distributed in shared object form. Every effort should
-be made to preserve the ABI whenever possible. The ABI should only be changed
-for significant reasons, such as performance enhancements. ABI breakage due to
-changes such as reorganizing public structure fields for aesthetic or
-readability purposes should be avoided.
+ - No backward or forward compatibility is offered for API changes marked as
+ ``experimental``, as described in the section on :ref:`Experimental APIs
+ and Libraries <experimental_apis>`.
-.. note::
+#. If a newly proposed API functionally replaces an existing one, when the new
+ API becomes non-experimental, then the old one is marked with
+ ``__rte_deprecated``.
+
+ - The depreciated API should follow the notification process to be removed,
+ see :ref:`deprecation_notices`.
+
+ - At the declaration of the next major ABI version, those ABI changes then
+ become a formal part of the new ABI and the requirement to preserve ABI
+ compatibility with the last major ABI version is then dropped.
+
+ - The responsibility for removing redundant ABI compatibility code rests
+ with the original contributor of the ABI changes, failing that, then with
+ the contributor's company and then finally with the maintainer.
+
+.. _forward-only:
+
+.. Note::
+
+ Note that forward-only compatibility is offered for those changes made
+ between major ABI versions. As a library's soname can only describe
+ compatibility with the last major ABI version, until the next major ABI
+ version is declared, these changes therefore cannot be resolved as a runtime
+ dependency through the soname. Therefore any application wishing to make use
+ of these ABI changes can only ensure that it's runtime dependencies are met
+ through Operating System package versioning.
+
+.. _hw_rqmts:
+
+.. Note::
Updates to the minimum hardware requirements, which drop support for hardware
which was previously supported, should be treated as an ABI change, and
- follow the relevant deprecation policy procedures as above: 3 acks and
- announcement at least one release in advance.
+ follow the relevant deprecation policy procedures as above: 3 acks, technical
+ board approval and announcement at least one release in advance.
+
+.. _abi_breakages:
+
+ABI Breakages
+~~~~~~~~~~~~~
+
+For those ABI changes that are too significant to reasonably maintain multiple
+symbol versions, there is an amended process. In these cases, ABIs may be
+updated without the requirement of backward compatibility being provided. These
+changes must follow the `same process :ref:`described above <abi_changes>` as non-breaking
+changes, however with the following additional requirements:
+
+#. ABI breaking changes (including an alternative map file) can be included with
+ deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option, to provide
+ more details about oncoming changes. ``RTE_NEXT_ABI`` wrapper will be removed
+ at the declaration of the next major ABI version.
+
+#. Once approved, and after the depreciation notice has been observed these
+ changes will form part of the next declared major ABI version.
+
+Examples of ABI Changes
+~~~~~~~~~~~~~~~~~~~~~~~
+
+The following are examples of allowable ABI changes occurring between
+declarations of major ABI versions.
+
+* DPDK 19.11 release, defines the function ``rte_foo()``, and ``rte_foo()``
+ as part of the major ABI version ``20``.
+
+* DPDK 20.02 release defines a new function ``rte_foo(uint8_t bar)``, and
+ this is not a problem as long as the symbol ``rte_foo@DPDK20`` is
+ preserved through :ref:`abi_versioning`.
+
+ - The new function may be marked with the ``__rte_experimental`` tag for a
+ number of releases, as described in the section :ref:`experimental_apis`.
+
+ - Once ``rte_foo(uint8_t bar)`` becomes non-experimental ``rte_foo()`` is then
+ declared as ``__rte_depreciated``, with an associated deprecation notice
+ provided.
+
+* DPDK 19.11 is not re-released to include ``rte_foo(uint8_t bar)``, the new
+ version of ``rte_foo`` only exists from DPDK 20.02 onwards as described in the
+ :ref:`note on forward-only compatibility<forward-only>`.
+
+* DPDK 20.02 release defines the experimental function ``__rte_experimental
+ rte_baz()``. This function may or may not exist in the DPDK 20.05 release.
+
+* An application ``dPacket`` wishes to use ``rte_foo(uint8_t bar)``, before the
+ declaration of the DPDK ``21`` major API version. The application can only
+ ensure it's runtime dependencies are met by specifying ``DPDK (>= 20.2)`` as
+ an explicit package dependency, as the soname only may only indicate the
+ supported major ABI version.
+
+* At the release of DPDK 20.11, the function ``rte_foo(uint8_t bar)`` becomes
+ formally part of then new major ABI version DPDK 21.0 and ``rte_foo()`` may be
+ removed.
+
+.. _deprecation_notices:
Examples of Deprecation Notices
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -106,46 +254,42 @@ Examples of Deprecation Notices
The following are some examples of ABI deprecation notices which would be
added to the Release Notes:
-* The Macro ``#RTE_FOO`` is deprecated and will be removed with version 2.0,
- to be replaced with the inline function ``rte_foo()``.
+* The Macro ``#RTE_FOO`` is deprecated and will be removed with ABI version
+ 21, to be replaced with the inline function ``rte_foo()``.
* The function ``rte_mbuf_grok()`` has been updated to include a new parameter
- in version 2.0. Backwards compatibility will be maintained for this function
- until the release of version 2.1
+ in version 20.2. Backwards compatibility will be maintained for this function
+ until the release of the new DPDK major ABI version 21, in DPDK version
+ 20.11.
-* The members of ``struct rte_foo`` have been reorganized in release 2.0 for
+* The members of ``struct rte_foo`` have been reorganized in DPDK 20.02 for
performance reasons. Existing binary applications will have backwards
- compatibility in release 2.0, while newly built binaries will need to
- reference the new structure variant ``struct rte_foo2``. Compatibility will
- be removed in release 2.2, and all applications will require updating and
+ compatibility in release 20.02, while newly built binaries will need to
+ reference the new structure variant ``struct rte_foo2``. Compatibility will be
+ removed in release 20.11, and all applications will require updating and
rebuilding to the new structure at that time, which will be renamed to the
original ``struct rte_foo``.
* Significant ABI changes are planned for the ``librte_dostuff`` library. The
- upcoming release 2.0 will not contain these changes, but release 2.1 will,
+ upcoming release 20.02 will not contain these changes, but release 20.11 will,
and no backwards compatibility is planned due to the extensive nature of
- these changes. Binaries using this library built prior to version 2.1 will
+ these changes. Binaries using this library built prior to ABI version 21 will
require updating and recompilation.
-New API replacing previous one
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-If a new API proposed functionally replaces an existing one, when the new API
-becomes non-experimental then the old one is marked with ``__rte_deprecated``.
-Deprecated APIs are removed completely just after the next LTS.
+.. _experimental_apis:
-Reminder that old API should follow deprecation process to be removed.
+Experimental
+------------
+APIs
+~~~~
-Experimental APIs
------------------
-
-APIs marked as ``experimental`` are not considered part of the ABI and may
-change without warning at any time. Since changes to APIs are most likely
-immediately after their introduction, as users begin to take advantage of
-those new APIs and start finding issues with them, new DPDK APIs will be
-automatically marked as ``experimental`` to allow for a period of stabilization
-before they become part of a tracked ABI.
+APIs marked as ``experimental`` are not considered part of an ABI version and
+may change without warning at any time. Since changes to APIs are most likely
+immediately after their introduction, as users begin to take advantage of those
+new APIs and start finding issues with them, new DPDK APIs will be automatically
+marked as ``experimental`` to allow for a period of stabilization before they
+become part of a tracked ABI version.
Note that marking an API as experimental is a multi step process.
To mark an API as experimental, the symbols which are desired to be exported
@@ -163,7 +307,16 @@ In addition to tagging the code with ``__rte_experimental``,
the doxygen markup must also contain the EXPERIMENTAL string,
and the MAINTAINERS file should note the EXPERIMENTAL libraries.
-For removing the experimental tag associated with an API, deprecation notice
-is not required. Though, an API should remain in experimental state for at least
-one release. Thereafter, normal process of posting patch for review to mailing
-list can be followed.
+For removing the experimental tag associated with an API, deprecation notice is
+not required. Though, an API should remain in experimental state for at least
+one release. Thereafter, the normal process of posting patch for review to
+mailing list can be followed.
+
+Libraries
+~~~~~~~~~
+
+Libraries marked as ``experimental`` are entirely not considered part of an ABI
+version, and may change without warning at any time. Experimental libraries
+always have a major version of ``0`` to indicate they exist outside of
+:ref:`abi_versioning` , with the minor version incremented with each ABI change
+to library.
diff --git a/doc/guides/contributing/img/abi_stability_policy.png b/doc/guides/contributing/img/abi_stability_policy.png
new file mode 100644
index 0000000000000000000000000000000000000000..af9e8e0b559a8724ee0fa83f08f6a1578316739f
GIT binary patch
literal 61277
zcmd42d0dj|_djgYOirI%hMLJWoxohm-1kJs2{lVID^pT2Q#AL~Pyv-18>KYKC3ms1
z(sB*=1+6qUR5BOD1<MTsm6VVa!QXAo_w&s2`{#Ln|2(gU*Q>yr>%KYXT<1FHT<`O~
z?wmVoEBPP!|A>i+N!p!)ofi`mFB22{Vf*Jjz?Js|(__Hfu88xte~MKMD1HL|_|ebG
z!AeZ*C1Kx|*H6ITd#|5@M~I0@weS4f)fHOeBPO=8W(Tvn7>!sGN`CQif&b2OF+7C#
zdG*Tu*DulsWPfXT<?!=h9_6PaI+_hyuWu*CKfH2z9s>T++DWnNUHtw(&i&P!D0Sl|
zB}-5BjmOW&ie7_SK<W2SHQcN=aLSbT+LOKS_5GC_!s+@Kk$(J7o1%(MDD`qE?YUW7
z1S`LqN_Qqtj|*x5+5Y`nPmbIM?Odpv_Gd<IUJw&|Wj1-^(?!$8SfTB|2l}Q#%Xp+c
zVmC+fR+}4?M*sP)?C;x#|9lKQ>?QgA<EzW3Tz7x}Saop!|K~#?<|6U9ndD><pv{$A
zDrLI>QLfxRTa0qD+13yftNP=+NII}KKVG982EJSD)r{RYfnWWCq6UzG8?LPWbKA`q
zQK&~*fQp*g-xZb6DY30Z06JB<et*n6ornK3<o@>|2FWbD@^AN~e7|tf=1YavmmTB%
zlv=gpK{tC<pe5g|`S$+z3$1%GE;8Rl`Qf|m{+Kp^76U_W@)m0@tJ&t`YDSt&W_oS5
zFdf{hBC-G{71Z$WscRw_dzhh`UDmT#Zd@*0v@AH)JcarCTN^)tz3I5?<`J{;fPzEP
zGB)`N%3su?JQWYO{u-k$TImO6@jZXY^zY=%T)>5i@5pw~(Du#dv^h^rD=E<~VRxm6
zsBoH}z}*~rRe|2VpfqfTF<}2KZj(P6d%V`oxIQJ)Lg7MI{eWP%<>gyPD2(?YnXW{^
zhA`{ddq?-}iyeNO+Pf&fe{+pjL}PCVU6Wk8k6h*@?y)Y^zMCXy)a5;7?1N1GiW3k=
z2a75L3i#rlD{Xr;y9Y0^@2q}a`Fh2&mfssRv+qs$%ana!t>v~KHOG^u-%hp5?}!Tx
zB?_FHvkaH+Tbgi|M(3GvO(P-O81p<8c){7EyC0a|Iga&OmK<?G<*t42KHZB6r?>R5
zer0zb?vS1rmSZc$^8S4D{Yl5ME?w<4CvVXPmMeFU`DVp#Q!W<_Y#+FguKFTxWhwy0
zm}=UZJSk+pQhedV8^by~LPq(WMdhPOn~t|6|3}9<ywZCNygjpdYtl2cb|R@Mg-YOg
zj5e*=9?bQLCqEeqC^-1CPZ{SPQ!`@LY$;hrzc`z?vs_-;<gK=%<G1pPMX*p0$?c8U
z(09-FG^?bKP(B5t(d<bImF>9ug2{M*&~nwQdH8mCO!xG!oYfC_H-YvpZn$(H@DMSv
z>^I2i?d-2n@RPd>71+j0I`$fgY(yl<>@C(udnHNWc*`*Do70J1;cT?wJ}}B71ye#l
z%3z4+WYdyhYBo-PnGlVGt?!9}x4zo6_L@7lX-UJ@?WZR3TcU9Cxx)vJfxj>HSB!|c
z>#A=v3rft;xA>h21vLL!A`Rj7gDHbO7fp3m?Y_0^pM}JUzStoHAr-%Kfvo(dJ8%HJ
z#u0y83}NgKyWF8;DW8hNI&^q&|JLoD#zAMA*Z$s{fmEF;cJ3j<D+ym+JCe9R8Nt>o
zLV(9(NjFRlzM5g+gywaGLMpfT&O+0QLMk)*xr^S~cr3P~5j3PeaYCI`(eq@cq$y$Q
zGep?sy!l!*`(n7T6iv1lY@G8v7H@GZYNOf~`m*SM1T@i}Q6l$@YJdoAhm<en!Y(VN
zUsLaO72eA>Xfz*q0P?)s5YfS3*yruj9%Gm60b2c=F=x~5!;rQ$*PD?8rK>E#|3+|F
zikN7_$m%SVNNiyfF9XZK9GR~b^x^q7nlBLDO$!e|a9WvJWB1(~m&-3ql#lk}CR3qJ
zvt*ZzRvt3|p)C$k6~1&HbwkCbQ7j)Cqx%rTB`3!q-M!x?cT9nlDtHd6hbi{TyrJ6I
zkKd0ChN(7AIGgJ}R>_){eTlbOrCP_-7}GK5I~q{F?J=qYAxRA;o@2jn%Q~&kOG8>m
z^L%#OPY9wAbjs$i=*zQiViu2=pL(aV<=M~>B$ulUN@a$Ps&SILVLUa6lDv~>!40g1
z??%w7uZ<e2Sr4Q7amiy(Qh#;LclOMxHSfB#tPesXm>Z(47_T34Znk=MBtk@5p0Zc8
zW7bR5B9p(zz$HmPYad9|`k!%Vm=Vi2O~d;V{fbDR+|x*NR=m^DVr)8SM!nPhgf1x@
zQ>nKxHw7yU;dDK(>~7iCpzjMkbh6Xp*>=Oe)9*EL>T9#0!n^8v@bzbS0W8McGyB5m
zUN~*8HshJy%1djr^3m<_LifgK4(F3&#d+1VGohdC;vPElD+WLk3G}2wDC^E~weJp$
zJt-wlB&9ZgxjKM2l`hc?Jq^7b72*0G?qH@F4ORRq4*qh*w`C&K)7G)~Qt1@J_Rk=?
zl<nkLo_kX%B!Q=nG%n*Z+SwS+8am1sW>+V5Auc+ToHSjief<YsN9Ol+c5^Cf8^0T5
zs`ypedYGK8_GzRxAV=(2dnRGL6=U*QdSRa=;ojGCu;RM}n1p<!QTlX_Sa(|w)^njr
zKlNND77k0r30m+$vf5nBL#h1b@)34fR;YYUL1Sw@OycxogL=clHZVBxiWa2}I&&Cg
zupC%fD#N4;yFAj`Kl=(v8qXRR)C`x+xR^c!v%TVz7mxmgcRE;3(0rDyXoU0N6!eVf
z#qftV3Gs6qt7!}$zWC)0w1vG;Zg6YqIkTU;q03>8mQcr#7aW7GK>x|}&6N^)1?{(R
zhdqBD`ofZbTGp)6E;COO&6aq*PLvc!$ah9yiNB=oUA9a*MW-0jpfAF<CcqVhlEmv4
z<n3&>NQiIoi(*iXIaU}4n3R6Oy{#dGCT0^t*{QBNlPws;Q*C@^MLLb;g@kB-m{Jm0
zp|V<r5rrxYOFYYF2%kH<(F9s2q5pYPXj2AWb;4vlYgq~4BDnE=qya_JMkORp?J+`?
z;N1AE^6oW;ThurP38=hxH1b{zjnCNFO>*I@Hbd@8lvY!uYu2jQ>(~DNDSMvl?NsR2
zZsE;*N^EGqG;-j3mam#1y(X!ov#^Lvuh&B*2#+EkYA@}JYBS-8T<gzBSE^V0Uo`$&
zkd(OS1%4@~;9fO7Z(R$tjfF9&)e_<b6sf>-RBQ}Ws9pG5D#b_pec|0x^fN}Z0=szo
zCzd8)g>G)BR!X)_2y-7My}aIL4!a;gnrl5)`ElDDR{gO_ePWX<>>?46bbHu!^S$hi
zdG(*?MLaYSIWwybIr!kl3*@0p=zsKA)534y`Cn8CIs9B@pHWmC^szeunS-^NJAPoT
zDaIAXWi$<q9;1v+JKX1OA`DizT0$QfH_?xq3uvBN=oW9`?e~;uGWajeSqM0t@~kBB
zCtOTNRl+<Nd)4pcM%QL)d!onzX7o@w+4Y8Yfg6ND%Ud#PFiPywB$tp5*C|Q0NZ^Q-
zmS!`vpjv6$t;?a|K95DVV8&qHl2U0Npe1L~a^dCIi=m`r+Y2&aRj3s5#08}`<)o7I
z6`W!!w1o80Ora)El4eb(24!nem=>8nGb+etT3O)Yw8xoFB<Y1p{drZ_ilpuio;kFl
zx}s<Eg@i{Qzh^+{Zy+3CMdx6_PPCZdQIs$F0SXoiJ;*DU%Dp4>XFWhJah^3aZa0L7
zJ&tX(Y?MMX=9quTe?e!Vx-LLSeZ@krVfCT7I89c>F-bvkK?Y_<>V%Wy_sZ;5h{DLR
zQVDjRB!#XYkZ*e_m90ri94NtR4IIS=KeKAjETrN3d{9%roCwPPd>PKE@(D5OFvj}+
zh8w~4A+B>(=2?%16D2ujJ>(XB^k5MFEb>G()7i*9BGLX}m2A$Jug~zlGiH@9t!aDP
z_7B?Me`@W{b)dNV9kqM5_0l0W=-G3Z%-Z0*D+PA5sMaWXR9=c7Y|d2wMW3JUGPCQk
zMZHaI=fD~0U^R+&`=97r@&4Gck@j0MLlXOl{OZ1rNU0<688<LOHg%1d#94CCvX@ge
z9GNf<Xj6KR3tIoVvPeSCEUBf^N6HEz9dCYSrYXpe!PZyUPSlX3Nhf#DT0U<hsPBcP
z7zaDHTEl^b=(fH>=|D;O<yppToPy=no-JYLVoD}(GP>7(j7Pe2xfS|}YH>jwc@P$|
z_xP{5J4$$Tl@FO!rlLFx+bsxOt`_ds%s9ixIe3o*uR>*tXfihebG3`L_n+M7fgD@P
zkYx0C_FCTQ{`ls?`q)MVW3EG_J7Wwc8DHp47_w&Ud#R*ayRz%z`Z0w*w%0%&OX(xZ
zr?M#H-pWw#+~kFwM7BrxKF@$-yAc5zT37GCy>)c|FOyy#Pz~Bte8g|zr#`e?(tn=j
z``je}Su%0HYYvLCEf0OiBkEEOEw9DOwU2Jx##3#5roQCkttbZlmn(<a72ZUlM=@JK
zVBc@2S6(3Qu3T|Yikh<DMAx6rTW@?y9J_`2USo<yy2u%wfT~};7>`^Z>#<pbXFRgM
z&`JeM%Gb3lxva|vNJStdSRh7n3vHol-nPeP*4TO&dYb!{tao)>>1qFAOL$$RKpzNx
z6C+Ba4yc*ql}X}XRGDiPc>A>ZO+7ioZoP^Pe_D>E*ycIdXek=UP(0fxnn*jR-~+_J
z9Mt6Hjy;}?Z|Tu;(Xrx!ob!_yB|quOEZCDJRI>)M@p+*e>&}5-hyRB8=SoI`PoQ{t
zjWb)9Ve*k}Jz=C`o(m>@A5BpeHL1famAzY&35bYDeR>Q1Ug>Fi2=mFwD!u1Lwh>=h
zBkUVShY!~U)LU;kgobOi+qj*gG%Wae_-*JqsNNyU=&}jF+5pR`V)l}%MREwM>|^4g
zRU$fW{yK$ysgP-TAoWg%eaHgp%WFJp;tTaSuj3%?e?oE|5v2P`=dtM%AIvq&XU@{l
z)<s4sJ+I_T6J^h0(EUTTp79u;%0H(lG5dE`-f~P_=-Q8THtUa2(uoezqjs5(!x!{*
zYg0ZpQpg4TztZlTDu#K7hKKNKJv{XRDVZM3s>A7Lz@g{kzeNmHxUDu@x)$3r;BYQ0
zbkv!?&p3TVnc!L<{B`mLp%8QRrJ(uZKj{J+E<4y#cthSWDAGy#q*;(`ztvhSQEpYV
zQjG|Gq(?r5YG>ns8m{W2lfG$W2~e-O!AzxU1+W${xIR|Cv8_#Ju*JovRTK-#am_Zc
z%DRIOFPv^^`>Jg_%<dDoAr7Rn`(`gExcVZPEz6^qnolUQk9yF|IMR{GQ=b**KZHUU
zN~tuLSVm?bySSNp@I-PLbo@`Kd<Wnja2gzAXk>f2+iZphnr|3>R_>As?<)6b9>F-b
zb#Z$;u$hR)yoVOlB&(u3W$i{WA<xw?brr;a_Nr46(|AQ21BdtT3VLqwd8&@GHq|Ej
zV60)(i|)=r`^(6apVT14-(Ds&F81%)3TrPCHRHoQyOyBDiOrXBb6+fBK37Gaw)Si-
z5W2G4{9s{9H)<iQ+h*a*EKo%+l9*rH?x~YQBJ$X?ZY!ZVUA#LWjEnaB!GY4v3#Egf
z%JdH#Esh^g?Rjdpp?l9he2(W_+3fE2sT`!ucF=;whAvGOs{xrkGpwIV8rW27!YQSq
zV4~mE*+4nI1@HKMC;yerj=q=*m{OER6BrNh@hEf)6D2I((Rqjz_p?zFFs0`mC%sDl
z8+@Q>$MztD0h|6XAd1qnbI#uFJ`7E2<5)^@)j+0O96IPJ?cHNTh0mdjT{<~8sU29U
zc=(FTR(IUo{do@F1TupJ4RtR^pYHexl(UFE2dmIOu-G`Dcvc7nr5~NhkredrVZ{(F
z#kyykN|avw$`o3umRvIY^0~kKf};nDu!8L68sUE7R)hMvb?*Dnu@3|<w9c!dPLnh_
z{+^n%xrNArMtfD9R!X<iVNy!B;Fm0n9@f^V|9qvUH_q5*J$mKKe%hJeIAMV;G)G=x
zNZO|Pbj^fSMnq`3W&Q90sGGB6NA1r$8^{wgHQeg1Q>Cb(rxEp7;s3TB{XpVa76ChK
zOWg-2-L)NyBOam{%I@R1J@&s_+*CdwjXr@#n*E}@SmEIMM6NA~{XaX!baL5=Gs^N?
z=X(OO5zVxdk^Uz?Ma_(7lqK~4Q&7HQ++Zlrohv~uj#&%r*$QpCiJ?FFz6$~17XMlV
z4cDaea)95_<dIip^?lDKHr8xb(Bik~Ojg?$aiNUAeXW37*OI8NQxr?a+H02;qt?q(
zP!{B=<~HrB!~eZ!KoGB8-@2+mz4Kg}hsbu5G8BDw<v8qY>o=k0+XQ@0<#X3{%Ifti
zo-odDSJM9hv#RQmj*FpNolE}0_jo!MQ8y79fHGs=NdNHa%~D{k#c)Wi(j*`#-O<8P
zMYKMOH-1av6RuMO#?E~Y3S#%|*b&+|Wa$w4x_NU8f2N}%pmt*0jV;h_{(~f+2}xXK
z&;rN?GfJypTF#cD{5$rxZ*&FHtXtH;iK159E#XhAi%qvBRll25%vx$bc4eMfyIy#s
zFgnr-CiNRb@$T0JJfX&{pzv-3ZYx%`R3=tA=n6iv6t&UH_+@(A&Ay(ErzrtHfaiBa
z@Lwbx|AlsRXU>=KLj#-}%Arj+Rw65L+~LB-^3so%Z5+w*VKK3XJMfTTldv-ocvYSM
zM;|e<|5Nxcf6)n4fUmTfJm%^8QV8M@x`U(PtyPdqvF=ekle2+SOfFdCg4mU(X5|BZ
zo9R)DH6hE60DiacJGdei@4>*eoNVS2MZW@ahyijWamVm9+ZbIib{iQ3ojDGQx+kla
zwS5i-H=2cjm^OXKu&$ZQxiwky2f(c0=d(kgM$ckv32qT+qn-w{Cv9tB9T#-Bz99^*
z`<dgc<-fg&Jm2SmI2}(}2%(1yu1G0ikCh3O_rLskDC9yg5t$TL_*3OQbph1hfQFjJ
zTNSp25?nj{)Wxu4sTW!!tA<!DdKQS*eCq7U1&2iG>5bwh{T&R_sG|pj#E*NW+RZri
zB)Y&PE?0CQLELMqKdRc9XQ;H7bl-jzWi6q1=FO%EbmXH>e|VDbsn7FcPY%B}o_BZq
z+d*!B45%ySLghXCY=m6ug!5H*sGBk&dF$}Gf`)-j)Tl~WE;8>fIy3lk<eMgu)#yW6
zp6O4oD$JI*&Kd(q*+}kp<Th#bAwCM04;ozQvIEK3KDhuU71JJeZU*(?{6k5+V}ND)
z=c^OP%6f9_ODqj%GVv!j{`^YE+ofC6Y%+q6u6E$HttU-q?t!XRh<i4=@Pu_$H=OKh
zEki{!KO&gdd5G&^SLVph0nu#Dp9zBW;M#uLEG*3Dkl-C&+&2HWRMfUOm=qW5)}h%m
z3a@*f?MH|EgbFsfM`m|{HssQYN`v{^c)U4o&7@TyOqf-JteIrCZDiv2f%S5}W6an~
zQwrD5X;%gXZBO9yE!7WuXj=yEFIKrl%8L9nXlzHQ4kOR>Zh5hDpC<C2xkk;{OS9jO
zjm(;q2iyJVNGiy?^z12-(mkQ#^QPv@`k@gB+FfY6Qtg!iuh;0WUec+KIXFzFGEGy&
zmYgvFHv9^+=p|W|ngVlY<S8!7f%r+?3~Jj0ZG*WrpxBGVP9Y(pngO<DtMN<(h~BP?
zU5O3tFb{7SdrvXLw(nZ9Hf^#|jl^k!*=u$gZm<}gnOSnr?4<l^E5nCv2O1JB`eg<m
zU4Tb=e1jK$1l;Ay1Q2*8AAK3lt%;g$;XgQg#DA5J_qubdrTEjdbzhp#z>oY}cy~Sk
zD(w9NPy3x>RKfQdz#(9<g|rq<pp@5%a#F?nByC9I;vrP!>bN(=;#a54;}quS<@PzO
z#o?=rWh!qg8pahyc+ii9pKBcLGQw&aahy&pMpK&zTt%9fFP-Vcn$;~6XFn8%+Drx3
zpp&_1Mq#VPL3H5Fu&%39U9pp=B+?a|t;kOkkGrrKmdh6GslPsOK^vDlONfo}>)14J
z_q^g2Twq5v*kUBsD94>v&sViQ%8t1LQ%bk%Cpo+|r&J7;SuXytqw?<WD$5RZyqaD6
zdzW5M)}8DOXr`Leezkt3E2#J21Mevfi7TccbSU*0xb?pFZBn5h02X;>JHeyu613fv
zDyIqESYM{_%ueZ8)`E*xElD^su&3u^BsY7mDO~w3=T5`n8Rsd#_y--%D-o)Np-Eq>
z@fj39y});M8T>w&PYBMb4)<R9k2#FUY$giK;#2kdQOLFB7kH<WmN+*USE;s`uRPQu
zFrkDULV^V5<?enF+gxG5#yAQTUomCgnRBCGFhuJD<MQwGHI*LxjNpe+u@1N+SH}oG
zEX?&WY0utnDb$pH-?Xg(ArvvaF<#=G4>=(*&T;CBzK<LcayLswMLtW{d8d>&;uD=L
zS%7VJka?Fh>wcvhwRV`95xvxlD<{X@{h*;2NvO1C`6q~qR`JCr>|+A6k|^&_w@d>j
zY>!*olY{$tqfjF@BfZgn{pul<L>>T{vU`=Dpg0Rl^virRrHcbRQvYpNRXGZ&Ur-^1
zrM@3(L`;{=-e)?flH#<k=7v9~(I->#qb`S%0tW>qJ>Qk?Z^0Ds;-17<1S#)1G>ggO
zjy>Q>HLV+cYRE!KHN{bCJs8!E<)|s6vK4<Nu0e@Q<WLKs9HyW>GpA;yl3L(vI?}s(
zsJ1oS9Jt@#zR4lJuR2JV;`iEs<5ozJ!^t^qi;3SPcSCpDG0+$paaX*P-JP(WOe>os
z4>Jp@v?N9G>&UdZGX+0>b^Yzgo%^~0&3<<0lg&Dju|taSRxDWe!{y5QN9dSdQqlfX
zPy5)*idiu|{4Fw`Gyor{z4}g23`kQ4Iet~S7+S&2^(oAoFbH6jMARcH&=PEPd&+cG
zVCYbapvuRe+`7M?e<(9r3Dq}LvzP5pSXD2|l&hbr4WnV4)(s()YBX`=<T}AjO;z`k
zk)$4#^tOcUpY1<gFM9On9ea0nm}HfUzshd0o4jlw2p@jED}`TyS0tWDOo)wZJ*RQ<
z&4LbDxlg9EyG{5BhzK)y0$<f-K@Ad~J!y4v%@^!5`Ab8s?IkB^GS;a>lP^uQmQylL
zR41zKb&^Jk+&2C^$zTdb$UO%#kP+GQ$yTTJR#!nsRt-bGeu<$f<6S#U&#yqSZFsNw
zaGe<_Fi!KoTDt^~8CnKqGaXK>w67Et_Dn31b9o;g#T+(B{Hf5_<TKhHn`{gBnX<e?
zu$fygE~ApREJg`ie-=)hm!?y%DW?E%g_=#qqj}|nskFWq7DYS0nw@&v@YQO_y=c$S
zwc9FvDtn)T(^cpY?RoOWtOq3us@PD=T4U4CaixVx-M8&9m$$>RCq?!%HKkuO@iI0~
zmZ1Cr#@zXN=v_+yL9DU=vu_+*A)zLs<L&o56lv!1w+DRO?Q}+*ddWsqtW^4)cEz=g
zeh^R4rg~K=uZG}28D~CQi%&7%v+dV<4t)tWDw)%=l^5f68r`;#cCIvdJWTnL4Dq*A
zONnl?s^R0(%eL|^{IWMy-sZiJSt9?fKVhOP>gnarx75`+(?TxDG;~}zMp^5ULu+#6
zQoB|lAb;mVryWiEPQ8);0DCIqU2wYx2DeKQ1$59b>AR66vqX#jA8l2!%7=6pIn`mp
zO#rLlPiW#S&)OY*y7beOeT>i!wk~T=$A*u8^xfV;G2+SuWl=%ChqNNkqg#mk!>nMc
zVW&Bbids?c$oI0!S5OGsOl<FuS1+vRDxy~Gk0!PWV_@r}ac4Uo@;z^3$f)G;E+l-!
zqF4cPn)9*VcFr`D<{)L7Pc|IVvvnxpm!~)@^Q}-`pP&VnN<Ek-^nK04&>VARgWOlD
zdG#xWZ<TF*DnD$BTS=!xJHtfkV9M5Y7%|;mix8LsyU+B`osW<LRKEEe(8e?weE8@(
zHKiL7HJhemdLw9^)eDp`KB(RwmvGAtDT=)f60h?}%qf-A=QVZ!kF`?MIgY_;$Kec4
zbr8Cb@v9U6A!Qkd>^O>jC&@xFl5JrIi6;>q_eVBP#!FlKY>**f-0x0pp&OvrExuD-
zNY@by8KlX!qm9pu8TDm&zfYP`Wn~y&k2s=wi0|v+igC<(^irI68ZmPH14YMis!d`d
zM$_0T4v{hFK(beUm-j8H4gnb^Up-w1O_(-t*sq#yYT6g6dPINl0ePX5*LjT3h`j<+
zeZEHndQk1qUkyrWd;t2`k2n%{9Bi2Qs^H>!jp%e}_&iD6Lu$k=?Lqqb!@Rj{X?-?{
z+v0b@R7U^hMhiZ#iF(lQ!1_ynx?u7mgYz56H2->R|6J_f+631c2lJKw+h6!FACHUt
zujSYyejc?eA73$b0|9ySbZR!Wa!^0xcKqtnA8+KZRK1Ugey?J$DkIVXr<>9bXk)j}
zrcJ7I((gQDGWSW+n>UY;d+u-wH<R%sAPWB|CI3T|Jn!9X`nm{F)0D+m5xv;X?lPx5
z*PyJ;1FL>`-tZdTtCQ@P8KG)hEO#k(?DNb#v9tQ}s6~odMLO_CBWbphjp^hzK?`(7
zt@e2rxdPHFTw%`zPaQMG+3WNZ0uxL%z}UgclNyOA)el_7tp|AX%nisrE54SW-n_VY
zg7Z!rdF%>MoX5v%fw5=&_Nyj}DLpXho_oWw@u5=e_xn9Yz9&Y{$G$%UA=2}fdOyC!
zS%R^fPwiF;b0^NP)SKB6UYXG|IF`?23MWUy%`*!w_V5K{`~vv}m&Zx!ew3@`aLWBR
zr<K6XI{ap~rO}Ksn4~HGoD-Z+$@KPUc7&-GvUw>OswPvj)Q*7r<TSD5MC=044$};t
zdFD&PYzo_i!eCyOg1BSOF<Y62Q=e=bfy8B9$UT_T$LN?Pvdm`wX#Ir+sJ-um?Tu6E
zqK0RmI1oK}KF2v>Za_~|$Cx{9e}PYhR0=$k)$6h9Yoq^xE6<r0+}UpN+kS|#q{f@8
zZR^b-Hr^;@SayD$z_(CixEM{^>E{ZF1221()^;XYtm+rjU$5I;L`pRK1v5BxqGqSn
z5xj5sVTGw`m+6OX+Pa^F;rejVTfDpPdqz{`z3#Tq5Vwf!L~B+1J-qA3DV_&({jX}L
zU&xW|ooB^Xcv8RKfT>DRxXJz)HCoqmxtxrcNSNxMs>drQGZ)qkC_%$m-<Bhf6^p&P
z4nI(FfNK$~Rez&!Nr(DpEdmN0C}Uk=U8oG?61M@3uKbekk~S>YADcHXo8_Is4wmw2
z^Y0?7qhOs6Q3mW+UIL4iuftq+34sfw_WkLjLjU)hZRp9)<Hi9KY;T=Kj_=en11hQA
zehvP<C6g6_ah+n6$s|~hmWwSJ78gu^0=N)Wm-XXhXzlV>8ltRysBbc$Qx-sXtzqfW
z*RS{`yR3Wb)~(DF;P7kXPJ7cYGlopD3XXb;(M}0l!BXvc<U_VSj6~B(7J$BpFOhsI
zX~nP%?+o%U%5qN1MSPh3QEas@wR3vWqWNS6yH7N$^Ry-W>%25`-{JZrkVI=Ll8V2U
z32$W@Te~Z!l20YIOrw>$T4eYQN?$alw>@n$cylkKVK5EC(%%-9==HGCS+m}w>y4Sr
zFA)%uUogUP8*8~Dm&$)RFq<}!D!7=i<ary&qt?$M=hsF3n@=xcyo&R)iVpWZc4zY9
z<_w9bRTWz2W5986w%cKs>H4j3X0c_`*A2Wm$AYLWalh;Nmn+*ruuNR=Xz*#SW^(|V
z?DLn<53<%U((88;p$)I!sgnV&z$JCEQ5SFEI5~ywqa`Mm_s356-&K02T3X2Dx02mD
zO0bWa_xq+9c|ji2xSiN1c4g1b#$C<q$V@$iC15?Xzr;K@;CNTo*xg{~YccP+SlL}-
z4eiIkh;d+53T$z@GXO@%S<2{PDHF2!qIUofCimkjpdcfMG~IZiShK?e>Jt17?CQ2`
zI<mk@zpi%U?JxPQZ89NBvWMaS@PWQP{QtNAEhPXvnwv+OK7Tn`yb@Tiy;5^o`fjl&
zBdU1d3~o7hcj2f09Yz-r(((WZDRb#D$MnY9_E+WQW8P~^jAI`4k!A(rp5ek<d7IRo
zZN8Y;dj1Z1DsztFxuW`l?SnEyPq(f|2!_6WDTYl0N~5)cQJneN;-RJ}-2QKuuQ>ig
z$^fS6IPg0+oyU|{MfkHFl(oNq1NtlP1E5F21OdRFc%W_XMMV15rtsbCJSK}ho)LWp
z7s_`Ne+fguDj>7D_-323B(va9dJR_^a0&3Y?#aW<o{!Sj=HQt+y!}Co8s(~>0ZTw5
zZ{U}%=n?Kk#{%Sx(~g8n+m<6&a~Vo>w<l$@;@qEZ9?}hk=1~0pdaRkZnu?Er+<e=E
z;GxUoG-uudHRk}%E|i2qcX)sYSe9|-S0~qW2w}|Wy74(`_tvdH=oNOhI^Ekt!$2RU
z7!uxgHc58D7P1z~52vD08e)G~1H2oX6Q<zA)50PEo1KI_Zis*x)x}uM9Nu{IGyh(L
z9#|olg%S&DwjNHe#COkE?`m0xg0gtKU6Z%YuC;>*$<a12eRzgl#t=5WoWHIdIGdS>
zPCl_+*KZVKLzHo<V`t#Ic&fIWqoUd0rSEnNH-kE&=J!cF(fUcO1gHcw;aJc5)R`Zb
zFR@2ODtX!p9t=>J4eVDB<Fj`gOu)$YfeA>0p`N+^%qG6;M&Yg{BAV3`G#vLYoJ2fF
zaIF36mUo9OAcAG`*UO=6*#g+)ad6^QQC`d=ugSF~Xc7S2H*iB0g4g_fM4r5V5jb+q
z$wrTZF5FSGiO~QbSPM-aOS&^=SF!gW^IYjMvvuRII4Uu(6>}Yhc1CWZst+f?K;de~
zoH^B@Z5{Ja5cD;cM@rRDcu1nXaf2uwOqDbdYraav=qdDSvmENAY__h#{Hubsw)H_=
z)jw}C3<Yy|d(C-6<8pLXBH3ri5)#qC6Ii>0R@d-bxZa1sIh~1+aP0dQ_BfJtUhL*w
zpoQ3{rEqtRftmFo+zl$(c!^&ki&eQKm#Rhl+!v%yZCV9d=Lzm2VE+*kN)qr@cU@rz
zzwz?{xE0}Fc~om3!t|1ykPPB>yB&Uj7?tjp0i66-z%SSDE`2f&ajk3dLomFxwnLow
zXZjH%^rD#yQ+`RQ6F@Ch3JG`9w>Xq~l28oi*TyZ23#eQo+H<LGorkZ74YzO3lfST*
zzn<!75TJ`m(#WZFEx{PRMLHU;3gJWR*)@3odnYo7dU?X=j;dfq1C^`XhAcrr7E)Yt
z)nw6RZk2NQaIqo9NTxc|JJ*kHK+(<SLv3^BO&0WuDjJg&A9)S09}h1ab#lyH?i096
z39K9ug+Nc!VK>H9i`9e3;GTn;#zN^aoS4>Ry~5hb(;M~?#9_fj<>qjYIh{UJJ$pn1
zj)6N4;af_&wfYaSK6&+GQ%-cL(K%l2Vp<Ob>QbYJzIj<x4~c4iUY1BB|z;UQ=!{
zNFy;}K%kq|)~I@Qn2#TS!t{KZER;iLSwNEPuPfubQA{X_{o`gjSoL_bGi7$N%un2-
zx)x2%im~X4s`0P5vkA>*PM2{CZOZGjeY)5D*`H&k?SLs26C3^bW=NX6WbNlZ?r+n|
z*gkLxqTthu`<y3<c&maN`ZA8+5;$$sEBn`!YMXX}Aqu-#Hn+?xeWz%cDN{|4oayAC
zYa4ZV%n6pd<cnI&6usfR@h?W+?~Ishl?2%@{3C0zd6ZGki5G{}z2Cjew~?CoS`R{Q
zOcDv!!(FS|kX&X@!z{u$+Ni#<#a7543gVYd<cHIp=3VOo-_`0;N!3d6XhuXn@<Hq!
zcTaBa=_E$6GcBDbA_R<85;!W%D6XQDBLlS2+K*)OgO#T3QsxaZQRSM|L`6R1h_41W
zgI)M^@_iZV53(cgW%rLZT0{0mbAfWxGN}p2J^L>2Nvar99h@9>Vt#E9o=sON<Ho%%
zTCZ+gL1#KNZndD=tJqrU32-f5%VvI$&xa?^Ot&*@EsK_V{d~$*T-ZFvCTS$y%ks2r
z)(7SKEuC4`orRZ|M>o!cv&z})-%Ow2!8Z-f<vSLQx+R}>rBSl@-ND*aeVgFW&XOho
zy}pUMU&WB+;T|U%ZYBr1`oB=XcN&KOR&C%j!s`5cQ-;@w+b9*7SF&OYo`L+&&B8XU
zV+OERupAe#f=BOW^{r=16AwEdFErt_fZ$Aa?|rE_{Q0~*SC7TF-(YXAwf2)rr|Z%E
z4&j=`smyjy>tTWO@I1qv0~nsasov%FPPcc~b5!?JqaoX<7~{qw9)K&MPR=}J`#Vw2
zMo286sbL4<>pU|Xj|={Tf{UTlqpThbql6utw*2C`+DQ6*=s_pP6f6hbYu<l3#DCQ6
zA*0v|>Dqhsov!_CW?`*_?^Ge$J|cxM!MRu&I;KnQj?MqFNdy)d+tg?=Hqj%heI0vn
zcA(99iTXQlZDLCfK1>;*&3wbK7WHpTMs*tmY&a#19C+Fl!)<}jZRqt!)!2P=>n?vH
zU%8fXb$ETGEBd`zOzwXzs$L|Ys=7gl9V*%GlfGd5!G7EoaZI37VfDg5UShn?OK`c(
zBBXJF^cL9ahfMjq&W@_wL$r&0FE*aVXv#dz*DjKOpqEe}gH*OH2yJJ0UuqvUuCQmt
z1`zipxkiqA>rk;;Z&Z+$L4^q(7bxS!&ZFs+fMgd(DWj#(gyq2Vj#~A7OpW?5%IJh4
z%Uz*nsbSV6U;|FMLKNn7835iko0)Lr77JNSZe_wFC&L`lTL6dPlozFW4jrA)zvk&u
zYuPT*oH?Yzv>h@7mU?*po@GjjP_uX1C#LLf_!8hjQSi9j+bEB^_+D+oTaI;xcVh_`
zIfgQ3nK*<yB%Apyptqga#}k4*O_kai`jRi&DRuS{Qa^|(b^+z#*crxesr+aP_72q2
zvt+Jvjcd75ZymGK?bu!~269*u#DW3v+m>=N*OX92E6&*`t0bEXr&I)H*9zBKBe!df
z=}ks6K&+q5SXMxet!}!1OMOEE!l!(EOO_YIlLMrKT_MRP!+H5%iP&@{inRt`x6QMo
zYWalhDv>d_w>PZek#c;7&Eqaxi9p4?&Z=M~7gcnHgqh`gaAv`}Y#*1Hq+>^yZr18W
zW6r~jzy~SgYCInThGYvxw^(Own9tX(V1ly&vw8@sSrO(A*jU=AvvG3M$|e6PLG`}E
z)VGc{b1z%KGy69FJc2!n(*X0iVH<B`31wg!-2K<Vr4e&N(-xOF;Q)8s`U7T~nPldf
z@WAo8W$yIg!QclR&lGwM>Z=(LEU$C{bwX9vH>iGYujKypEoMvio?BZZ82M2XvZjl5
z%<td>nsnFu{7uDS^{SHC;G;daktGU3wvV;krIl(B{eyu$y-wW5XZM5&U<-d_WrE_>
z&;20f(iuY#SH0_FA3n&vjOOt;KWivAGEePzy4As#OEq0ri2VFcy<QOqLK<xFhYOw%
z0U{{Cs8rJr%%NTgDSbhEW*8_}7KqD4rBJVq@t}8{X_9Qxije*{wL61c&y8FBL>6{k
z7VgFu^OO^NmZ{#872i2OB<Au7Y<fgq9np1tLvQKC_zTThAw@T@PHOrIu#(1h8hlHJ
z8_J7c-o-r$GSS|CBpSh2@>Iop_dHW<(o%Rry0tI}DiXyFlC0J=Kr%3&-EZ5$;f3L2
zbqi1{oIZH_8G{z|u$Lj_KiOK*UShMoG)U@RE?96qdz8ao-@q^NR`c;*(6bB9mGvoW
zN^~9L=gz_0$0HN)F3qkhpgNw36zK&<uktk;n|=~|!017nvNg2Pgw-?RdZRJ;u2q+%
zcY5y?Ip$Y$8;|MPGE#_ZJ!*wohqHNB4PHx>)Uf2_?(G3sLP{SruzgYr@|bK~^4EKK
zWwr(TP#!$tin8zc+VqH7Bz@1wv|K#vvqR~bT{n-bcuBrO=wJJJtQtbff{E7R-hB?Y
zmg4bK@nMqoluXGVLhm;C!K8+=CNc!vUVM}I$xnw;xrkit+wcD{4DqBr-Fkt$?~he*
zxcc7P^iI&2;2eyo%mEljJx%p?{xx~DheXYOU^^KR-DP#Dwm+$^+q>%L35;SQ+!>lf
zn^qz3)d!uEXIl>86m0=*v1FqA-^h%Krr-;5nrn)Mv=TNpe`k(s2ENTPca!kR3RIMq
z#)~^!Dsh~+7JCp0sPz_5%lCP^@pGWYt=l=0fOU|X&*GP#L8`aCzn0!klngZKSBlCR
z?%qRiv@zGcdLZukb{EL>M=$@mO01Pn(Q5!H^5)Ct`FPTfC`<xq5Zc`lZ<qM#m}Kon
z;C7Hs{%8rHUU*RTEhZXdj4C`n@TAL2zB+_!K0!)BW`!*UP_zVAnT?L6Wol}4LfRh#
zg1<x#tvW||bBBi0^N_^wiQ(7+RI?Ehw*c*8>Mm^(JDS)D&h*<t`Z~3HGDT1E>kwqm
z5%3P-=%~Q79O$?|V9fn#@8-BPzR~udA*ETbQFzu5Vs5A3$loNFY>itcu2O7>PW<a<
z=vCZA5M4Iop6o`_s>7M9!%*1(&fV?=O3?MMt@toO(|_nmVdvtlKNY00mIb9E@0E5g
zHeIb7ff2`~PYH@0hPo#VmR!#UxI!ZQCGwo<73NZCDXsT46Yg0(4}*IG7j|H@zUrm2
z@9UG)8+RYc8G6C$X|fmhn9jM4whIk^Ca|L=cucT*K1rU)Ods#Wgv%{x)H{y^4&~NL
z7zN`x3ufc@N26j5duwP>!9(nXbZ~eqg~SrdJ<C^~_$s&Y4b8Iw$r)^I7@XFFI*bdj
z8DTt_62S$Q@v+PJV+?%VP<NmPHS*Zfwrk`mum|6XQ8FVZUoAk#Nr#VMv1H4~8rvtg
zn?Zy--3bLj*IivmX7%%Ya4vrrg{I$=c9MT6r9t;`H{y2=+o6sjuN&a$a)cl#v}rMU
zywv3l=`}J`cvo?^d(V>$K3y=6$82-sN_b>3&qKNcnqA@f?;(%q@xRkHHt~fk7I_{a
zlt)0<A{aD}xC3yQd`?d3H&Z+FP<d><p5W)x+LSeA@LH~%53p^8?%}pDRWd!+<2A8`
zmz*6l`F1(@=X^-wS)mZ0b~|S-?tSBNmL1P0_gRW-7P>c>j(Sx#>g>q8&Mm;Vc(NYe
zUip29ILpIJ_}Ks`eQc~RXZ*b9J-ua~$HeU>ulsF;?k5*RJQ33xgxPwYs*JC*K!3mp
zXOkHip5_E&eWY@QY7aQ+6PHuEs=W2fop!7`S&TVBL${W(a%FvpO#zu^=KR2M&c4Q&
zQ94d8wP*q;n5%-c&1T=91QqzYM{hpj3UF^QTi0MBwZ;w#c#nqU!}JapNbdGW;*AX}
ziqSl7v-)MRCifGb;7rv&P5A3_gaS|-kVRg&3n^|!E5nlet<jYWq70PZ$#5yluW#`r
zC`uym+5k~wW1rB9k|8_%K{u$-7n)f#XCw-Skq3enk5MU`d0ioCvj_<@sYb>^9l^~}
z7bx6<t{-wv0bqEPv{^k+P>k*rC@Ix_B<%_4;<)RA#Z%{z^!VNyps7AojR0Jdv4#2e
zU`_624c~G|0^B#J7*09fe_1?vqS%RdR=$3#45{N-aH;pon31#|#gyUxEX9t1PGRZv
z^Pp6c{~)UU_x2P@Rp8YOV5@(0-ETkOv2aiC^wH^m!IPP99}enwj^wo`zfjX-?02dR
z&0$hXB2a08SBJ5hY@z-d$9AN6X!K)0U*08a6lb|)CN8%A`%ZmQ+WSkD24`daHDm5g
zEP#@M)@W;vcN%^NE1dX-FHMS~U~}hUId`tTz{+Swc&hPepke@j;=>*3iAz6yUX6Kl
z82g8X1(?I<`vKX%(DE2Ku|2=0tGBExJY%vGcPK0jwqDYWvH*jzj5)GoHSKjvZ&>%?
zO^bZ(+mkEFiUGRy!eqRif)<m$uykZlv+H87u5w}yz)~Q*Hs_cQ9Nk&;D;F+1L*czO
z?gZ;t=vmC!QlKQR9tE-W92<`}OJ$FgA*)!zoOwlUN=8IOwy#1&_DtyA?1fD&UwGt>
z4d)vS>W^949)xzR{bR%&{;va!iXzOa@dK~dc_-ccq-UH?lZi+t8=C9wGPO1NYlwKX
z=jWKV7QgV=fpT^XeBMyE+Z;6lR4O$ne^IPls|81ai2RS(y8yoOF4r`ff-rxn=cqqJ
zS!atQKBTdw0h-o-D4AqPc*%(3%E}<9^qTsvF;$k`*jc(;rNq7b_7A@A0|2-tTIXu6
zvVC8*<_^&K4mf8O=_$otA7;$K`1QNT0*BM9$fPJA)re}^V#L}GI98Vg<e8;bKiH)9
z)!dKUV(a^PDnZ3dv0G_)Ma=q35JhPZaK!575r9E>b1GA^kqIzHTwvF_e=;Fr_Sd<5
z@+jc@A3|~bfJYha9JE`i6WEv^1!I-S#-3syOaKWH%J;6>HdN{Z5*<w6c}k>r2Nbfh
zf0Y=s>*j2U+AZXX{i}0$cp96lxer*czx_k^0uZo$-}%Ek+unazxPU0N-^q-?#cyyM
z@D99ohTfb%QNBHP<HGl2kz)TlF86;8{Ql|x)ieJehhDLfA;v9k0TcG=0omjWSD5t;
zNxZoiIG6ZJLJvL_QZM~>&Kon>YJ$MRqg1~=PzMAkmTlEM)mmkV?t4i4cJb!N9gvd4
z5O&Q@`s3IOB9)0x$s+INVG-o}O}oCS*ti{?KS9`GSAEBmm+W?+;cu`1Q%=|SyS|_P
zd<BLceS83DVPNO?9jmjP5rP^ewJ9=@9WMy%3Z*XpazPA21^J(<sIt-lKD^y%>?tRz
zO8<YGsw)S~jeu5-XL7e^v)@0QuI%9h;9j#h9Z%PrTR+9YnNQl@P?yyOgUODzb8!U+
zb^Czc^fcmt?kABSA=5lTIVn1<RqV`+yBpa9y1*S3Hf|+?dr5N2cT9}0f)+OH;T_Du
z2I^k`pGinG$YnQ$c0AOJ#&$eTnMKr=u_G&oU*{nJfOH>F5&|r@p&pQF&<ADfmfvTT
zKyM2mq?aNtNV-SHrh5<n7=+|SLUe)iWt>LQr#ogb>5$mK?Xdge)^l;OMmtB@e(Lf5
z{2i<8cycEsI}@NtMJ{J+KVHFE!?X(Td>rS)+(!xUer^t_k6L6+QTFjr)-90$*uo6F
zb==uyLrPOf6j_S#WpkI9>km85d}T`SmLLVM4;~kqXw1yuGwf!TJ{gu-87YiZzZU8O
z;Ms$F&m8YH9GN-S`b(VA)%Tqr10mr8N*vN$cjgR;zPQ<mw>C7`{ugMu|KLDl9cDIV
z!-$a-Mr^V?a7laFp)T&o<Kb%F8a*jIQ&2qcT|g^RRIq8}5v5;J3w1wz0`+eY@ZDTB
z%V~yOYR@a6pfq^A)n5;D|0NabmNP|S^^(tthXx3N&cU5;xc4|NXfqsGT<at^NGoIp
z4x)E^D&G0}6fb?FkSG6MKbPdBr=SU_Q4W$$QkUc<UBA>J1Xtjr(Xa~aSe-qq9b`F_
zmyh+RHP3WR>9(PJ_Q_na%MUEA5A2HB5xiW6r|KpSS&PoN?Gdq27FcZefeJ{zv}0Og
z<Hkm$TL-~5=IG<@+lzVNl>@7*F}HFw*-cYCv*-(Kaj`3Rb{0_Rl{cF}zy!*wSJ&uO
z(df|dCrh`|B{MA#ra~XdXe91-G}UA=!6e%Y>&YSp9zA#cWlncD3K?o5hUpv;9vRH#
z2}9@iNj7ekV$-LT9`HF1XM>2;3}l?ramq)$0ye#Bb<_j!uruF|QfmIWuBl+GxGciA
zsCkK|9<LnACXbQC&razFIgvozKVU|fh1g%%Ht*G}H3QT+*3Su?hhAC{q8)Z3a8wLv
z`hBN+DQeY~u_2(@6+?u%5tzfdyu<hUr2yc2Es)ijQzCcAb=9C&Tm}0$D89GWqRvv*
zX~oGc^U#eJzo{;C3R|aoiL1_Al{I?80vKPNhM;+JUYAKB6MBm6ol17;I+TP`=9d|V
zLp=$yH#h+5Hudl$`M=CB$=E!8WIs)h<vnR|txe6evlcZhL<%evbTu{dc^%W%_JZ9-
zXp0M(8|WJQ($nK3%-4FX!G%pO+9Pq!kY2{gEz~Hg5V%j97ar;tD%6wZ+AF${0C-s^
zQu57WB~A`FO+NxK$gp`s4%+RQ;9TLpbj=fsG3rdAKWIokO0)O@dt#V_iq{m7^Vn1-
z#iaO|srEr9nZEF?h=QceIB~HmA>u~o(*mo(!v0M8i_heQbTO=x-FqL8?0b-S`!!>S
zof<}Y<dsw2JTyV$-Z4~B&l9#baC+<AA=-8QkPRzV0?|owOpgBWqOrYXfTDtG;zXIk
zXzuDu(d@(6zz|!b+y_9{_L>{eWB0HrN;z4<42ATD&kNZJWo^l_wNs`o*Mxkzc7~<5
zlfZ=4kV3-8DxoYS$Fsskz7m+W<e1|MhDl*v4X}z(cW}ejlvxTkwnd!m>5(YW+<Jy=
z+=DK-*VjLrM~iATV#)fw2^fo$KG7Lp*0fT|_c|9Jsav6_YDo3#HK!Z|IE?n2t-NVu
zj$_`9oY98B7zXCX<mQ>Vms<dID?~|{ZD)B9&bl#=2a^3WH?i4v18?CK>CQB19!7sp
zY+^3TQ|)*qd|(1IRm9p#wwlPNv{_tzH=WhZ57u&P<CJjqy@{M6e-Ld!k5%N9DP+u!
zdS1<pI6A*s4AowEzRwpbLWgM1VhzP!t;=T&_bpO=&qy8}N)`nHyS5N}+0j!bT6Pl`
zjsLtC5M9>){<G;RvSquxgH7;IiONaxy>|HqU%T_e6Qk~-Q{L4p&i5%kl}Y~c<EKxC
z8Oe^2y-3WHQ#uAE`D|p~y(>A<$0$aWkTKxsf(CEX=1`(}vFPXw6Z}jg3sca)+>J&#
zrde}lRe<ic^%B>B%R(7p&&+eZ1Uiedw2Az#in6G8we@yg<g$o&yb1d~hV``-{Dcu`
zzc+CYuQw!@FDre%GU7KB4jTF{OkEXdZS`?;c6eTzg+C3ogvMI<`dnpx!6RjFh>nw#
z4%Lf*qrd1QEJQ-lz&4UrBsb(vpj_sZ@%TEtI*Def`bTlub0;p62fS$Php~q)B9hEP
z#mk#5WY$hxG)0(JqRSt!b&BGs8HK6yv3W?Bw$N(DgB!61$7ja-t|{@YV=5JV=^SW?
zkeh$OKa=~I@ywt77o312yVJJ~IQ#&yo6Cmk&vX9m<|8FGT}_XGTu|a9`D?o-*y?^S
z27ne-?v>7m>aEFa9qFw@CYwniH>{78Q8rcmvkxIJ&U(K1he6j{oq;A7%=Q{gbNN6f
z_A{m3wc-@AP*M{qHTc;4^@f4KwAhf4p%Qe_z!au&LAQ)4c_Oa8f(*37pIWsxLV25~
z4W^@9XVAmCjzg+t#n8^8<jcO3k<#aT1Kt^(b|n}u3r2hH?9XXH+#s+QHrf^eXWf{J
z*|uT=G!9&Dy;8wC#(fk6z_L6{v1y_0p&=)MZl4(|kt97UigcYSOju?&ti50wXB}5H
zbxbJO*KQ%6NJc2E8@I0;Iso7<GA&TwdyqG%PKo1$%`;-Q{Rf_-2a@0{z;Le{mdmC?
z->r)U&g}HGUp;O)2IlO|=AYZ4=k*{I>Gz}DI>dqV>1(05CIA36+2QX#*yzDyVL;>T
z@0dqvZN1y)VJl34wyLWHx2~veP=8F<_kxr7cD-1`f%MxZYGD3h&c2QR8&?lFEb{x)
zm!DQaPghIgs4ZF17h0LpFwGf74L*?!0H)!l-vQHuF9}0RG_(VWT57bEtIZ7K5}WfP
zCnKVVtoXM5d*1(-vd2lpA-2QLO;w))P!gxe=PT1m7K*D5AjjlxYTcJAe2Z@;Pp&o@
zQ)9F^*~8vw1L|;^jYLx&Wy7)@%^tKy5NeU-QS!~_EHh>eS56F9HCZ~4Dk<iniH9n`
z9V|yrF8*iMLV4&OsNI5+kn_b=tGB6@<`6K!dh;(^x-akpu1M{al&>ED4|3{~Zea2&
zJvz4y<Jc~LK2rm`XbwQhn9%K)myB0{a~CG|AvmMH|1toBKkqOAJ$_rgg%5)DCltW3
z)CV6GEEQ7k{V}9YAp*rl13?mhex(-Z6J^^gF5JHvno=^o4M3`6Phx`rCX?|4<r{Ol
zUoE+R_f~^RApi>3)+6)m3m<gk(WQx+S&K3)^J8Fz{%R>r6y>BKwQ~eos<R|ZARR+(
zBQo#ta~go{jOZ62o9;0EYT=bkyNtP!vP;f{gVpYYblExva(EiBll|tQ04Kq?0SA5J
zbU{m5dQ*CwMP@Me3M_8=ke}1EZJyCGb_yJH)x(5})}`=t*}B;`PTZ1&>=c%o=Qkc;
z0p6g^llQW`QjjOxK`KKs&Gc8K(Pf|bxgv;-P^g#j<a{sUU4KF<0B0h^G_ZIHg~uWP
zhBN&WX!+Jgr!0=l{Dlun)*>^*@nkE$(5<eChCT#e(NM_FCAoBh2mV$Rt#d(iH-RZA
zo`D0gc%!-j%2`NfK|YIU7)t1l(5%c;%+<vPK0~f2m;Re8c#j7s7+_}}G^r`Lh~HDK
z>bd01_w$I14Go{c9?xp?)7v?&JF;f(P#rUHy!T<CsW**1h=N}dl9zey2Gft?z5~Kr
zy658x?Pu3fhir~$%kZX3T|4=M<j>@g|Ha#Thc%sUYr~9n+zL!|1QCQ06m&#-hmdhV
z5NToqqzi~hlTHXlWrR2=5kZg=qM(2jX+r3rpp*zmkq`ojln@|5XdysI@;-^q?6dbd
z-}jyGz25iaFE2xW<tgh~>t6S|*Jsv^q8tUYiY(EUft@jHFjt4RzZ~7fU4n68A%>hM
zQTBjE7K3wim^geawfNZ6z8GmE^@IU{4)!ITq_E`eH1;>1%w9N?O5}DTudF@CmHVve
z_<fX-PH#DiI^6Al@t>?h@pm-lX_)jgd=aa9U*RxSnyeV-4XU7J^_L}1;mu?{S2hoa
zkXztBeg1xf_g^uC#1;ZBSms<2+}*gZuz*z|DLx=xdQ!aE^!=<V+gLWWt!CA^OC|C*
zzkpY30nl0nTtQPPFz>6nm&NYOQ5)2pe$3#wzU&hZetb59dF$Hbx0g<ueSdv$<@#W)
z^8wTkr91e0McRG8EWIry*W|sMik$d*`47j$L0x4!?c7zo!LmN|`h}Qt51iu9xx@0F
zLW`UFGVE*S&ZM@`pT^dEtEIo(J^I!FHh9A!B#Y_&EA_fY1XOW6ccJ0~;SaDM*1&#X
zBF-8L$=N%pq8?Q1UGLRcs@6QvM?R3qAdVsDK0KSvGD`bw!@T`<#m0RUW>WKXM{iL}
zPhi{?sTOy~)QW4t-`}k`>Ja0|uSsPTEy}r_aLZAXwO|JoICmao|E0?$$R;H%@HodT
z2J2?ooL3;4(JMQ6e<e$_e{d&A>YD3?H1FWWdw0&Bg2u_2zX$7F3Y7V~Xxwu1iw{<x
zhRS4nw^chT#l%)-7<nq0(`J9LVJ>I2Ix(awmgN2>GA1(*SIsF+kW|I(D*PkUUB1wa
z2*de~MoBjRWHQdi((9X8^03KgA95v`wFxJfbtU3%v@9dSXMM|Uue1P*@mX<8!7Qh?
z9)kgu9Psrgq)9Vo{HwfUkXb3)qW4Akm0ydpB^^@T@#H!iSTgfgxV2%{sTx7DXxSc^
zQ74HMv;W<zP_F`P2`54#Na9k|pv1I!h~H7Ag|;0z`*_&zTY7MOX*W~y284Q+P-4PA
zYJP3B-fqe-Xp+AMmAEc+H0{wN!payMVotcdUI!rr%84JVNOX1Rm>o$85C&}jcLN={
zDsvOZcon2pJlwM4#6}daKvdJYF-!2Z_QUa)dma=@cSf3%MG{$7AbS2zhkTUwuXIR>
zU)6B8`X-#h?bQ&AfY}ylD0E*3<vur2J)zTG&aAo0l*dK(c-8i!0u%=%HdU#oRm934
zv_ZcF8+z6kgRyfQ(#|$WSHHW=gb?!Wge%hAw|u}kXme%Mi6lp<Bo`fsf@L3b-Zd($
z`i=i64ft_q2{v3T1dbV>V%WTSdQr1pHd`S>;vUw%zkl_Q%mFaUD#hjBMwxgKrbekU
zV;w1YLnQwzC~X|o{nV#(!k&Aff)nwH<;M*$Y~aGl)0>Us6p?K?p919D2*CG`FMSX^
zCfyzH$sg)^GCd#?!A5N3ff~*G9RPNFiIrsTv;f=ItWY2hjrUeNG;qR(IoIy=E?PfZ
z?Xkv0A*9PFrth|+G(z@<j7fNCYrcC%`B7DTs_e|pOPc3WT%<C(ALffks+KD&B<-j{
zYqwG=q(7=Z>*kfS`ZbTFSQ@*v1cE?d>J{M3y;Ro1t-VJ+IfP@MU_BK?+r|QslVhX5
za22vg3Y8F;seqc(V%liIbA;xt5}7}(=xOvMK?}k!dL{<9XBf$}jTNMxNZRF3u1oij
zu=r>AY^MK{?;J!TXNu&8#om4clj&)w7riC)2;FvEyqjj!4F~jg%itr4>3i;84tEi4
zneFtSJ!Y;UP1Sc3^KhtYpU~?Bt3Cn~EpF9WXDgS?m@3Vo&zOUHoQEpXMnVVEACwO3
zVNS+x>D@ZKYI{*SX?sPAzs5LK+XfWco;pbzoC1}vIoxWq3i}@1+u86ShZNgrKLfVX
zCwcM<kt<lm-&5QZen}=`V85NjBcdeTk33ybePB&Dh?KbGm=p}PS`dfM`$UK^BHs@2
zrmWoauPFokn|9_i25YpMzNdLV0Keo@c?qm5p1++Z{KQ9uPnVDB(O;xKk|!+onf)qw
z>F}`(pmE~YW>s!{MhKh!sQguEd%0Uq7r)fM^<#{5`It72as)y+q3W6A12U3)sF-jh
zz#;8&N1Kbv-P8`YL>9ee<M(!-j&_&cK!U2Yqlgubxs{8ImnmO*-M*_#F9A+(F*$6y
z!mc<2a$2YlGScVaZY3CaR|Tp<5^r_2tfx{h4<nw~;JzHbNoM)?00z=6$<&dAkvtW7
znzZ+@zvAr6vkAK%GfazD-&<%aA~0S5WQ6{jbPOCTrtsA8;(S`=4ck0Di&wi5Bs*1H
zlLm<7e%)Ua|D;Xvz#L4x*K9i3^%>r@rev<5D;%nbE8v~8A?$`3M$0c1QE>%`Fokb`
zcvnwtxv*`PRFCN4PXAA>t`%AB)QuRNo`<ZtlHw-o=Cu!%HQk#x(RApizc)(}nf@`-
zQXu6-lea+WF&-fKHlz+Zjq5TmsauF>>LXs~<v_OFr}kvqM^<eE(;}pr`3S*G2n%X`
zzH1SXh9q)T5kqb<Dy2utskUMl35+_s0dl-`!WE9dFQ}PcqxE(O*ctO>s+=Lq`u9Qs
zlUpl}ajxSJUQw7Go+)*A{&B;UiY+?lhsoOob0GD4ogyrdHJp(-?MM(PrG8Usi%(|m
z+L}jBK`m4Xi3WFXy!Sc~dv2#}D&9|MI~wo15{2deauS+D7gg@^xdAX-fEWXqQsgLt
z((qc7pra`*J_|dtREDg#4qO{SL>kKK0`}t2<BH~~lMRI{j%9B~M{_uinPv-AWixBT
zbs&sp2I%<U5;n<B$gtb3eufbz$ItNihlZ^l(oy>3cOi77KvkQu%rK}z*LM|t-6T5E
z2tobXKFL9>>=ba;<73dyi7I7F?~w>2w|j+vmF;-Y|I!YD3}ZI5X|6b{jjjNpcZdqy
z`K~?3yW77**F!qIX&L#B6DKn-^J^OR^2f!F0ytf0vM*LKzN2k5JJP>?t$e|Zht5{<
zD0~8?SBB@;?z1~wcQx<^qj?GpicPleydQc5<6T?H{ZYV-KEOkYF&o_mfOE)`Br846
z1H-1*WQtt`shS|EaA&7>1o5-fpFDWF^u@mcx8VQ&dl)<a5C0GSX$tKv$F<-5yISEL
zs40J*z#aadA^!i9xBqsn?-Mm7&ypcY^*-R$G)vRMovsxdpPlsQ-fC@FzJHRJG#*^a
z!d1ny-r@$5<cI5nn-B2tyI?o_1O9kquvOmt>5LH04`$||@}tIr29wO|pV{)^6NK-~
zKmE@HUS~OgPl9>4`ELYXdp{IBW#^Dcd;zW8+QGIBC`>i~?q0H4e-$ROG!D{Z)TWmp
zhSdY)ZV!eOOj6kw{;Un~Q|<Wt)Zd;L1@=Mq1?YB0sqxFDx9dU~Y>@1ncU;geQ5fCM
zpLLt$B#=fLfc+VErWGpvr>?P^D-MvppD3z<v{7@F&1HvBG7@0p)(#Cq&SlHjSYt?I
zBR!eyH2KbBoijjZ4Z%WH`r}&kKBA+EH^lX2dg4#e^p21FH*_R6R&ur4W26I`^u?DP
zHnwelZORnj?;nwq-wC>PQgS@Tez^;E9w?KDP1%Y#+kLXbL;vFG;#N@*DS%&dl*YXP
znPK68KNczR=Y}7ksrut8nDWb<aX>utdE0?WOuG<$tMxP_Zu$OV1tf_>o*YJwaLS#;
zEFZ34G?bkLnrfn4jB1Dai+{fPU%>T+%YOmaf{i64J=`|$csQ+g?*3Cj2BD6|!>`+8
zeVw(J(<1{6WmBS*ou2j8<60ccXE2G{Bb)yvs9k5~P`mrvtp0=WIhi?_0qSGHbxpu^
z;VFW{0Do?%7jVY^*Yw&koT(yh_vGGEQNBg^oZN8WqaZGY`3Z;kL?Jt*D0=6QbW_jT
zA4w#pFU=q&6qLGZOBi0Tp=i#ZNX0cLR(PwO^sEaz>nAI}He+eo8o5wHT={}YYNWRz
zsjfsmDUp>$48GR-_Tv$pCuNgoGc|+`?=ORo;GqA&l`F>IJRBskyp9%EF1D?zc~&XW
zuZPREO?X#N*Y1>WQ%)5N?s9i4DX=19+as@8L(`xi%c+DsP%f1tA<A%5?1blMWj1Dc
zH-_pv8Kw1H^L-VOt8F<OS%7k=Q3yCRHP@B)AQwh&N=am4Vo#IbdNNj(D9|?aMA96y
z)a4GY9Nstyjf|V8K6h2^DX;YGDGQGVG($?}Mn5vyz<|nrh9lb_PzI&wK7A`CHXk_;
z%S{1Oj4dN%=(b;vUgjY88S}HUC!zS7B=rVVcRIbe3(+{aV$zsn9%w#5@73eph~%o>
zYWQ%>AjQ%yj>@vQn7Xa`I*$>M+A#glRJW@g510V}SboC;mgTJmu$me)^27BA^edON
zcjaetw{|xXzFzqoXJ%m7E;nqeeDAck8{Mh3`lTs}Hy^cuHnp{9UMBqiqu7$RF9M`_
zxdsXZv``aHMrye55&sf#y$qr~DTl-ItU?U>Sk%nr2gnoEr?&tN^LsM8ytC2pO$prm
z)meIf?m$vSy|K>|FFPnaZZq;rlHB3{9-9_dlo#jeOUn<87>UXQsYH0wS1T65o(ZPk
zK8jT9G8CL@(juQ4?iktargHmYe`Yjp7-JPeQ7ymvq>#~QS<5M(E0sF_&~y=H7xfHK
zT`D9}gBz3E-D$Ov>0a4UC>MNWcg7UEKc69zvwx7WOo0XvL6wsgF+;gPVdotnqe^m!
z=~T~wiQhT?=jPS-;TyADLt(GShw-I-mo@8AUX!=dg9)|%FF0o-7o0e=`b|k+SUTo1
zea}BWSdL%MWZpxnlo)E%CNn&?n|qpx4cH9WGkeY^yH})|YD>l(c0BLuXvEGs8O?Sg
zVQoVgERCiT3m&u9ebXAkh&mWN;S|^~k$`XzPM!dWGG$4T_U!EsfndlPRRd7Da=qEu
z)-(A2>=g7AkKH&SUp_|*eps8^m(6@l$_gG#TG9B(V-Wn`Al0%im3Ta1o4Gsfj*Lwx
z=!v(&p;U^pPC%zJs9n-Msh(hY2k@7-#Y#PwH^5aT`X7@9$NC3e36vh%KUn8=q2|V_
z$M#Y~EAn}pl8^aDFHJbEWVKn?%EYHyZ$g~^qie5voj-5J&3myot10O?k(Pv|>fk(=
zM|MeS;exCTD~ircnSJubDL%PYT|XsKBDi}W0%a>`N3K?R&3p}4kiSmw<gqJieoRVc
zmqs2AF+c5v(Rh(n#Pmt|7{-`%=5url<WlXXR+5R;?zH*C!EXew0S2u^C6BSv!};Wz
z4wj$4Y0Hz}80Gzr<P~|{h3Z@rlBVnBNWEV-)QiiZF>Aqq4)m$iOWQmkO09T8wfqE4
z?5!doexHa=S}ObXF9h6k|KP+L6!#S%-DUOz8!AN+Mo01h^Vo{|NskWesL}FfbuJwW
zwh-b|Yc~nC9@TL?8RVGyQ{!~$WykA1J)*yE6mD&ySU!+{FYA^ASp$OVX`m00k2pB>
zlXp!CoZGZT9Oe;h>;4OZt$wt^cn&x)XV`5)7KS9861{|JxA9&;(@0|0+#;f<!30Ig
zPDTim6t54h9eEgL2~@5qS|guPXU@y?PJr^21asFR4b-V5J?mX)DNhK_0>LA9pMCan
zF3v%eq6a-GEe?rypXcrVl?#?axmj~*d0F;BJd!Ex4L&xla^GMjtWdwLpT|~n2yV%z
z`92+NfYmLE*>1TRO5*rJ442FEmQ$qzU{a}y50~114M2Z53+Zy~TFJ-3=ryCsj8<V#
znaATP+iWCn=4>4C>Akwap*%CUiz~@d82tyIeXuXUtCba=B&T-HY#Fm6Exu#fng$r!
zt4b^K8E^hv|IJX=ro8MIIzv0(${9eem!IsxUC`Xbcl@bRG~yvs{?7ARxsUL}_FYx}
zvQSX-;V7ux`ZYsF|LO-X=_#v(mNW>V8bq*$+Spw6c@4{JvN3-xI_*d9$!xN*TaJH!
z`I_f}*$((?xwuP~-Hw(CD*qj_twAD{{kfm#(QNMi$&v;WN+M#~svSOH4eOPp-Ft|k
zYqC@gA86SeplOG`y<+>3mFlYDbEMHEc*cmY2<`lAOE?Iqm|fJ!1ll#gVdR&PZ@(nu
zkSchQ@Y~8X(kj|1!-F6#ehkAa3)i@yCoj!B8Rz3gI<Tq<N~&ffyLJ`K%|HG)Pt$pL
zYqdx6bo>a(GlBAQxzQJcYIl_Cd2ncO!p)HpPCG$Q6bwwd_C-VSS_3-xQ@}nE7dzC%
zFSH^#(KG+s99s(=d-xIx(gay@9rb7c$L}J=yg!BeX0QyWk<aNqbUDuNnnSHJFPgaN
zRLAaOcTBuxR{Aag`iXb>8H47mBZ&>imQ$HAv8ks5@|#eGpZc0X>8Ls3m&IHVFHI^%
z){sUp)<%HOd7h5UWcEcIP2N=i_?!w9Yj%tok$7`I;8vTswUFYZ%O8><iGojp{gV1_
zYL@6lr$cSimypP*#e|rmTN$E0iQgH<6>q`=5(kpvEAX_AD>g<o#YBy#@1%Q{;E!q>
z+iZhJOD$+47>D(TdL`Uvl0Pf|deW9oecSnY{3O}KD)%(kp~yAWCq*YBX}0eXP_QaB
zM3YLoEI(@x9}xj8f__S9O1+=MuLc`WjWQ(Gylq-Mqjq<`G|~u*P#J#!NzzaPGBa8L
zI5+qy#fVcYYN&A-NXzJjpvrh~YI$fUh=@_NJwV_t^3@BC=y{A_Z2#e4c{!^DAB{!5
zZ@_Nas+O7Spih>E-C)_AN`iwA!5!AF0+>{cNtM@f^ygV4^1xQ+F3^gIl_y!MbZvcb
zjxvu8I%pPk%FglV(o-SHW6c;z<r5IwalG_JLXW4rC9PytR`mLC_;q|Y_X_ai>$_`%
za`-N(()o?bXrYNz=#0^5%V|&?2awzCmo%<VDt?qRzYgLXwOPr1m7tEd#a%-y!l|`7
z(&7Hx2d(^f*{c>Nj5YQZt}=6M1Mp|v6dSfTbS$v@r#uBDLgeMtixO+pV!H~wttI31
zP$HhgK5uYtpQ|R}++|QL090}Kv_=b8TkwA6OwXr`$%1@vSZ3VF3b4D<piyjIk|(FW
zAR(39jC;9c3K%%NdH{_EUjXsqvGRo^j7#T6*&|G15Ap<2+4twfGc6d)oQF0~^!uYT
z+p5E*OWyt0#9CcTUNCvbYWAq}ilh(W@soxI5Z)zoUvNi}ZH8_w78+V+!@PAGGr1Y1
zm<^XIjesV}YX^}5!$aoJxt+^Ab)RBL4mfs}tGS-7@iLuJdc1nSiXVWSjQ4Nn;g|-e
zgnTB99(Rqrckv5kf(K)sgzis={uqD;!q+nk6$DBZu=7<U5vzr#HlVUyV*pyg@~*eV
z)+i*6q-?Uw<+(?tdst`w9;NHNRRBim3S%;xQpFo5&vc#Z4uFB|j{l(oxZUWMu2AXa
zsMj(3Yh%n133wTmOtGcC5C$Wy!OOHSEZMV!J}f&R7_$X%kyin%<xLB?L{qV5jZ)z-
z#=mMqwcI`_>7n#R;b=$S4CMzAnEF`U(xDLVT=jyi_OZc0%<Q~9b)&q%>-uB(Y=Q0U
z;z)^Slj7vEtp|0fr`PAtB3#p<#9(+NdBxs3%b@0y9SF+G8=D^H<5P{g>~#42CgR9Q
zw6{(%xBdDT?gPbgW!KM&voy(58>TBg#*@)uyFXaV%yM`10HLPP2kopo?lJg08GTUs
zC_wm(iPFm^tX~c{{<&o>_@Hlmd)AQ}cF7EEisLbHOXJ4mmiM{W%%eCO*=REIb6->Z
z37mT}>xG4G{W#z?C-<4Eisy3lr15Ejku}|Xy}<eaGLHSV+&+*SSf?7GBfeVkVgaZZ
zULW-7R;f|SR%>m5W+xk`xM$G;v>UA1RpFgZIrNE@HBpp3myyz7S;M(9ko5Xk`uQz$
zL&yG^QAke85ku(fEPFR-jTzXV<+MBSqMR&F0%ziFF0?JM?)<hQg_4nv^>q5NHa%LU
zy>B0_PGi0tS0J8#ew(KIbtOEyO&KA9-%|uHz7_qkV}9skPrKRd)Mj5P=_@#a98jo3
zZ#ibjLHEar?il>}lkpyB<+J6Dz8rv$Qc&BL<2;UQH+HYMU%|I3tqfe*U;-L0@b%In
z6rgJ>Weaz+g5fD9jiF$IHb+1@Chd?S4fo;`jqxZQyPuz?s_OVCaJ5h^J<t0q%h@j=
z$2|wfhg+R;sL|sU?~Y=*M<5%RC8nX;mxjF>E>2vX@l_-!!l#{NJ$?j^BGJRkhdW}+
zz0GdPsx8Ac{&0`^`tm`NEsAoY<`Lm`pueHUONx#ZBoXSjOo}DYoE<4zlRrgDb&LM)
z8)_s}`HYzLNJ}yPS*so~v*B1e<=J`dsR`gC=z`QL720NdmLOdIDe`<_ft-iAnqYR;
zWy|hs#zi<#Gc>2;e6=FwT>%s5U)0NA>K?nE?(@mz&!&^t(`j|j+zDs#f-%VxeIswS
zBy5BF!Va=jANXuN{Dk<2hl*0tpJ~;Zh8t(B18>si&gh+gjjxdJoIV`?m|^I+y~|4}
zse2;|-ZQzGX@}n^@BFs-LB*vft#6AZ4r&Y9<ol&*)gvT(4RT>S>qV-RfW4@dR7-dy
zTxvNyGH0sTkhD+o&T)9JSn5>iFH~mM=!^atiR#JC?Wt>+^84A|E7k#WVWYuRmiJDZ
z^8A<;l7-y9z(bf4YAe-i<E+-FiAsp}z%8Lr<}~t(TC#xm0X_xy43h;R#H%A3{+-$D
z^^)(hj*Lwr^8gE=>G07#+i&ZE-;~T9N$VJyQ%`Rh+16~tr^(*Yoou+k15+2H1rQGa
zWB1aUew2x!kd*E$sYjqRfbYG1PWpf0_1b9j0i5X?zt$Y-a%wLR6ivv<`eUBaiwyaL
zrL(=%pARcNr+x&-{mAig=*BjIHY>|*xR7Jx9YAOfPqh`f^R01#Getp<{ket*MxR3X
zPg7ZnJg5=@gyx$A(0|bjoqK9PK|d8soX1^XR&KcIBvCT6lmB<1B&IolhmWYeCC=|<
zCAIAf=jjv-Me^UGX!yAl1HGwAhEJ2&p5XtTQT!ormXT;I4i4ahaYHiy1Hk7UicTdU
z=|+a$8Xd;)<r@TsDtP{r7oa9i&lhK~lEBk}4<dz!?(#qX(=AwV!2#SQExFN}U@tH*
z&Wg1>S~2g5>tK7p1I<CekmA%!dl~A{{?5)arv;iNl^5F(#(q*;u7LX};2io5(FgzJ
zHHs;<{|;p8u-AD=)WTuq^9rmNNZS-1$K2V}K9cbTRl6|kr7MQF59|_NbGp&eu%#Kz
zwQAb>h|?A0jtO#2YUTypYbzuVS`809J>~t&9xof9MYPC<ME?v@Xb8ifLlf44vb?e0
zF{hhDQdTLEzqgjU)1NAW8wiw+E)3j4r=rjn)ReW;ShtLy?4DPYL>7&3HWne2s_n@$
z-LolP)yIFYdB5ukP(p`aXw{YOM&4vjyac(TzLO9?)XlY&?mrmEaGVf7p02|kpJ2*U
z>#UJmSWJ39`YP>D)h1S5(`WQMqq^yA139c+v2V@U4hR3G`<9n{?JUw3istMG74d@y
zA)N;`z(SR8D~6CEOGQZ9%Kb(eNYl?tcNZ$I(q6Fh&1>f-WOzMgz{<TIT1Oo1zJ(<+
zaWfW$4AMfpcRicA`LSR`r_*QmiGFnORYR<OOu&7?<1T`QsSl_)MAD_g)GkRppr!Ko
z-=&WV#+m{1k=R<cv*1NeNdp_h$p|3+&>7_Ml=sDjH$1H0<!!xFklc77q$ON66=6Z;
z0g6TKd@?JzhJbelXlAf@x1w8z=Ee5Z81gqT4!tI&6}DK@>4Q7jkG>+jJ4+_@S8JXe
zs?sp8pGG^v2nHHOd<BUT5XtJjY&Y0M^~<7Hwy$NAUb1|^(jGKtx8;Jq1OftfFZS&w
z%>?;+9f5*Y9ysr6m7t&$Vj@xZ+Z$eq-QIMXH!n%0vAz1DOTEDH@&vyPlwN@No3=_(
z_PP!<&8ExFWIk9TKWY~qiYTEP@}9c%{+s&cPIl<6O2@wF@Rs@cH9_9PpA~g@+f+$U
z2=HPv6;EJK=b43GO?Vh?T(h=~|NPe?d#2lrfkIl3a6=#{ngvp#Pa-%^k;{c|;#ck;
z4bWOfF(%?6(V|N)f!5|$2~aTR86WT0e(SM*)qaJe3OQ?HLJ_8O+*IyLq#Zh{MocGM
z4ouY*)Wcg^w})<nF(TDT1HGY^doWO^n=IU568CclUmZ@<mc2xp{7u_?U51qit&aCF
zn$%esj^rnG2DN8iCUtv)rJy9S9x^kB3LxT7X&M;QcY#gO*`6P}J?=E19!!k#l_#-r
zf#?$oQsGJaP6lw1OkNJosJ>QHdmvtB_)3H+=q$35HB11)oRBU_GH<lJt0jaYobvjQ
zRF?GLid-BcZ=iE@)Ju;BOi!<s^@`|9>8-!d`#xMym0PlfU<+#P@p|^jo!u6<Ci6uq
zya{CW&n22&bvlsPF>iJirgMGx=s6(aE-IZh5;;g)ViOxoG<xsnN$C{B2;ZwPhLkbY
zo3nVw(D_RAeoXaLy4Rb=&DjZ~&ce9u^tadO^NsD-*m0WnYr{@a6BP7*#po7yynRq_
z2I>?xe0tiVFlR&~Q)Se)<Pox9#m21;5vbM|u63L84?B?5lMM_u6xtg_Mbf)O-xDB&
ziOm!wtWz|$UAPTIp-Dw=96-7B$T_&;+*F)0Zw$n<M}0A3Ryw}~U1_;X?rWr^Sk0d=
zGD#t#8#khL-i&|^<hMa>F-|6?Smo-{J|QPM)sqvg+A2r4i-Y<T*`$_xQgh~#;v!r1
zyuA#7_f=WT+X1b{GAd<T9Y9JC+>|$$&N0xfq6)`|k7*uNehu?gyMut(aitxJHt#u_
z_Ci@={h&t_GgR2i`uhE-8M$u6lA_~n-@3qcn((y5Y09AfRo_QoD_4UFIf)hIt>2ob
zQ`@C$DTfoDdgU7%9r+R>?EMp%n~|rGjUS#F+`}AP8bGRf^CpMuVkld<l4_5Nw`q-Y
zg=U(pZKeqDRlSrvFp+DIiY=%YEP(N9b7DDha18;&x4&l0lbaSjHA9B^77faIm0FLr
z^Nmd)IRYaxdZ-r|wFwz;^L@L8?0{mPEVBkVcN*Ptk85Hmn;nRxGmX_KCDl=y<I0e<
zqHctZWqJ{(7uobMED8v2wxb@MV7_lWJwDDCsMh#AUp(l(G31N2^uz?5uVNe$irD)W
z6_W6j=)fe`@Ner2ZX1kK&)na88t)7?)bC(J)yRNuSF)%Mgo*I%nEiqudbwqJ1%f^b
z>Q$C$CF3rTCb7h({e@f&n6qj5W}A$@Brna`ylb=5x-sAib-LFZcHXjfwxDpqF?PFf
z37CQ#I?MNIC9^@BltEwUOtvY>pXtgodnL+2t82X3FF?8md$*3tBz!3YCcO%@V=$vS
zK(zOTcD`KNjY!(d7zA1>>|x}bxdY3r6&y=S<41t!13nounStHf0f<7zY^E=k=s}{q
zOmaMQ05uz=gRM-EbTF#hzxVdy*LO0-Iibl!PLc)OMd%2c5;D1#5J3DLyuG_zg>Ox)
zstoAy&~jI?a9+~pNxd>u!1VUmOq@l$Vl%<CaV8h<5E><PphpD3b35~Q%OT4dgnyK}
zS?#7qc6QUDbq0C;+P6CxyA|eP?%THpT7;oX)4dDR25k@M=V1GVyNKkMra2PaDp=$v
zlThD=8F0{lodS*5hJLkngi|{5NbCj7qn@U@=bsN^Lh&ThjNWEZMfYK~04edW+3`Z?
zdTxd#^u<WRRzFoLa`9{6Q=-U4Fl>{c_Y2_SJ;p9a1pN_k|MUyKuWIVQdXqL4QWt5Z
zdVTOY&{c|VCPz29sS6P2z*yBVZWacYa^J4sabw)j4O?t3Mqj0_2hiT?|58@d0M71O
z_3=M+G{7Mt@(pu4sZhgd4qmzqrxUz&&_+4Z0^ToL;)I>VFrH4GGhu2H>nS&-9t7wD
zs}GDb3mjp=?!0q!tl4F+Sk=$YCj0g@68G&9D%FqOK~`8oPd{9@Sbf0h_jC}&6bsY&
z33T0ae7Zx*jV#B(BMpUMJvo14F_d0bmCY)t1MYpGpd3$B(OaKan5E6DJqrZoG0v9H
zUZqx&zJPB^H39lkL@EX;?mJM|&K+;R{jc%2G+|KxYWZLtkDIV2#P_kUUMHh<6d%R;
z=X6aw@H|@tlJ6j%TJpk6teff8zh-(XeR~wYZv)}Jh8cig(c4D@)L)(}Ij8*175?PK
z8E@`dQY!uHL73ledC?%;miFz^y!vthfm7=FfBlIj-=ixHc;q<9Kk}L-fzJUpQRkP2
zBE}nF*X}g%GxT>Vr<vuA=jpe=&xh*x{<KZ}ocriZ5C_F;+$Z2H3<R-ko<y>)0WN|D
zWieGi;1AO`utf^z82GkyH@J^{M{iDX65w-T*QEIMasq2={2>5;@c&(x7tfRZ+FAEK
zN-$vORhl}5j&AJU?Du}ntDMY(!81?TVTuJzMS^cfHD-|!A#wk#qwL%!$Qs#x=tHP0
zc+~woiD>@kU-})sNW1eid{`V7Im}@jTi|A7jMvDkq`i_6msS{W_pVq(;<<=H6Q@n8
z(>x41H9bmSei0qMDJJUO8Iq75x_Xm36$)cUZ<twd+h2$;L{6Tgo*5N$Dd}NNX*#lI
zIQ<h_{X0GM!;-23Q{ZR|7LFd?ZBT-nk<wm$aA`t9J4aV$?iIX&Ha1;$G)@=YZLVCR
zJFut#gm77CrvY@Lj&ajgH&t&gSlg47(6yHyv1Z^ni1B8Brbo;U1r2|M_e77g^!@v~
zoS-4|BK`?d>wd5ehc^ATJ{@e}=*k+sX+GFbcX3gXx+=_GlXDulMc1v=l`+$$J2o*y
zMT0$JRa)uww7Kpdr@A!hVoi|-q}bd|Uo5?Ed9=HJ6wO@KjGi3<6Z|FgSvO@#jLa6Q
zcnF8B>S}I9L|BIZ9%?j%U#OquhOZgGlKZ>h<ki@wFtg}##(gS=(PxsX>bF>LL7mPZ
zvq$0@<nmTBZ&nV&`p~^!;C36n9wgr`FRZRq6tzNXvon)3My?rqV5~cjp@GMlU#P!Y
z7d%>*#eR(G?9h_S)(0H}nG82lcefYTPO~j^Gmjcn)gz%l5)~S1q~r%(+4MQ78R8KX
z?E5Eejl-5}egfpciw4jOK`d7tbfl>JWDu7!;YfZ8k89d^)3*OkWeay%y!%-&x-bjt
zM{W#cRb0bLd7@oN4_49#Yfi_fi<0|L>d6xN5q+MBU`Ko5ybzsY`bo%&1-)mRz|f@x
zNoY8{E^^iR0U+oEA0WKeF5w2?)o^L}{>;$TX-?rLxF9(y4>ho|43AEM>uM<W#BhqP
zW=T!WKZ1kv*@tagoMZveX*j~&k@bXAbh|UP%Z^*0WU0R~wQi_SwFz1S*KH1%0njy%
z>~#WRC@~8wcG#??b;CYmZkX~j+5o3dcJvUCw;Xu4u@N3cCmbn{V;sjzO|M<PUN+kU
z#RNB>m<9E=&e#5R3*Z9h^kW1^S%TXHDrSOoqKO!9osbf1-X5un;ne}|GqVZRL34Zm
z3T^xU2SVy^TlC*8gUyqWI`$)8LV{Rq_jgd8&U?tr;cp^+HT-ZESe?9oiC4d0?|Z?l
z_{?l!Nz_0v-jUeB^HHNpxhi(!l%@?9YLlYrNa|P@_FK2?^uwJ1p|A55{`(G{1A#|3
zt_|1~Q&x%{C!Zy4m%|G=xA-9^);*bgHSDrso>bK3($Qo}$VJ~e<Fsu>ppp{YTb$Z?
zszk5mi!V$)Pq%oG?+!-4(2Xg+-Fr69!WVmObUA*1{rLD^OfD$+S|%PIl{)x`o%B||
zbKObY{L7ILl*&$<1AK$2(LK5lf~^1qF?K+0|E&frcn()?Lu1*mO$p0Hl}^7?nG(HR
zt24Aee}w%y<k%UYdlEaEWA?)!aOG0yWe@;WUpExHI7Nf_J*f6-J;ATRwktd=v^9)m
zhB~=<U2ZXPLa<<LO$z^=v`%HSa_)UK-U;+~eB~%rh~$&ahfQo+mVqA8GHgvd;?rIb
zxLJWF1I9A?QcI74j$W2Kkew_9J*?t{GlK_hooJR7I}^k$YWp{v-^KJ?iLx`?@Oh--
zJzoj=*RcXZ)bjWkX%bJ@Zwu2T&L<98JOIVM&q;mXpYvZs%Lgi%K`CTjXbI-3srZ7h
z9qaDDy|x*6ZR#MBjtsln%GmgwoQMR`tRid%1m<15s(J6?Qzi8QT=cc#)Kv`WHli-l
zAUgdtm>1D`zxleHt!rHN-cG-GaCNKL;)c`jQU9pQC?B!t^kJU3bVnV;u#EQB6`(Zo
z`rFms{uMPwUFN<XYsP-!L~p>z*e^4wn6Z~YK!^AbmdwJs)fvShi@&Ev<IW%I4^x@(
z3x*NuJQZ4jjSsSPpC{3wwv7Tu;|}P&Ym6L~Ton-5ce~F-4&4vTvWwQ@`~X3Uf-SL*
zicKQvgIOQZ93Mi3^Og+a?<F$?+HNS3{zLr!oO6AJ#F@|%o`FdwRF`RLm%&rqgBM1l
z^}50N%*UxN2WTxuqPuIfM?EX`*B9wq-KbEH(t`n7Ga-dvPB)!+CoCB;b}5-GVijdK
zkqKa$L!ismzul24tv{UM4GM*ZE;kO1>|Zhla!!Pmwbvjueh}g`ZEgYNMZgj{t?$qH
zQqsUfiH__0cd6rfs8Q&A&q4tR7R@*Y7p{JV)sL0qIm_aT^yIJ&afQ06?08ay{l=FT
zlhA8NDA+T}<nH#~(M{N|)3R6o_0@j@ukQF)=wH$_R4;q|$&Td`AbTJkT@&ixsB|9?
z?|@MMXJ2g3!>|CZr?tAo`ssN)-?YiN=O9KTe~S^f3dUZ3ixJ7YE=7sYR6^hxW&81l
z63m=?O1Np?qO89c!)Z_-oCLQ9#j}&8UwO0wN_^MLmXDKUPLN-D#_g=BR0Mw!d#x;1
zbcZjAd1Mh98C68d&oe>GDZ0fW>i4W%wH8lxZ!OR@o_yZ5@d~3w|Iifnu4cY_tOWKv
z@NKda%VtA8HmUKOq0j(^;=G3u|4HVjeZ6<N*PxS$3s$)T;$YFs2J$oX#h?vn)5lXz
z-oDdqCzzmFEKmXUX$6(e2hu!jfj{@-e~uDOLwCru-1Uo#=DyDx9rDE1QbdohuctaL
zlYBSCEW&k9G(B1yN9K#z2_vUD`M(Tcj~5z;1|mwka^<l}0Yqr$&(87ZY+5E2XfPmO
z`Wh^`@AXo9s1o6_NyN0<Q9-TwO(kH%D8=#ku_ea#gom8D1N%0K$WW%oc7d@qPAClo
z59e9Qmuh7W0rUmIj~U7Qn31>QB|H5lK&Z;KdgqTluIfF%W|cw&c%mc_EZ7O)uK;Uf
zFb4oXBt;y}Jdy}~(<UgM<a9@Wzq09j(1`J_cCZc__8_ipAJWBQooLgBX_FE7%zRoB
zK`8(mL24Qn*>}a-A(#pDi^Cp`HER+R744(1<n=2yh0shy9hMX7usC73ZZZClMR>(Y
zV0eJs0XgMUX(iF*ZzCBCY(od7^ec!R+hTXZN?6{jrOI-~5o+U65X{4{>#Z;MP{9%u
z;j4W}3ga1BU6WInEPWbY!vqEX00AYneV%PM0+4T2dO)9v;-|4@;2HW9=NQ9=*fJV}
z?_zeN-X@T&qF9zXl>6F$8^RXwJh!j3mPs)TN1%V(gcE-qBYeCHEwM|B4uA4Dk}&kl
zOpAvkWUk=DQSr}K_kIRc#=QeGL7k{8xgxm6*{>LhcBy&+0p1XuvTmcgB{(OWRHPca
z38#wt`eFUr@k8Fmjf=2?Wg>G~!a_&Z57bU8o@hG#d`p9rh+IJZ<9!-L@nO4*mtXv1
z&b|5Uzn4l2I7@=`MH8V}fi{a0vRX+U+Gb3aO!_isut_}8V4KH#ZMp%}lRs@cGjjOB
zi&CpLH8YUbt81k6tN!glTMsd-n|men!oCh#6fWg7TF|!d7S^}k@cDs)J=p2f(T9@>
z2Nq%{@$BD|!vh~@CQFu7{IB4~+;*|^8b`=F6t<PVRIk{6r$^P+hCtb}dH<(APziiX
z;L!ZySc~;XsyldVOh)W>vdX-SRg^LP1gyoxPn3dPnk7Ct*fK0N<o!>ZZd1qo(M%z3
zmg5bYA$PmDN*-BSLgq?bow7tExvC?qdMf0_$APM6x<R{#;>H9(i~@N<uJNWNLE0)8
zgN#!R@A+7}A>LFL7!AVB54`9{(F;oyWnd%^V9It+sIIxQ^W-l??3xljUgrqHA&%6Z
zeXkmA^0%5e>JY--+m~6}Gc5mp<Ji8I!?~#)dC><Gxl4_vt)+_U+CgEQu47Az(zAc8
zfcZ9k^58LN(q;+Ej|Nw&mB+In8vwDN7xvcwr)dE~Ue0j9c-9ylh<3x7`73%cd9fmh
z7WaUVv-xM)neU;gfPk41$jRp_ol*4S$FRVa0?>|t(krX)@g?|82k_FGla2Usv9r4G
zEFe;k`f>~Tfun#OPXp$L(q93qfPm5w-~v5+cjtS+3S!mdV@0n=HOD)F`Z8Xo@zl0E
zM`!=}<tq3Rsz0s_jvcg&<8a0f=f?%#gI{p17;{q_(2EzrOnLS)mEHk5T;-=0%Ha9h
z$-4+rOT)-zCf;yF{d)2yU5@ZE0u?T<7%$+w+wGsPqmhwZ+TFS$%m7VX-=@+5?dwdy
z5m2RPGVanb?MH`M%wA%*CDXVh(&Pp0fwi3QBnMDAp=l0U&%OevcNH=oktDDWv|Hnc
z+h8~d-WT6*79m;Sf=Mlx$%UHQSbz%kWn`M7hs=Te_m@*FaO8qswhz7vugPk5KY}nH
z-ZTa|xCw~B!xwx+0JOQdacTdcwp$4LTvVu`$&Z-*OVvOQNXW?Ty*kK#h#C!iJf14I
zRKx29RVO)&Iuk&i9c%ZS=7EOb)0|axat|%MeQxe<ra@uUD^?K|p!1$b9)!8J?yl_#
zmpqhcvIw6HSEQSVJUT2=uPqVgivt{}HoX($1`vuQUJR44UV}W-!QiTo$3rMlLuo%p
zJ5E$eQ=v!4XMvbO%mOWvi!4v5*JLu!?t8*o-9#>9a|UfERieccqf-qkJW4jov%-(~
zp~BAXn7ap33T@hqfx#}e3IF--Kz+dLF&+~LEUlzMI*<vp1*RVJsQQ(rc53L^EeFv1
z#-?Z9s^zYS%Em!RT4wjlfYV7sB@f%PrpQkfFPBrpY(0c>I2n+%XC+QJAMl(}hO#YE
z5Gx!jyh$_Lk%=`*4tCqmaRRM*PZBd&?0ew(&emA3^N6~qQXA}mZgS3DIityj`ao!<
zhmpN_?}*2cSS*Ua3k0SKfBX?Ft3q?Ed9^!W)HH0^<z2iz6Lb<ZQveN92`zAuwA%$q
zvoWzd{TA#3wHB?<*1y%%zAT%-)w*u;UZ0L*j{qV~)<G*e;?f(|51?m+S$V^Z3ncBB
zl*Vd!ipSbZU)lQjkvAr!?-Mq3J>zkyuh4GC@LxO>X|=moK(p}x`b@lZ=VBjFhi;!q
zrgiUFK=7#B76rnGMf45~=7FAzvt!MH>+^uy^9F0=CXF45)<yhc87M`U$kyX3Ae^`u
zNKI34dA(ya$NUW}ZYt|?AR>@WJ)1n=bu8GyG3{J*fL}m9=<x9bmfFKU#mW#N6iV_Z
zNMPu<zaPl5F9v=%?LzmlX=yrY#H>IwZYNLTi)x|0Z^MrP9)jI{v^4PMTrHIuCnE&@
z!)UZkH<kMWX|GQ&iM(Xk4`rcFa%J%tqNeAzChfNyIY13R%%cV>n{A<kB2HDvfKF9|
zw6nbK<ZsLhJf28=e*`O<g^jSdQ2il%4E!2AdJ=}Rx!g1AZ^D?nYtK^wAh}tiwR%YH
zX`dCKEv8#RhE@xv(Dvsp-Kr<qPKYE2c&2Jvrh&g@##<@PxA5<q#mFbHqNpE~w|+yg
z?m(SK_t55C>=w%Gg?BqXq`lK?e}_~t3pSE9SJl2YSvu4-nqiN0F<~UhWC4|aP(QEi
zzHOa|NHS&6rS<!OZvQ;)Wu$Bd=si&<eIwDm&rBol#$T`CoF6P#M3Zb!GwUMt1Pf0@
zC!8?2m1i5fv&XTCtC4LHftuaWEe02#y8Sg8$hQ(eN2@2TIc~y>61%{X4bNXlW@dIz
zuwzDfQeC{ysTAmA{BguD(cstTETs__nu;_DU+gUdqdFzKnxw4413k8B(@~iQL7G%b
z!_yVcr~YGcPU2RvbOmN;H%l`WzwFo(91ulfMqF;i0OOG*soc?!FZJ_MEm96B0u9ii
z-VI5<^!z5=M5MW4jquv@fC6%pL>np;Q+6q9?V}~8Dw><vAF$b;$kuFN-rF5YuXAk5
zs){5vMA6but(48|?TI!d9f)$RQhoGaqH1!qCkiVep^ka|oaMT#mCsvcmp`#uU;8Iy
z_M#zcef1>9?bWS?hrpxu{S%%#wfdhBH2J1e&#+!A?{zWk$FAgtsOeDs;1Oo2Pev$!
z+PnR*#JE~TNqQf^+BtX43EgiO;4CJ2trmMEY&|t_GcJmiq<E{6i5^Bna??UfH0@Vt
zqhe#G0FyzTE$Xk&I{OLn(U1HXZ^#(+CBLH7q2YD9^cg?SKn-iHvVg9H#!aX;vX0yk
z1B$PqzUv$7SwTfm7pJ<F0xpdp%)VfOUum2Xj9z*cVP++Vm2*%?(+qRW3ta7@M=i9}
z&g;W1-d>xRn(3aNtuKzis`_E4Dv4;}rWJy4q`L)okv_86nuA{J3HI2*8>Pb0+69N6
zsCG+=Tjl{PDJ&{-x2j(#wUStzF`*k-^`VVR@vUZ#wakZxf*%a3mXK2dTtkhlskzm@
zs%<~WcjzE&@X*J3ORA85QjE@l*F3wS@(yi6JL@U5motfUX}Oy>^_d|2u>D16>glBY
zpb)2Qp(4wKR9Dl`5?)KzluqjY@SJtofMyX|%T`8rH^!?3)N+^oOX%<GIi_-lup_x$
zM1Z>000<0doOLVt`F)D_n`t|IUPwxR=52$H;+wcjy68b>G7`>U?g60s#B^^?Dwb$0
zoKoG-SvYswfdMYmW>%XCZ=oK6)^ZCtMT7HL@{A0lK7xdC2WTm7<vFbIDK!p`aCHRz
zJnf>d(q1MBk=;m+Y52yMuv3}K?kj^Z`ec%2qqcrxkG2Rpa@0_xR1fTI2j=-V2tDD8
zT%XR)7po^E@hzMBkTk7uUm#!@b@Bu>OSHEo{IcJY3u|z$3nT>Vn^qa49#BI6yqt8@
zFXJqk^L`U1;Wnx4PNxy-(kWUOK3X+f0u|0ODh~a;Hnn9LjH$c^h!jqSGiOHUjuywM
zBf8_etJVB`Zr_`8{|o>(qiW#xd4L$eauw5E)IXCM3Nkzi>ibk0HW82LDt{q*v{@JQ
z9#ZY@<%5uxCtqGKyvq!zo_A5vc38NJUJ`}`m+KsOA<C2$m8Bm<cMPA2owwuYXdr$O
zg<H;Ssrxj{V-k3^VKTW_!k>EVuXm)6AvePw>aKlwKHqROyGWDX>+q!aB&y5}X!@!0
z5+NJV5R&0a@DpCnLm<P;VGj*hp{}D98E+ES>qu_+37Om@B6_DQ@V=_A9rqEj5;s_6
zBMI?Mk9Ps=V88h@<*BQI;YqIM6%BrmZmwnR61ejkL}UkIQ_&ymaXfs!)Sh<UCUva(
zlF&PJL!fbo<R{SiJOur-75Qt_<?9+r1JU~v>1ohAyN?&rSgJh*gzehyzlRMpQliiY
zma37$#t|p8*E}FFsYGVl8>}wEB(<rF&AR2ctd-^*6$OeyCohat2IzGs<Rh%Pk*)a*
z4J<Y#nwx;E#|B5D=A^OIQ{Y5tT7V7CvhNkIed8E)<Yur?0?j+7=DxeQ&9Q-sNADcx
z%~4(@abBEd>807QP`g5t8UZa@K85pqV@C<*%bl{X8o^8DQeZ*?Ku(|mJ^NApj>%1=
z6FNU<MMRIfmSsL!djb>3%u7q!@w3uN1|ZQW6$cCM0of*{GEe&1d43*AfOv*Savci0
zM8XY!bE&KTc#$TM(){=<v19Rm^Ra=0G}W1gqS3nWrA%h?%K^~6q5179ucEf<_oh?^
zExaeTkDq|zMd)Ch0di=9w*Jsk+~6S>z)$epndhsVc|XE$eYl%3F=h_F`TXrCxNkpk
zxB1Hfu0rv^&C)Bef2FmYzn8b>5%^&faA@cGs(jvPqrZQX%;R(cwDy~{+_Ou6g$vDV
zyd)AYFJ@EDPcA8Zzb%3f<l52rqL_RPns*y|0?Wu=toH$6ABCR%S<ZON$FLh0cB56z
zsF=k8f`lhl8Wpn-RetveO51q9XahY?K`~`ut0_3)nGdRgCr5K)O3A6;8YLh0N0Hn@
zNlic$;?Xr;k`H`_CP4GTZo`=sX?+o^aORDOW8)+ItTy;=E#F<XC|V83H;@i3wMV>I
z2F*^+@`^(v?j}0^Wc7p%|Fm$hH`%~18dNdYf^;}gMgCN~;Eyvecj8}MpgG~%)%Tl~
z5WgAogrFO(fD8Tb_BF0Dv=pQjd4ie4kkAL~VXfK!=~qVYbr4s%L7;hw(*v#J(g#w-
zQ1R_9@4h)D0z)|>=u&;{@odthvLy-#)|Lcg7+e0W6Eac7tT{vGw+1quk9);(79!C&
zFm7UwI&p^;X(w^fP-#R|s!Kc5coJGgV|FmZ&KdSUXHcQFP7vjeWzQjKLtq_Ave#Y#
zQ9#|=+%1r{&W$j%U8s}pW6N}gOw!1r%AION{rTE>7o2&m6XWq%w3zbE_C@&?rg3r;
zi}HQ&ocGG>xI_Z2KoAe58rSD(!tcpI6cfU1fmxN6Qhl%)3Wx>u#)icstndrVDtwn)
zw|@0L#2w00blkmu1%g6WOG|??z$VMawo1rxh`kz~+lsV2Td#Kg^D=BGG9=E7G({G!
z%?tI#tIGHyZz^cM_Iq@Km`G!ZHgw!=lmT>7%#G1X+tgFOGqBt^(9mitckAdoUWf*F
z{6Qe|>%V6MDx#?TWWUJG-aWLHP1FZaLy{DJ#ZcC&tz(iBCw<;>CU>ql;{n|zV53k*
zvOh8;+w9Ga>+pe1?j=L?Trp*ttDJ37fbSq$rQ=Q;tO0FbBA(KTR0Ljc)?joI`he9M
zr3wkcx*wdQ_Gi|;swf#{zZnLe43DA0OE>VAP3hHmGaC=#Nk0&Mxi2sJg(n^8kZcEn
z*bX%j8o#En96g?BBg{lSjPz<iP_ZTwb<<FR>k9h+G`3Yp2SlyyE5T!M?hltckma1I
z<+OpQD6j|UGQJ66#;sxy_a=)!^W%(qsBrm<Yrpk0y{3AUO!)oTj;HSj;DxPO#eBTR
zOw{Nh{}b8mE|WCF@T@?_PfG*?=ym~--|q#60wT|gqzTiHmk>QV*7gx&{f(b9_)*cI
zCm6JTu<PPvRK~Pk->@VY8&5tSKPxL;VZ2%`i9IV@7<fc`HMkRD82wxm{t0F1woN-y
zVH{AFrE549GM$@&8&b38l9!}VdLlqc#O$#XMsbz$(_oXB)!He97VNoZjat7iONa8)
z;#t`~8W80L@!!TE^L$(|;ARd5d2th=<NuoUS^YdJ4aV6W<W81~Lh42vg3a^3q=o$a
z3vZmAO+=u_){A84Pi8#$!V8qkd;Nz>N4qB7Wn;Nse&wf<s_1lc$rWpMo0@{Z-U;pW
zmWO!g)7?H~SD65<6-A?<%|vLgoR~s{)s4}$;G4eIY?+-ccPlz#j~4~L+`0Bxm*JP1
z2n`})N3!D<L=wM_oWA0>;pNB8;|^{K@%KqlhxCELw4dG}gzyG-mZuIR$AoukBz0%x
z!uNp~yy4y@SxSY!TGByFq>cN%@clVek?3;GSx}Va;BIrf<?eQI`uChuC}KL~S}`A2
z9yADzKX-c}VImy&o$zPm#<LrD1j{~24i+974)2cyVw7E(jt7@sk|wX_V<Sg*3GVgP
zy6rqMI`11=7(QL~ecWEHrqBVu<w#9~t6@sR(I&Z+-iIRzF1&md$YEtHgVW{e^<HQZ
z4PlXI#e62|K6HD4P`C!7AMw|4Df;*%k^@z$Q-N%KMuxyt%ACj<h$FE+EZIY9^3`L_
zcOO{`?vC>LL$cP+@Y}*V%bF0Un{Hb2fyG0K?tasQO-GwT>yEU9TuLFDZ}@0ZKtNp7
znvJk=60?$X3~Zk=iMmgenAJ_H5i`#iC<G#cnn|-BXM+%Z3_<bXEp-TH*8I%}Jz}Gf
zw*2}#5D{is16v&8E<HBUU~;GHg79uhKDapZqVD)Htx(+eOpYhB5jK0e$$9`L(jfhQ
z{!mqF`G{|)U)@PwuFxGA-$v!}3&1#I&u-ZpqJL?yY;~1UdB-k~I6BaxPrW_PRUSVI
zoWu<PE_t6X8CI2f^XCcTGbYV<ElXTu%L}dKyMWiyKDKrMsF6-+<;!hqt$4G_9}-KZ
zP)BdY5_W1>hzaQ-rmGSpzkHj`lx_~-5z@WL27VjRSDd4{^as3L9mwB}CQAX6p#u6a
zGn(S_<NUfOS5C<U{P;S6o)e91AA2czz$*4d_l{d@&lDtqT55Wy$_6v|B6A!W%q%Fz
zF8N8%&ev&VAG=!-x5!&pX-W!hIW<l(107qdYe@UGT;v(B#+7HP6#La!3_hTQa=oeI
zJYf%~JY~@bYIEWi=I5)v&!L;&Mu?wV%IA%*??m-KAmtGTV(&w)kUecB-9gnMIpk6l
zk9#67<AUxW%spLAT&P%jLx)Y3e&uRhe?lQ0whF&dg2SBMM_bA;8Spqe04<MFy*@<t
z@LRHCmim%f;F9jdSh2VBIF+s_!CK>R6*aL^|Hd)##<9lYkfyjdAQ|`P4(b|6?N-lK
z@>$5o9XYYMH#O7Qno^p-n~vHw_RmfqsnQ%U7<#j^cIFenPX8Xmi@^EGn-fG>w8qFp
zfkKOiY_rxnnPt(sl5WvW6#^OuRTu}tG{=)_K@*8AEG$J)&F%M<zkJAY>cWXQna-*y
zRwQVpdqMVmMQGT9QzFhyg`EL({`0%X`ZA(1oMX}md(F6uB~PV(P)qHK#rJv9=`yiC
zh)8dJ@ue3?iQz*1k6@b*)Ow&(WeFw2&t2YOUo2_IMMm>_!j<oaf5giy>yPyBo8kNc
z05+2aLE)kfkTDFhs2iod@X}iBI{L1LIG(P%VnN=5L2I~g3ujM9<6u(y*TEKkvCk!g
z4E@sdFKg!_?!1j?+OsGZo<nlQO`0VJA-INoRt)B9Nz*;-6$4X|a~njnT#>Y6Tx$r0
zt_r5|L9YrI0_kFMR*>bz8&TZQ))O09HPP<q8Z+7cI8b)H!4%V$4`1kyFT|4$^s3z?
z)XD|Pw#M;ZG@z-%IH2*W)PxmL|JG<l_o<Kb)~C5uk|GiPaoc;GUQs`L);v-~=Epho
zyjTYA*9wHuFuc3Q&rp`O61Qt27PWg&K5jc1KNG|^dcyi!x=ojaM+J|+s~P=I)XVvx
z=+A?`m6$=a*x?31X$1UBu|#*~JW4A>H#Vvi4YK%`qJU`u7V;E!4kV{9%KGAds`C_&
z-5!0xkP|lk5c%XW?4&gPU6#dzc0{N!;4yzfN@&2MhXjfUa)9|-iFZKw<h9DZshLVI
zENR@VreP(8>MM@q{5B^B&Yqd?L`?UW>1#mhs{=uQdMO-7H(`XHH5l^M{Ez#>#HPYL
zN+o;2rGDm7kM`t7sG%EbDWGI6+2Ab@v*b`=6eXsZIbpkQ5}~F?+n&`Too<l5-76Uy
zB$GgUVF!#UkF@}7_iy07d691eZJA}G`jj^;`qozFpk0(@r`=u(Qx@D7_HVa69S}AT
z%1-JuBDEI%RXouK<ETqyER>mcx|f89dRTs5U_ZWln(^;B&1A{iVpnL8c*|WU|EbVu
ztYoeD`Fnv_iH-LR>tG#Y1Z;;FHY=$2IK7%xMglz+y;f7m2LgU1a(>)1x+9*b?5+48
zp5xj%8_NYLnMQz_ZPGY0h5i_19q^k#ch-WB)VZ0F2kW;Cv7#h_4JlT=Q!vLtb5FSd
zp)h<Pl1Oky=M*EDa;U(!{Tq3lV|H_M6+Mk*Uw8|o>N_pg6zdBp*lUl$A-r2yQm;>E
zI6T;b+P<5gSR(+lFf+pAYLZ-t^h(Iak-}DWNEh+-|KjdFqngaRzhN9l919|&0s_Ja
z3JM6K5I_Q`G*JOj=~a5KQbGvmpmd0c6s3tMD4oz-(4mALkxqa}389A&NPv*!J?Px`
zfA0Hz*SprUo=?wneQ+(qb*^*HKIiPS_is1ACh|k&|H=GqwT<<p;2O*VaHFA!5(U0{
zVtgk#)TAZf{B(lyMM_9NvWtg3m7q%fVSsOE?5_egKA}82&I#(pJTM|YVCe++bEb&Z
zmXMLwHp4=<8b`OxY$VvOhYtaJEXOvV5&^g=dti=t9<Rcw9L8GaNLHt<<2*V+0m%`N
z0pRlF5951-?L0o*nPqFm=^di+EHk*&9=)DaVm_!iM#{#yC@<s}6xYGQJ<`wy{DJlM
z*5#-5%oliN=Kb{3V3@K_<Ca>vw!>Nvsz3I`9A6~aZ62>e8@WUNu`+ESx`Mn&*HKk@
zrzjc75Ta{Gh_jL+=^EqQ-XO2LTgp#EW|30R{NHY|6JTTHXl;I-_~pk=oLx)G$Z4c^
zEnR9#^ihAay25*tk<kq%r}Yo^3nGplT_HucTLywo0_O3zLD?|TEHP(Z?EkpgpDT6W
zo8v<uR7qo}J_Cy3aPG>FtDpT!^w_3J2am<EnA^k&3XilxHG>p7-B2$892tV!Rx|&n
zPhh?MH_O-G>l4OFp@AIbdeJpN=quR8sGN8>^YY=p?C9#%0ivZDz*3uThO9Eh(7w7?
z{zGDTv}AtDd)faeR{%2A|2;n>>>qLhh|+DI?JW5!G=_Z0d5~EEY6CrdU5FMH2Ild?
zkjKy7bGy~<Ik=84jQ6PxdwplSlT__;QFTk{>hyE2{@BOObNBv7=y3xD8dJvyxfC}@
z86s$TtKS^8W~IMjIMR}v!$Z&AMDh7|j{-CQOtTO<5}xTj_j)A|B)4gzC~bZ{U|hVO
zA=wDE`Ig`@0`2svzt{xCHy{x1#Yrc2J^Ir1wdwWy-bY`sODe9F#gV)ax3LZ`p|~2W
z7BP`b-o3i9m(zBXOcK;^J0X>be{f8GwZHgseeKncG1C4vGjB{XZ6Amhut0t6!O<gh
zZ?4sY>};2_K>t7x_@~?m`E$?r+7~21y%uqRl^A(fop>BiLOONQ%~&2Swjk6ZgI1ah
zJnvKS|1yC6w;Uz{==(TceJ@ViVWiZVz#Lk51wx4)hlAoTtEcqg57im+p(he(r|yvA
zc`Jb%(B9()Qh%j_P!49XeC@!aesm5<wDsdr(gt%0aBXP<1$f;|kDbX3y{f$%6FKKK
z<r&AjQQgB^A?Z5gQB4szJ)8S?@rKWtVtdZCd(UUUz5c&^PJeiY?Y;275)0M`@*$R9
z|54$F?6v3QcP*p$DR8#+a%NrR7ws(S&vM&Cpb(lM7n8(D%pyh5`o4-7NaCrzh1_52
zqX(bK)5A5f+MN{<v+c~ZN%)sAPZxA%6>PF87{j4jO=8h^PSl6q+umWUf{NrMZ9FqO
zee&4~a1G6wJEJ^SyFGzrSg9oTtNP4`!-W*T!R`U)&q|n|?JiFyE7BKuOfF>(FH|(`
z789Wxb?M9mwJZg>-h``lSO2nnGh8niFGweA=k?cKobwBPj5<OIqK6QH1M2(UKItZk
zA}8EJ_l<n;q%yFD-<cS0<_VeRQmyQDe;sHU2;4*0QB1p~h!ElfOl~4fxn9AO#l%_C
zs`hiTd~{h~Jww+uim2!Z^;iFrrL1m-yM{C>?YBPsx|Ep>TuO;E^uZH5(v{RorIMYd
zOvcw$R#bPI+rQ!U$;5u_)(ZbL+=8xEceU=Crv!eb8&?Ed@h3L#Iow*k?pNJal82I=
z@)*7or@sKV*kPCIV)!r^)&s@UMD<<ON(3h0GR5EDvVNjY5#6t<oL7#RTbrWR*I)e;
zw)A?~^G<D<Qci*I1M_McNp;^IZuhx!VnMfBHMf5>KwndrW~H*a>7Es^Wfh_pOosNk
zvzk12nfhJs(+vnZH3OG5SjV`v6l;-bl;%)tk^cYdnQHxu7E-XWVL~OSv?YUDU8j2V
z?wk&UG~dTC+H|b+XkzFJeIgPlOi7Brn`P!ffWSjlyK(#!(~Mpma9IUn<CZ9e+Sx!o
z&oG&KC7o8ygo$MZulJCnf_uy6mzP9QPYHdX8u%gLTc2^1a^&t!iyshFOilzq_yqXx
z*H?L3?Nv3|+v32i49-L8O)X^A{}eKT{qSAi`)lEI`0BpysXQU?`5VCbcA}oOkqTTr
zm&jh4KexDEYqMXYBc^s0pn$=(sy%ONq%U1Q<hK`W$R-&N@__r+0K(i@@uzn>v=p9r
zIe2l75k&D6nM5pp3P1eQQc4vBeiVx4p!{>!%%gkAg(v_y8=cur-2eA5K4*eV&zJLn
zgJ-F#VvNAyUB4eurGObKgYw7Lx|9u6=GD=khq`nSIE8Y*omg$iS5=H#w#ex4*c%FX
z6|f5HeXAifL(xpnRpqqUJ>+(c<w^4R?4Y>x*LSSo?~%c&*Y6xMeZG-@+;UUcIOeNO
z$H9~LX4CiIxKY5nLce*l^2R5=C5etf@wdib5385hyt{BHizx8+&jWYA9l8Df!i}lv
zduEBZi}p!AOJO_s8y3xCm0z{%bqL5}q-JDf5NY+rtkKuX*zS<sfVs?CQY{Iri*-}K
zHOJQ{5*{`4EnYSE{sqUfJwfP&$z50LRmiEiA~?7BZ{Lk4?5<DRB)|TR<ARC3{lfS5
z1|Y%D00!_ub<i$sZvNjbu4{uInjJP#g2y6wlKzf(rDuZgg!jzbBQ}gw1N(FcKg*YY
zM(2Iuv*r-pMEU<fox%4LodNtCcIe-}5Ny0T;1wO+M!2^-G`-#VZx<4_)!SimpWigh
zHd($>`;++f`_E`S;A{d0XdS1k=(lC_uUAFEluI@Lg++rtI%NONh`IPcX5$~v$Zs6~
zs;A@jj2)D0`SCz*h5>B*$HSlfWiMs}JiuGf`d>_4)9FW|O7N?-$p6}BKRZB0ZH(Kl
zt{)20il^TYtWMkA8_IPIpuk9MGhv1(JSq3)z2j_+(5!cT2T<YJ*dC_?I#M&Zy{DW|
z0ICzyO5BJPpoKv_<hS>(mM53M?1xCZkybHV57p%)7kETK#!75t<LtU8@?&yOjmq@B
zUvUdK5wy3gNkdied#itb+-)DnP^rpj`BD!kEx3Tx7?*$NLyHuO+EQ<kC!Tl!ic(9l
zo=wy$DMO}jop#G|i5vq7&hfu7L@<m$kT@OY-T<wSI00AKYf96DTtxMwoeFY5PFb+z
zYC0kFDbEk$WCeL68FE&b<sZDgY>(J>dU;7-=pFSESC){@HxZw4&h?s!Ss=jzI|Wz*
z<E#^0oKaiZk(_fUJ}7~lox&1d`>S8+r<_`m=GEagQU}0_!Key)mm;#N`o`|aL$!gB
zi_<j9f70`FGFYs%gt%TUt;7?EP<6u!ke!LJ-I8j~OwmlNS1!i#>3s9fQ{NqMLT%kU
zy%F;t^nCY)kD9FJ%PHc4OdU#8l)zA!soMcM<3@EL?VA>;^BMG#rnPIYKFLY1Uq5t>
z#4jv(^^L5$yIzO4YWs|TA^?McKrYNz_{eiuBlwyy=`7-gip-5hpWR9jeUPuYAgwhH
zeKVdEhsVpl8FznS*f%Le1eE{lcqQZfKj|6+=-P;ooSw+(n5K7PybTC})bJ1^m9mHi
zt<C4)$1_}60y?Xm>`M_yQN23aW{m)M!wzeav{2l=l-)P$vA>h9`QN(*94)G06Mkbx
zq%Gn=iBWvhxp$LW&>m^^S>^n4+VBy|p{lVG^;xeq17JB&7FkSQc&-=F{*Rf>pXC_3
zoHeXIyEYEMGha&7n|z<($!sl(XQ+*AEzY=zX2c3MYk8oVXjSPX9bBjL&yCIrrnH?N
z8Q2U6pBWj1EYA1YRkQE*F>FcGN^H$i-~9&m2Q85+EV79wfu^LlIW@D?xQ2;%;;8ih
z8bgx^VW#?3!eY`)lK^?UZ&rQOIHNhmVa6X~80y&azR7BCa-e+GPY3Q1U*jSR`?f|7
z|G2`bxOxya?y>&kUQh;0Jx60k3UKePa^aKoo@s$9=u3^^f<3o4mPDv7g}S5G3}Q~l
z1l=5o&nkZzM=%JZlGhV@3Z&dVate7yj)f%5guKA4h=%U1`PUO*%|ifDPUUozfN0gg
z01xWdoIH&NC1L(7mLy0hR32oL$C-lUjBD@Q(nc$T1SYgzW=!y(y1?V^66gotF|7Hp
zan0)YN0DPIV*l6+Z^dMdn{A3?+UX8+FTRgj2R~AO1KDtb7&cuJ=1(ri3?Ja4JbY(V
z@_?jhn9^7f>LRN++*t0cM}1MUs>5{WQWZ>48<=~3ph@TpHSxcPEF%=tKKTudn}cig
zc!pYVlcJ3|M7&9AdOXr?FUk{(n<)s&cx=+b87~?@SOXRE+6C>MOW=<EXqT<64Nqa8
zNK+o}|2##4-~`5W2wH#>XqBY)QrG2jF$m>E<!y)whd-F6M$)w&uKS_og-P&?xS6IC
z`V4o&UZbU+vz;YpbiSPh%P!F$BIYLcAFBlz9XIVwWWLt$wFtqU=q-$vxd0int$xvp
z0yH4WZNm1i84o#@GLP!x^!|gf01j$w4mqcK!REL(GLoio_oZ485#N1+7N06>9!RY~
zUlAsySTNktbDSE%3V*KFDFR1bgWg<*?ug}oe`s%^!Sla=Nkrb*{*8@3`GIJzEGf2c
zZXJ~tGD7}wM;2cH5svs@-TdhG|I&Z1r*L#?{hZr~rDw?EXlrk8c50AQOISeGov`U4
z8^^7_)N7^!D*pZR+HWQPb?KHi#tSr5@E*-P^S}|AnYxg?g>#L`$l&3LKu#E{9CUl}
zuRFa8Th@#e4{t88amfr{^seS^{SIr*78y>ad=CHnGItvF8mRHPAgCu;RdvDR%*oc(
z(n_+^$XI+LCM0q<gNxHZP^gkVn{g#nC=Ti%<#|R~Lrg$15GKRh@HXA7b+9!%hbO83
zGkXj3)ZRp%K$cIoeju+ce-UWh7oq6pu5ZpCN}v9%dB{H!N>!UBN0DE^y5WMPi#{*}
zbhqtkJ8}LrWNtn?rLC#<420n1y|aLo?4_@qAV9yq8{zz+MV9I+<yfW}_4I#M^iWtF
z+=j;4rHLUK$7?~d>pMjEkAgg1le}G)0@<`(QCYmPg`%mzwV;}GcGHvey<klcZfS3P
zG6mDz**w%1RIGCEWO&iUaoBRi7Sng0<jSt$ZMr@1%|@9=OR7ZD(YZMxjL04K#gAZ4
z32!Ezia<6NrtdceEOYNmRjmnO^)!=9#{>eMWCU~@urA)6pR{7cnkEs3N;ZxIG38~M
zPn!D>kF!oc8X4p6uvJwT%sPSJmgpMQRF|@P{Pu#BgriAbwF;wqI$k>TkKA4X<uY!u
zcfp9q-dm}Y6fkG`A9K#|UtT*AGqpA<BS%`R9Oh~LE+4I?byT_GhK{_CA*zgWU7xqA
z>iqf$X9DJq2vYIY;~C);=@@gBl5~5EA^DSA3Tk2fuE@}x^Td{BMN%L`qF7M59QIQN
zd|MDN5bPzxYj=QgpJu`#&)q!L8WdgWX?&8{);tzo6t_uie8FK>?*Klplmkav$vau|
zuAM%H*+C}v%eoKL@64r|Ivyj*3aH#1)wZ%&s~~17@g6OAtGB;=p?Lmgb0v`7bb=@k
zuu}g1sBcr*T%3~#DUN8fn2aec+r^Ljb5>&#yC%+VPs>mgyl<hQkD@N0v%+W`!^TI5
zrv!K;Ce*j%S0Sg7RUKd-X5Q1J-gJxTY|kw9TmiS(O|4^s;_P9rHu+JV+c7a}S`&_y
z=gTrOPLrRG<PJgH+zVOig{;0R+-3sU!=$a)E67b$i8q<Ay44jb`B&M>wTj{Tx_2pZ
zjRvlz#cmB#*Qgn66|3!E?!<igBf_Waa8Bd+@1bz_vU1Aw^jaCNlQ{n*<6WQ)6smN%
zzc%PvOH|h;d5!;I^j|N6&wzvpt!i^VSobUIfp?%kH#^QQqTN>Sf(ZP^=}{qS5fB;8
z7U#2xElqKgj&hwnJf{vKYp^ayZ0gOrj(?l6Nt1iO!3TCqMH1#*itFQzd(>jfC5?jt
zG6IvswW0{w>@RgFoEPH*JY;o@KZIVb_X=AnOzmf@m>ZeWYJJ*kAS0)Dez(TVwweK@
z3E>(PwgbJ?GnXah&RI#qZ6)E}&kFM7C3Q{9tMLX2G0&?z+HEamGSsR^Mdj}b!f&1)
zJ<{3E{7G7yoFg?uK3RinsOhizG!u)Qkr0&8YERip<yCN$p*KfIq2Szb%*u4w_iKuH
zA>OCEV!Z-u6oR`ilQw!|*_RTg)`n~@2a{7so6jY1E;aF#V{E40v)>72Iia3+FFoMO
z>WzIl{bqVTyRh3)ZTg&8#J$~H>4B04Tv;}wg$lc$b@lC#hal(H17Wk%ja+x!6U)Y5
za}BEtx+n6VLn>#-E+4SB3mvJZ0!5&#A=347;3LKAV=0=W^&99njjdE#e;K1<NP)YA
z_NUKF_C2Cz>=AV>sbuyy-4*_$f@Zp3m{)kyO-=&u;l&kN!?rh%3h^5T_*A>%k!@(c
zCzY;DK?$vxe!Wl!Z_D~-pXC4(nBW1&9JTdU?e3UkC!e3)Hs!8$d#q5}ZaYaLzvimy
z0LT14S9`hr%hl*F-}lQ@uAbHNl+ab&eAJp~m8HDkZ0r^`npQsX$TemfO@0$z<o*Im
zZ5CbUB{)o`Kz%=ryLFbS4Y%;nV||Ks<hdi_RqE1n_pvrUv8I<xj)prXmrq>f%``k3
z0XysG$idzkHjPiBx;9U(#eT8&(p9Xl<_<Cki0zX~7ql?iNA_6i_=&vp-hcW!pM2wb
z`lYn4ou&WV3r9PKU*<}9Nth(?!zy&D)c<P__$*2Ai|Jby-*A&q%)dbgJd=7FhkoSB
zFN|cAeu2)ps$T5usZJ@+c%jCYl9(YY5KQR);&s{EE|n&@(^9SynpIMP*r)B?$-_BL
z9T`^sZqzmEc8?x!bJ?j9!xZNBc=k+#Bci&Exgx(>GqWmPKQ&SU33_VaX2n+619)%S
zjqVNjjY1cNU`y|<k*-d~Av;G|=z^cVQ!(YqfVT6I=_lfPy+c(xZ95bdz#-&*a)>{#
zpj4O5_LR$Uv#byYZ%m43{3ojTUlGk(>aYcs0uEE+bMx{R>Vj|yySq5DdYp*%+#<3z
z=vJ1!RSx%-)6X<zAEO0IuF70sKSB%Ko#JdHk(i&!G#10B%D{6cH-e`mcuQ4^Xgmnd
zTJM<lwW1Szij@!gv$t-Sz^oMyjynh9yo`5U5;=zTSAQwx&gl#n=ycCGxGNStwbm!j
zJ0zw%(i^#T{bZ_ZWAs(anX16J2o2tso;vHZ_1%QAst)7OWAx4W-*B^UesYhqRj2Ij
z^jZSk@MHVeNA|B!i;FkvQZ`SZNu!+TnfM2$D+#nWc-9JwMhc#yQo2Tm#)Gi*-I7r1
z`%r4WPnBZWm;E+rR*RNfMW?xu;_N+^^W^f-jri#XBm<^6oQWJK?Q2D5u_}Dx4FE;7
zrP4ODq;~_!ANtYvdn=c`o9KRolb7dqro6mb>nFF?m%6r-TFC5du6>sFVUNa#oGjGD
zb+5#l3tsM?9;-;Y12dO~Eg!PClfEEAZ$?)N)g@yvs*IPu{{HNaugWv73aqBf@Se=z
zyFv*kF}9i-W(1XrLL31Bl~}WWj%fNTC3f-&)4RLex-oe3JQBI)kaX>oF?mgoX|Ok1
z{E|wmD*PK~vTxySLHeTyu)98^g){+XAo)4mximc9sUo}XcT5tSZ1GC=_|cgv?q+@I
zq_mt;H_uz>cY*2+ndT6;{k5+!axZ?|D1Wh6VeHb<n{Bf;3;T6z8I!*9e<-gHrIBc!
ztS(%!;$?x!#$cDC;+;7y27Y+g3h$5U?JeDeAD6PZ%nPkbUw;2c2I_ln9EQZuGq!@s
zi`9*FHFTq~Pwj}_8FclQkmK%(PrPXG>V{G@Ed(ppS?jMn&B-z(S2JnKor;E=W}#j>
zWR5JxV=y^s;A+wuAQa-Sh{>NV6N2pB1U)-57*^JFKj)SWc%x){d?GqDcy=E_VA&HK
z_}r<|nrTk_MrY+XN}`vx&&iywR=aScirt+mP;hh;Y8q4Jdl9*G7OC7fb7j3uuW7dX
zf&Z%8^~HBzzDp2~MikHjgQx28JI_M5$jHjEg4JAlC<DJ+fDzoir&zDpJT*$zDE3vL
z8(Qpsp`FxYPypSRXqhES1vR@u%AeA6F0R)wnCr{97#4MfWu-V=><JAaO`j#P#=v%J
zq!t;GEY(qmkWFhZeWDEjBX+y8x5Fr3Fd_L{v-0T|h-1E4dReByn9zn$MXeLzt!rCv
zc^~ii>w(FI`=An!3Gj}i%CFXZNVm8>#L@Bw72=?`<ngf>$gRc^IT0wLaebp^cr%@Q
zPw4IKdy=7#Ei<Gj2u$#-p}AIsp3_D|eNJBi{DIwF7|ELcGY^3!U+ryZOt+Y{4m4fP
zQ&P5@v%Uwtz_g&#=vi6*OF8HbIVE>$=cQi6#Ov0JR@d(J_C3QRa0M|3gICK5=1QoP
z&Z)dz%7oUN@+fmbm93oCmAu8C(Q~lzkWgwy^(mpyuMT_$x~#N%9@g4QP`AAO37oCQ
zjIr<DioV7SRGJ@)nQqe^mgk5ui)dZTA&hIQ1Tyoax6(>f$*De7a<uJ%PZ*(XAr{rO
zYgC~~+Jo=tvBy=#j-`~O6C|h)A&ww<*Y8VzCVbWUazk-?#>b-Y#oNnSY)%7dSwa3l
zTuA^7sd;J&2rY((k&Hi%O_xWQ(_u%+V|v%_o%ED;RFdZO>g>?HMw3Q{;;@Y20LE=a
z+#4fAmxvX-M~(G`?Bk#N#d>WmaDGi$Fv76~K0dE3=*wFz&|Bm_3YUZ@^u_9M1njKw
zuNsZBY>TQAXk)omn2mBI%!h1+-E9LaKe~4{Q=4<cujzy<xhKPgXwA9nTcNu3#3)DT
zc7&Oy8y|NoS%FLsRK)2Y<VG7U^5fU@Q!<$!PVg(H#E~|V_iJRob?f{ogm<@9R`MQ`
z7C$CbUzZ*>Ig;~bCpjZyiZbT%V<R|?C4{B4?He#}c9vbp%u*z9AVVnoEiHTf|9U&3
zsC4XWy^`JB+ZxPU-n+}>t!AMhJf87%1{iz@ZUxN#Fu|RskZmE$HGJ3T^2&%m_Pb@r
zl#Fj`cU9v;nxJLF`V^}s-PQoez-fG|1NZWJ#s;XVZizjpJt=fKf~G=9Y$c+pT?F&`
zW6MXQDOQxvKE(k(HQh<!-R1+*c&}x%-tAed%1L{Mr*8dl5q4`;XeZ96N+wJg4i!vY
zx;d^>3)r|v-QEQ_HshUlA-=^?^&z;8j8Dp+xe%@Os=UzY2b(Kh%JefOzsBzR-K~U|
zvAhBey|;`D2)|W4x*fK8DeQ)KtCA%vI?nt(z->q5rE1n*inZyIMr9O<PkYwVM|7lu
zk5_!!V~RtCS&W>3lSITawyV}3F&n~pA!K(qFUhyi<pE%g4&a;u=KZ(W`gIUGcl^(=
zFK_FswN4+?I~p^M?5vHeXTGjx&Kl{So~%!IpEyv$+@W-KJa1TvleVhC*y2}CwjMdC
zLB2|GbDgFIZ(P?Gka9Fh*xM27slqA4no?ONz2BbMMifU$Q%@7j`8~vso$>42*C;z=
z-Pjkt7|rf#ZKc5aiwK>$?)^5&dmSS8D(%tuo61j9O)s;@-`L+m1K=qHXar$bd?dOD
zpKPoLG&0j8f_D;&@GqyU;OYMK=X&Umn>as7yUR6YyvMG=k(hdpgVoGDdB%<h5|i<+
z#^c_}BK$jhW8TFIeJ#!j0hWh{-3K+TK`zL-s<?4MoDfijrJ~RARaKn@i_hO}`?4$d
zjAEd!Kn?=vysFh!?=b*;EU5c>&%*oi;pHPd=7Ak>+X#7(?lafj<wA8P=gd%D1E=*Q
zr`HMe<LdfTc!a{i{cjJYb$)<C?g-MeY4)>Tfvz&VkAvE~0Uy7}WZYG(e>L2+nNOUg
z#fU+^*&`e;Z+%lmES}f9;3z*GzrT4#zJYnFI=S4;J@10b@0fPSlyY#Zw?^&PAj=UX
zC0=4O6@o7CKQwJxd;9o~_x#5gPQKr9ZO&YwXK?m7o?kOgl{k-#g|`Dfd>Og$lw8-S
z;IRBECnCP3$^9rL#1*LBj;DwWe@AKSX9iH}_a<o00?xTmocfxL5%A?XPpI9!qo$u)
zwy``R4O`!wtu7MBHf!*VI>6iacs&nIsgGG&nS`+l)K+ZGhrGT6bL$PKjI`fN`A>Fk
z>`EB%Q(bGq*Z5ioHu+Rp1jWlaI6hnma8Sh61-jR|wTpslt0w`I=~VJd$nR>I5Xq`8
zSDP6p_mQz|IHCAC01<Goy9r(VA}L5IN9S)JQ#RqYixqt*T(1aNJ`_;Njeu6PM;$1Y
zHusu3UlMy}(zkQ8G6*|`<guzrVPA5m)qBGI$WO3{s&rR8MDFoil(CytM&tB`pJ(l2
z0*x`vvue^?wdk!^zZLD{le<h-;1!r(Ru(*{WTi;%6)~DL?)7NV)gKpVDvvvwc)DaM
zwf16~!0N*=KU)Y0(!<&L{t!ap(<4(nZ8xSgSIhZ;`9UFY0Aa{~l6Z^Qq6Tjs;%=Q9
z;pH&HUH)5{B3uL8>u*RUqF`p1{LY8ZDeA-#YpcNJ@=U+6Pt|zJErD)mS+I*c#4ohB
zK6L7HROou!Rr#^EvQoN-<9V8=FkPeZ*DjY4XaCJlVfXv%q$gKAszYem#AQR?44=+3
z)w^F@Lz}&<B}TlhFqS|P8X5P>3%}<674UJpG`D1gt11va0l?#tUg9yQQV^6-q~VsP
zWrH(^mu!!4d&UiSJbDk10d{aO+zB!oQk$<ln|FFIA$jt$n7^q#RioRea+HS7HlEg1
zB*fS9oh8-Gyh=;1LpINa`&^AR3mb#JK24m>HTnB2{D=Cv9Q6-A7n(KrKo?<6o*k~o
zx-{GHD~e4wmLZA*M#kRNT(E3DaL^U#BgYk?O4Iu<&B|o(l=Qc|e@LDBsd>4}Y<}!J
z0U~61k6|T_3H5vb3i+e=5*roi_9SGO-ztOwEqiF|$9FV7<$=5J<-tN7f{gomCA}Ro
z%DFyru(KTv7)*paN@%u>6D2nMHW02joXam~Sb}+b!6Q1*Td|(I`CW~v2r&pS%oZ9P
zIz8><pHUn++-E!y-i+~(&{btLq`KrUH+<qfCX`*q2hjqS!hU~$=G%0aVXwf#^mqHb
z|15hA4c9^Y3%M`SV}~nsAh&)hVri0jknJ&q9exaqMMSB&5)!Q=JO0^qRirG#T*dss
zj6q>_5qk|nPmq2VO)Q{HKk=#Z-sbPX-&VZCziv~~ucs>TAVot`^%>3D%YL*xsJ8dc
z^*w`~>`Ls_br6K9@Y(x-W4T9cVw}#Ij~?$FHL~2rQ>r=u^A`flAK(E$F$)CHzI4c>
zzP`H>9?`>Yluhb~8+MSYJ}A4(-9l2bF)2}a+_rKLfg)R23E=6K1UFlo3}B4l$g8--
zU&MI^p)Vt#B00x>soz7AJxq(|Akw@LkYdmJ+7G$<%FIxf7nOU1aL9HkX&Uy6K#BV;
zG?3L&5-<XgUdZSlKXnN=8~EMM_PL*8MfcPF1)IN{wRmkc_(aP^yMyQficP(<ozkW1
zCan>6+<CAg@U%Ui*uBNHGE}KV84p*MO*X7u4<m{a6!F<P5ixt4s+;ifKy6XHrxi@f
zo8hr7FOYdEG%Ic}&NA{BNoi^hn=Al>eD@T1!uSqHM1;j@e1pP*f|%e~*G&7=g>(Yy
z5r6v!Xz^1h_1P5^&#MH+qaTZPRqGhj>s69blJAm?fF&0pC)?HRByPc(?Bxta|525d
zZfPtDx0i&QJPR-5NWd^hE=D`HG~W^7bd1k5xjQRE321%F&fm0;dR4h_-%$2M_PEC-
zV5S}vJVn<?uNXfm_D7LHORFp4)Q~uH=xvXA+tTc*dY}ORRA~OK0K@0j${H|>V}SJ(
z4;xRQM|Abn8!k*git#8R(-b#YIh-e3Tgw_WdpTZWn2}IB?0mVfm~?cik`TFCOnCiW
zScG#SyHh-X@ccR?hH$){?g|ZV0g(nn!n)U`x)Kw7{EMb5k9JgRk{OnLno<<`6OYo&
zPyAf2S^syj%LnO0;F8W%WNgIDbb0Km6r7E?>6bOg5xqi8-ru^~=pn?5bDPwot#FBP
znDJApHO++*2zV1RZL}R+QpW&l*S;?4<73i-u!WEwg!r{s_?*NNo<ADagkDXq_1r~R
zK7PUDweRbTk&<6ZAzH^QEswHGUmy-mElz*!i|KS8I;VRpL_u&!>713mIoETP<uO<8
z+HO*yJI=0<wW#1-sZ;#=zm`r-i?dOd+4xwt{GA&cYph?|M+QWaS?n3)C*lR`q?pP9
zvi%=tS{L$EZ`!LV`A1LZUyv=okM;(9fO%a%-KZDkkkGXxoD%1mn1Gono77St5jFec
zKM^bH*S=<K<Walk%{|n$dL`b&XDc<^bsyXOdj;A->mtqOB01&R{!3+(VxJ;CvDUIJ
zMl)*b_8P&3xEH~fI!Q`snr3x}G5*I3pWTr^=U|hjuStFOgK=}7x#r{k<0=pP5@X|$
zK%dNvv@e$2!=Ly-ddGQ`!Z4flN-=!Mxz$m{_gpTJ8EnqWpK6|hwNBZ_Jbd`|1;9C-
zPiDp)?U0v@cD1$O`LyL~6->+15Qrj$5Ztc}`93HlHvERO>zXYSdzH`#@VLJhTghkb
zo(!)*?5A=O!AkmghKD$NE7?Xt!7!a!sb%r`0NZCsB!z>`R3`rg_V#!~-ner;_L6u>
zJbtwHuEwqd5F4*fpOqk_@V)d@$X~9XzOFw{D7^t7D$bO)nIpm}dr07NQrWV-xR<=v
z4@Sw~*&gx(wFLRct2EMm*}mqB927D?$_`_dspW{x`PfQQ8m7M|<VNV&Yn$v#O$wFS
zPQP#$%d4zpwu_*8)LdhN!g6L>Sd?=@%0Vr=na**X-7dl-2CMXS_g<T%_D^<P=6+EG
zF1&8D2f_g7xW^XhC2XZ!o`{{-t!J!yZ|4Ac?;x@NL*Ykd%cBvPX|fYwo0sab-?W?G
zDV!`z!(1@d<9jKc*>q7bUP~Z`rNsB>7CJuKJmYn3Y=?M?)r9EGb{_=h2msDA=YMP$
zAc2a!N-&f1vH!cvJ-fsIozLfnUgTCK2x2?hwPn*%AiR27rc1nQ_53h{8tKw_dRR*!
zI@Lad<UA2_?p;t5vRiEK-X&jn3xg7MKWp6-(Pl!%a9OmM2<QgfZt)eK6w<e%ig2m&
z_C4PnT!}C@%sj4N#5%p*@{1uz&##zqN`J+z{CJ?xuWX;M7%F#|SKYUkrhh<X7fYPt
zbhdyMeGE)>8*DIm!;DX6D9dib-d;lB*sE*rnj0ATn&=<v^*3!v6UiUfiH1_Q=OVoe
zB$bd>v8ffk;_>RSW;D2qTBFqZdaw;=E$fqt?`FovrIH0BVNnJ`x$c|wrHDiVs0Z|J
zr(RM7{~!%JfBTpO0&Bk$9vNhLW$qGXZ-fVwMgKa+!)B~;LhN;y;o)U_RRhR?d);(4
zs<=V)6lbEG!9saF$>}v-jX%lQVj!=rchM%*q>sQ@c(+r<k?SsfB4v<ZE=u?wd<Dt6
zG#wx;n5p+_eIyU9b9aZM1hs6rNN)?}*T1BSUM0`e_@VQv29QcbSk|Dodve&|^HfBa
zYd&8gQPCeStJvV0xrpz0k2;I&$h}mrjp()Sj9Fa7^c!?Pm@7g_ewC>$A6@#aqURNI
zZTbsW|3JW-L}9&GJ`s?GU$Z~aE~7J{QE3%ng(USQrIp5Rib}!lC-(l`<p8{?n#ey@
z*G0WD+J#zj&;K(Bw{a0kFS1c73VK<4yR}!6UIbQzc^aZmele6Z{C@gTodG3zh<xS}
z9qu$uaZ7~ta#R<U`KATM%4K`vteLE;l&G#n7pX*_($kxGn($_zbCNx!dQ(-zXH*;4
zv*m9jwC#{=F!=zNkN5^9mG9tSUD576OAt9@F-53i#O9IkycN9yfq<2oJY?<NL3mGl
z-3QO85ecm%gX9C(KBB|2#BvR84F^!+t${bb4ZwM=u<y^T2R4(04`78!6g`nS<$D<s
z?5@-t4fvbcec#?skx+duC7pl4ioVh9lkBTaR+S3&4?C9`JzqtgDuJ6gH0lIrx(`%C
za~Xy*O(~7KSC%k0bPPv|P?PS(ZXa=Wt<81oZyrJ=Jxdxs_yub%?I?FsU{yLHMhKVX
zyEwyrO}e-m4(f-yZezJVuMJH|b(8&&0(7E%XM%l7QF5BWoWZ7FS26rGwm)|VZEv`J
z4?o0xQL9R(1ZQa3In}XLj7FN2m=ra+66}S1U;B(gbBh-f=M-~46`~|dpY12u8qMl{
zZ;OOUh>5~(wB#RTbTvdkx9N?2v&GV&vMG6cqSgW7s!rV)lNNjW^?iPxwuc(6M{MrJ
z!E;%Wf%Yw`kqUwd2a^}dxI%v~>kM&xO?zl!KZ;iPZ&xYU6Gi8&*Yw>k&BRvVh=Y%p
z&T4exQ8qERZ!7>58mfjG7Ds+m!3NGU?ap8hSv=Ni=>z^Qcp-8eS5{0QOhraCh4rCt
zc9oVzC%|OOCZW0Xn7&zy;F{oNl%$Bt8E;x7sZ%VsJvzc=;7PT5Hw8ET&{Con+kTrW
z6xTj%+#=rPQQx?3=)9w3I!t;ri7nUBRfnFNju2K!1xE+ea##1rxkyQrk?Jauyh<5a
zs}^40m7(n~Xs1j-hjmS=80PAgS-hcESfnu_9Zjl&@sCrllsTt$+S{upht-C5h5u>7
zp}pDm%3j*kU2TwUy)V*PvHi##>xxauOpUvrZ;Ju&XMzjoX3RJ=YC(?Asm#+TQO9te
zu%zZh->qS;thm%<oPBTY*f)%ME|k01cF{im0bmyMZ+A{LBpy2{#G7DbV`P+jmtGIm
z&u8f{SiWaKA+nSbtL%Zjt*Xb6TRhloAUPITIZQ94Wp*3SMZa^-FZe6p>-lj+<z2nR
zovPP3gx~i!i#R3<_a4CdjwQPTzDl3Il%aPE4<~ppnR=dRs&=m^f{o8*+AZb(Ryd?|
z5lPwT9Fq`BEJRIegzUG#QlNM>zlO$nru_+6#`_R-sgq(&Cu*rGYQ_7dC_2ZFDag%$
z;yg;QX)OF>&lj+enEoLC58^Bfyj6S}(U%nAUdm~2ljWn>A--fW3^$E{O)g!@${SO0
zV5sDpIE=zVUAvNFz^16`5myo)^oO{z<T@a8l|d85t9Qf_niCC5l$fEls|G51{5!9@
zoT+zUhs>P&l?y5PIazV-f|{EWXv*RDuwxq!if6t>HspVLX%ywqofAcSVKg?Iq?Ob#
zy*t)tXUqIUxa48UY?5iHr!0(T^p5+kWg0At)SyQbKG0EPKa|Gv>+I!DQq)K3lmk|c
zzuTL0PmkVGb>1Q@-mcJ2h9SGq+_>|cACniH?{BKNOC0nac(!BU2h(8CG=b#5j3Qku
zLM?}s86hM5HaEV76`CNrW*{GfW*w00N7kbb<rn1lyLMq3AKEihMAjEqZYMYI`N~aX
zLHv@o^Ot%A<2`wGR;l6wC2oP<>VhmCdfa1egV8HXyxR?tk`GiKwl$3PE`{_{sWnnd
zOyvp&Y|g-7z9-+ji{#M9j%qB-k-xQhhE$#Tr0Pq8%Bv_u2O)%378!sPjUh=0VGJ;`
z+wkB$M?1H`IxpZhDKedy(@`r~@8%&tcu8nc#5uOhH-3?|R2`XWpM)(x1W!7%4a?_J
z;a*6nTY>c?pixql(ZlL?X9(h(bxXn#VqMqfUDT^~$6>BmPzZv)6&0x(vlSmH<&lQy
zJYV{}?#j79!2zUvo1vOlZ%)X1MWH=jEwg)8IaFLx*301OL`4H~RMO<5gD2VTZ4diM
zom)FXUp@0}5AL{?iGQOBEGy}pV(hwVr<lt+y=OAp<m|N&jr-`yG-1?~qV&0sT~jHP
z7Vq84WCXISfg9)g`Vlt&aHPQ893rP9ae$^;9^U=hLG{=Sq(fBv@vs58&Pw8O;vT|u
z>gjfxQ@%}Janz=Vuf5uYsJI}wj2OADrzl;fL{fHFk};oBb(Z^@F3wVvdOn42!PG2>
zF%c4y3F%D$Fy73UJHm4v+LJqZb+;OK;;^ns_QM}yNY2TPix1Jt?{=UuM^zgD#b&65
zcZ!5*)i7o{=9Ed+*m<8o)VQ8Ku|`bO4vh-aw)FDyP`7-z^)a&en9QY=&zgP<f<v`4
zy@~kStC4f=Tc5J7m)P{t*5tT0>4rpY9d6B~YfrKCR0AI|<q*=xhJ09wCKdX9yE#%L
z9Bb^~XeE>&28VBRa%~dY;DyMpVLp-tdKqp$S2>byz}F1+?G+OGgDFu>1a;HLvS1^`
zL<{>`=Ua}iio4N0X<&BCc0Khpt)eH1!_T5H&|0Mm)9ANNaawLDfI4<%+88{p#TcVF
zCY^Cz>b1+r{8cHeVrH7&z(XS6>qJ<*^SUSH&zM&rabv=cY=LCGltr)}<>R6UsGV>T
zxDH-3S8-4w0zQ#GoE6*d+jrxVk9vs<K5g@r;+W-dxJYtyA^|<`DBQNxT5d^ci_DMD
ziJAR<Bvnezcv#P)UzkI6^Td3Gn+Ee8Df^nDHtce6NXs?xqw5rgv3jEzZObceG;RAG
zxWu53x@neY#vdMpI2)?{RbwzjH{t>*!KA<X3g_FeUthv&JX}=!G^F?Y1IA|l{)z@I
z%?T9A`}MF*`OqM5Rldtc*pBDx*7?5LYWMjsXORJw?x!e8#See={{k_B4ETMTe%d-B
zn76b;?C`m{f3ySj4q+A)Yv2|ZFh;$iFGYVe^gM8|whdaX(BVsK7*W1-&2(D4tN0EN
zzP};A@5y}z6cr++;0dA?)_3q-nA?JSvqi#9Qa0ac0|$Hp^8~i9E*;80QitmPG^|HF
zV6{D27sP2-`zs{Zm18JW<&F8R%!dt9?_3mgy9{Tg*9B21(r8YHMa;vs9uWyCRD_cz
z<TJ}u{k-1Wb<4U<L94+Vu+EG?h}SSnmD8=5PiOTF?XyGD=A6cU80XUR0n?H-ENOdK
zBwusZq%oCbq>{VHb_u?3ki<1P;Fv|M94Yhz)e-OG#h@xS4KEYsW&?_D22{CkLDvI~
zaK<uOb@#YN4&W?D?+qtKALMp~r@NV{AlXp~?kh6MZeb#b3nvID)%+vmb#9Z>pF)bs
zU0mgw+MNOp2cLJt=GK%%92kfR@h-#l2IrC^R+z!1!8SQ^CgfsukA!BTScaBi4qJ~|
zQ-5_sd;zI>5&wXved-oE!c}-(d6t$?y;d-(HguJh5!rCyfgmjwsW5g5(Xq#P2CN$+
zgKRxAcZpG&ABqmIG^qA(0wQf^N(t!{8&C~<+t$gQQ9DW9+bom7hj{%{1_aH{Uvfiy
zAUY?OdNPSH<h-87!>(DcIg|O=uKi&<8vC~!=Q=zME$`Nw!bfQ}-L7ROugi1!o2mCR
zesg@@?Wq~dLYKcI<6}il5)Mx+R4q{2XWAh~-9}^iznL4g#&?D%R$7otiyKyr(h5^O
zZ}vAuxaYUz8%dF9>?rqT8N*Q$&Kk`^zly?T-qGA6m^{s5TfP0!g0E`*u#8S-q8(!L
zilVvwXx+GI!OlmMP2X)?Vb$miVWuW`vOiacPrrokfKz6?Hlk}u)&jlxrvP_rQ{>0V
z=J&Rh<}6U+5S>T9mRLNmCiV2vzz6oF@jfG3jJcuV6QZu$=;0NONsz&#H?yIZQaoFl
zg%r>!)`7f_=M0!^36SpqcN3xf{&~HV>n_WZPIKfE5tMcQ?4euKn_ailRpUN}VhYe0
z4a*o0CzOnv;Al>L-EX22GV8N6W&7FI<J*>~lzjycvXw@w5Q4CVen6GHD)C>24a#y*
zQ^jM2fyPFbv;rN2Zut)#_-2TqP3?E<s-3cCP_t!Zq!wTOMJ4G6Q5Jco4Tu61f&SU{
zE3o)1+zo&Aju>rX&y;wV?tM9EZ{3+kevcjvdD~%--a1WrI($O<o(cL-c9Mrp@{d7)
zM2Ij<S*8g%oHiZQcT~K(qP*+Z;;5CFF_>chi`m}k+Dyc29G>JCUjHns$7i^JsGNtl
z|FW<RHV*7BN@t|ASZGYU<$D9q!XPAh`Nl3hrz2_nWLf-~5JaPNeEW$A4MLQb;ltTd
z;^vWG?jO5K)#57HxK+@E@oVDlam|-;<5J~MI7J`*`YJygQU1FaX)2i|&!j(6q&I32
zYjgkyz9Z;}3y<C&aM1W(C+`0cl3utH9pPs*vLfd0fn565LV9E-QD*K{TD?f?GoZux
zm(5;}M3`T>ZxNY7<ln|kETxM2ZdFBgKU#%TpdqGc3=l_TH;)_=vzSf5P3pyoiVb-e
z&M3RzPnO%{+C()Js^AKJP@b@i^Vmdbz#8IR`YI7G1ST)yu@y5P8W3{`&BTmG>uTxR
zUlp`hotTaw62sTvsd5KW{b6W|zL2HTk=Y%uZ{<T*i+^Z^g}1POiXHj1ybt1D3+oBW
znkgZ-Mv^VtHd!T;xCWG#OQoCU;Pv{>AdD(@g=_Os6~^~kf&SKVq`>`#9bBw4p7kyZ
zF<v<QPAMPnhA{ic;-0H*f6L@I-+cFMkuggOP72Qhs$OE2W8xw`-o@$~iI#S%ZV*Lr
zt8V(eY#2X;$2Q<yA1qmHP##A?SBtLA8T&Prc?_b@)2dnLY=C@<*UtPdy)(K<FV)8A
zZz6Vuam}cu4BBRuH{Oi3qFf14mN2sDS~^!SYh$)<+=%t_L?+6T&lK&zoz^LCH!D{R
zvWmnV9Wt_px?T$~oJX<a>P%4GRP;;^)-}^7pr5(~8Npof#`*4vx)ZtMgM*=I26yRk
zLoEM3fnt=~PHUvOf3T{hl_!(ix1#BG=7~X7T%e^PLeIj0b;(f1In-dm%(A?&XOr-J
zaue@ed{wKs5S=DPG(Se#Z8Fk`B_S8P0u+&-IBV1GRFhM#e+v0CLuSP;IQU@d^Bas`
z19n-%eb$Y`KXijX0!!?PWk4~|W#%zL{r=1<n~z5XPi&r9MJ=6SL1zw(Z3RHEx@gP!
zVSJZ(7UGR_iH7_@sMVmS>ttK$ZH9c!;*B=wsd$PpL0N7a1}Pe2<};R|cu$7B40fMO
zzNO=kr^D;$VbP)bZsGzTEWKz)#lc|L{|2L_EZWcL*Ew44tq<4lk6vyf+m^;c#0B4U
zFP)=jA1w;aGAOi#r`sivoX+5E`?DNRfK-C2^pJ+{*2#@S6Q_P|?V1%Z$E20uBzyTH
zCaycRit|%Eju=IT0<dGtUp}Pw9)4IaMmx(ni8f$AgPVI#DlGPdona~KMUM*&DE;c!
z6M}t|Pn|FTX|9yH216jWC8Q5Wc3yVDhLzEi6`iQ-2L~lGueM2i^8GEAs6G%dlE^F$
zS%WUH9Gw@HV@n)&?HqRB+VmxEG)@vNpSNrsljXE-i;(G=Y;NLCQ1k0CtqS=QwH)M#
zup(;3hT;cZ4YMc#!iyz|Sdg%VfoS~0n5ryAF@3C6*<;XDq`@DTcI{#bYsg5s7GIKd
zbS5ABl3vRg$SFrH6BphvGBGAp(}3X$eXi8Srl$Z;kP^*!D^))^kkT1mw?mV|(n&6s
zM1)o1WIxbrP0esKD12XC6~;hSk0;g>e!9OCxr2w0VT*)bOH9;~%X0UoyDt(b=o9P@
z@SQF0SORKEr$7=Xx1Cnpd)0x#H0+qkZ+hA&1#LHXY_ywHd1tB@l3ZBu1{AAWgfF6Y
z9C1s~0#BHZiQ^j))gfBZX&C$k*{7m_3P1M1fKxdWLsQ><+ya@E**u#vLts>-R!shY
zR6@Qhc-|e`9@HznYUo^=DJI$&LEel7%B$VGXlF5KVr)t1H^+3pn{o@Ad97PFb_28T
zl~^0wHyZ4phUz!YHEidHs&=6<_LecLAE`%P_rv&_+b-Srme*lY-z=Z6?S#8bQy#az
zcMlq)nXzyhoX3~C=13nuWrh|M1|(to<>-n?6HDUG&OMo|MS^_@&(g==3TFc<?tXaE
z8Qxl9K)z?9Wl3{wRRa_h$M;>U!ia`MSX^ETF7_D2Dv{P_x*R+f?eSYAiO|q4S^r)N
zyc6GLNuQA#DZyEP(6gVeBKjn47Nhn`EqvHH^cgDS?DT2CA#Af^4;a%<g3pkAhqSRZ
z<W0Su*qP9)A?q(#O-2W9wr$b*RQK_h1X}qw&fqay_#swFP*2eLj*8UBsH$$bIkj>=
zTzlK>eP7@t`=ndyPHm)iBeeiOUg(Y}Ih<2xz?WfUQWeuCp+97ht%OC}ese7;xEV0G
zH)A;@<>4=LDv$WZ=k_`2CQA%%j~C8hiXksPNGhc`FE6Vh_sYjrJ<NQ8bvfRB0tdY@
zv2=Uf?Q*{S023DpliL(*8&2ftOk{>M_`IDIaZL^)XtY&ejSc>=yIU}uYIu4rGjcbb
zGyY?N{A0qkECHF0cS}02>l=QlXVSJ2j`o_32mKqL7MbQ+9=d<2+Q`1`9mzXs*oHFa
ze)JB-a5v6qe5G={V&(D*5U`iww}EoNF_!;Tr^~RRIOk?{6t@37n1XgI^al=p9ovKB
zO-^2MzVfL|d`6iIZZZGTK`#6!-1c{j<Nw)!`IE|^-F|6Bv7Gb+Y;6}L_U5#+aDDr?
z&+Hr#tw*kOny^{NqV{TdzRhAMh_La$Y-|m0p5Cvh%zF|Uh*`4o?q3C=S@o}y*e;vN
z9ge#4J1F)EO6q<NAB4v&D6+E!d?@_&$tWuLq2@1ar9~iJ;>g<|WLb2kEt{8^{dtg2
z`yBXqRR)MM=9gVK9B9<K|8vHVk2RYAXcD*A<b}3!n|N{bK{n80x%C!^dfs@t*W$`j
zX8>sN1PqXR<rDNr1NXfKPfUI`uvUFw_{(QEP2qh{w(V<QBRjR`MS;Z=Vx1{5X>p44
zb4c#q=p2&5I&ZNlm?X!6ze|C?n?m%!7z?KNjUe~Nl9q6b%{pY$=fU=G6JT!FySQs#
zdL}d-c!IvS_jSsvP|%$BQ%cm|pEs?zwbz`<Hlp`G%@aFy2qKS>VAz*T=MJOollMJw
z|3Mq>i?mk>Y|8<Ooj3pWf=MpMxYZz(ZTVp5ZMM`NFyYHhCWPE|t~w1)*C)w^K7>i}
zu3tV^{ph~yRRHLoT3}-T_U{+C3%}VGPUow#?c6mv#8!Sf>Q}ZodA-_C!OvE|HuJqb
zpkrP(+;WZWZNc8?aN>mYE5AQsIAo&q!2CdoxzFs%tAE`IC(LS3ZigF<sP4ZG`gyWP
z1Fo1?X<&}*fJA#cqWR7IzFeNuZ1h%P`%KXHjgnuVaC~z;%I2^~^~assYzm`Sa$e5c
zbEM_{s9XmcnR&hz><=<&o&y27;J+ZawZNXV7^ooo#;iTt$HZFg-=Rl?h6@*qQsmr{
z{~cpAbhxoV0<>iZZLI}~TDzt#dBWlL!7|eZAXody9Vr$t=8VQJi5MzI0#h9uo74M<
z3d^BV#jtg*Cmey}&r|H0S1cMR93}Ie%<YOu5Il-q9k<Kc1By6Fy%U4dn&yx<>GMu4
zQ+?;y(%p^#>W}ae&arjp7Cu{f3#I^9+(VWX&*=)b+o$u9|B8HE5{ZvW9q8ZD63%a1
zw2EwRI%A~*H?dNA^J4&P)Ds*-vGnlBCE@k_9D!9@0@r7zpYZfqF4dFdFTw8s&WzP+
zun#D{fc_9$x13?Yk>jh;g4GBP30N)eQdOO1=4i9cryF?2>BDZSSH?Tep4gigJ%in~
z4P@UR@3RBIvq|&TFLX1=HKNyKe{g7vyyonSDQT=&+w6K1fb>?p5kpSDvt1LBMhE$}
z)C-45PNTV!poYB(>MC61p6!;SgBsZ&khx?p7|0sGWE18$xm2-Od2iEvq#(u9AmO1^
zv0!cdl8Dd9sJ3p@R#N0538Ygm$^3n@DT}B)=Q@C&N%-wamEH58Rrrs9-dc2)i;a+h
z!DOhcFn^I4EVRW+)9wQ+fN+BCt4zT-p9sTRJ^ucEi<5`h5U9=z=XBz%e_{Kaykr{Y
z2Z6FaJFOSYtq|yI>)tQTF}5P8eGaC!8}dm=%8u7|>YTGvhvg~cX`R)4&LZ6TQOZm|
zfVJ;%8E6QUPUO2QNeoK2i-~?D<yo4ktbTBL#q??E0ZP;8mc@!u`Hech!vbto9pcq(
zw_sn4Ar6iui}>iRp~yvJezvMn@tha3a({w~&B}e1>`R`=_9<Ho;qsu)1a1Q-9O5Ed
z`!cB6V<$7x#3(eANB_(dN3}H)QdCKdKYbOH*w9+#BBOt^rvD?HcrZj9uIAYK#S-;$
z>aRXKa(tv7i0`xy^Yc5~S9s(x>aV69mI;sGzU#xwey*{%slB~z%R+If5K!LJEyqCf
ztcl;RV7Y_d-{BY<t<eY0<)ZIgj%N;vc)gSmUOZT@DyUy=G_wv}j8k<sEs1`WRiq-Y
z9ns0=bf@t?;+w|xO~$|-;l(&?|L#q7OFKJRcD6c?6JYA21=rw?!&;8NJW(;WKl#!?
z>+(Lfw~hd)un>xd@T(TbZY{5lw^s;0g?EHa7->E{Sig|XHToMmnk;y#Jp6!=&@m{!
zYiq3rW)Fhi^WjuGAk^lGh{_d*YN`7nvkLEZ9j+qADjRjM;5XGLC92R&ZQ7=a#5+^1
zN%{E>uEbeKz9#I~3O_@i*R0b&s}k5hs?6NEwtjfbxmo4Xy2+EoMp2N-zL$aDi+*-d
zl_ULje7jmz1_W~#LiqNxSqxE&WZ0}#`Ae>v0m}-wSw8$2{uL#-kYm^^zdup)4G1~T
z;nF#ZF_L3G%-tN%loJSqd}i9HLv$V0+$<nAO&5eEh6P{i4cKAHE*YcE<Y5uiBQ(<-
zonp=csIJVekoo`t(O7<^7`<A8{5X^lw?-4x2{G6Vq4*;4ju#AQhgtoR+R6%~t8c~|
zoxVqL!OpuE=dB(TVV<+uy(+x}W<fAm9t`md4mDWGn>)QvSlX>a(CK3JpcS9aFV`I_
z5r9*UaHsJ-|CNmzLlJ3HV%a->G16F;``t?_ZRKp2<PozE5*$OY?`EVkW}DZGg?^R3
zCK-5s7~!oB>3tZYXwwiuxw5zT-|lJ5WVtuU%(rr-#EIs0hR>+Rrut96pb<j;pW4nn
zD(P&C<CJBsO1*SV%V&XUWof2jmX9(-ZHbQNBWqGeBhfTU^MNmFobi$kn&w;6lExBD
zElo!Rozi@#S^2;RR6Z~?5eQL0yoXxNU-#ep4{QC_V*P$+?X%DR{l5F`^Vv&Sfk|ZC
zpOO&$yg50@5;XP#P*wwKD*gJXo4K&SwxUCuZoJam9`NXM|9Yv%^MEJ3YOb_`v6KPI
ztKbGx^+lvX_Aax}JRxP-k%zvZzeN>N=*4?7J+u>Y^^r-sLlAIL$!^ogzOBuIUuXG!
z1ZHlgCMgLP^cGu-Pd|64NLTmKp)oHb_mRJ<EY1$ACC=I2sm+UKY!bGl<#Q;Rnn*9-
zqNx`V{*0{13Fa*R%fip*ToSjDyi;GpCibAHy-;%X19LHI02}Gcnly3$7e$H#j8?~m
z$$L3hS7Jd|=1}MgevULP5^>m5lG=}<{*yebVZu?cjn8zx(C>?QXpgb59xQddNcJ-+
zEv=T&EBp(NvKCvoAvaekqt!!V-;A`*+b5T}9=TGkPBNjootv$q1k#%_E>R8S?lU`j
zoFHd6HMB!+G;v4w8K4fYyYC*UKH=*G!0*q=7ZXs?15qWVmpYpHw0D)}_d=L#9rVkk
zIZ!aYeD#C3+7v$iuqjL%`wV)0bTu>2jP=;gu6Loq>0&UtTS=OiVHi7TAB&Jrc_T#a
zrS?Q{lD_fsP!b(LS&5y~wGU=<iPyx&`eziaBG_!Y%CbmYl!IrU^=_&DY39o~F`g{+
z=UHoa+tH6(aUJC{kaZqkxddM^peAdKR7bYNzXAbON~JFvyryjHJe^Y6*178EkImA&
z<$AfY%4;l!wSKqSeZz{S%trT>+llKTLv3JfeW;=#w%mfcLS}E<Y)W!Q#*|fN94x@_
z`kgLv+AS(3F0(L3ITJNGv^Rmatf+dHlJ%Ft6ugI`8$=p549=o@Fv2#Uo+O<Vr3&Xs
z#VVuT{fh@ECEEMdMjnYgi!J`Z0lJG``u-M9xNDzrj7CJAe*e3=Zn`9=F?1+FuRmIX
z1KC7F`>pe_gLiO^)!oXIT&^heR1+pxGzN=I=v(zlPd)6u*D(p($S(=sFE+NfEG(wp
z1Yp#`Aw_H`x{gub4$p_w1iUxhdEP6D>32^aNWBhDIBKOK(8rc1sGEca%!C>(*dDmH
zAH~?s^WCYvu9v;4)q=CRq?3MTOK5V}bICB&`ZwHOb^u~Dr1ecH%H~MWu7bNRIZ=};
zJ?L}8z=O(ZH3kh2v|*9@1vv}exTmynUxo?H$-T3kYXd2ab*`6eJcxIfjQbn=edd=F
z!+`!$S_iRs@Z@|jC{+6V^1DA#1<3=<#>FLKsP3LxRm+~IvZnZi0owGTYdAgh%a}Qb
z6tH3<eRME9PQ2QrDe_|QX$*an(B|2|R05o<!mf>%n}nSsOMt@($2lRnMKFyg>OFVU
zbu>PGB1(Rm*w3@R#}=E%aA?);r5h2UZ+9f#McFuuZl?)?O5Ks~#RH<9_Sl}`t|Xk#
z0!MD^gnTSVhzkbeg)n&Qiqe}@o09N3u;gpd2<t6fif>Qapi^dVrjbzv%E09v>Zy(|
zNv8=*2J4_X(85}1qfy^ou3g!Yu)Ckk(hh<$uX^gw5r%BNH7z5~F8a0J%;-@PB-)Eg
zK2Gw<(4NlzWI+26ulxeVw{U2L9Z#{7t98n2(PEu|4q)TQT7pBbkJpVI;c6CM+=$$(
zpb<1y(``y2H*)Ku6;ad>MA5d*hM;CDNxkda1|vMHckJ3z7NXvn9F<#St#v6nS+a&a
zVH!?*L1QW5p<o~VtKNI&9myhCFd3*R<%w+mFE2{Bn?)R5%vPxuJ*X`TKT$dzhg{lR
zVU^r39G&Dfgk|gdwq)%?&pl`UhOyszILtZuC#3(js0?DLSh*+VDgSmz`6~w8Q1@qV
z&B7_W7RgxRYyGt4s~aG772oHZ*|yJsdT=WwqKo!Atgd@L-<*Z;*3X>U3l$cd%N>xp
z1ZD;2_mwtvYn_#ZZUt^83=^|KlD?Vq=xaMLs3Lf-pXDLnYuj`AJlFx|ZtIBI-(B=}
z@}CjoC*!V+p!GInsBJ9~pRe-Aje1*(Pzy5G6)O=+bUmi6+7V=#%f#3|U5pQX`Y>H2
zni2Ml^YC;JxjoXW)#srQPT5oJd6$7Gl7o!Gyt}A7wfoo=Ql`M6Pr{ZtBBLu4(QTEO
zb@wA8z6bt#7t3>)5=}d+(#I9Y_^-$GvIx?D1Ik7_7xXmptZ$K#k5HnzzNSDMRmrd4
zY8QpqRH@<bENLwT%21+rss7H^$pl8wEWyd<c-gaIZ{r>i^yybdLz>KB#vKRn`ZIL*
z=v4;6;8O=Ia`U_+Cin_sVWyAT%%U4PtV;7^J8!x0#HXNx@wG1)&F{u9hcpq(?Liw>
zsqguQ_BdyNe;B*}#KRd>F+dOxSn@L@q-RB{Qh$BcAAZkzsojGIg9)MF-QbRTe>^7n
z0<ykoYU3rQ^q46nhHuWO=~ABKfBaiagRaJ@&A6O0?S&gZ6eJ%-TRt2jSy6n;lj`Ru
z_zWn%x+J<&G|U^bby01!;3cVQ;9FXD3``}3em7%4#Do?+<Cf9tmpIp^C&~(607KG!
zu>p)JY^{QAVrlx4lB%>eP4pv6E(J$hIz%9RLXHoXks%ysI>^&EJVg7Y%V)6L>&Fa%
z3$)(kv^NfuW@|k^iverLjnt_^KSDJ#*RBwDze#*OzRQ(zhr%w!`%T*3IfE$clZ3A{
z3bqvMNM_Ywg6@3U+IG-$1dp4MNc&1&4YFrAJxa^h@MW2;XJk_1ICRLH?Nnm$-nc`4
zAZRGQgN?f&PhuV46FVf5p7s^2CdODa&JnURqDB4Q4ZR@>9T=D#L2_6R87t>{6~HsW
zu0E7<yQX|Xqdu!OHN@#}Z!PK^H^PHy_WsR>a0b0l|Aq=4NU^CMld=3hEBZ_MF37Jy
zz?YwK16o=oFUNYDm<pqyI0S$Y%<6-l=H9cI6D<7&Hce8e=Rf4%Tj%>FF3+}!t)5Am
zMqAuyZI5{NLtKWZ%)ugm8_~faa0v+j;1Y9wB)k%|L<ROVoj25?o>>}6?!`tn3^%lg
z)dPB#LQ6p)`Co#eymUbM7yA4O@GQ2)Y#lR`gp)t;Ea%7j#?IaYL|+RS?ylrRV%dz0
z$n_t=OVc$`e-8l+*7v*APCzqT3B%Kjheo2S3KYx24*o5oPD!HYxZbrjA`M4c77>f!
zTtbn!yGo0?hHsO?0_T3H;2R9Z7w<<mCRQdTccE5&kV`Ci4sjlRA=c(o2ux3l&rMt}
zIq`#=HU9<KmI70Tii!C{t=9ih?QbbCTF^DK`#Db4UM?fjePdQ}`WpOC#PhKGRzDOq
zC?*9^1_bu(wy4c{cU5Am-zt5xJW%p%@SpIZ0Ri7F<>T{db;$ing?yd01oSTL>m3X`
z!mFB{&5!q`HcZQE_EpIQc|19NVr>INkw8cRnE}Xb3jO49g&}G%;855nP1Up){n@pD
zejMZ_wi=rr2!R9!B!EfZ*3)%Yp}*hQDeIPfpSAEk@JvAV*;KV#;Y(mFv0rOazlrT{
zmyrKO2s^>scKahgQsCD532D9Y`afhb$t-py?$L|I(a0^c|B%PN_UVcxvK_5K%VlLd
xS-C<`M=j_HXqF0N*1n19?~nPv{bJ8}$ku8%nDIM&wvQ>E;^OFrta1qa^}l17L(c#J
literal 0
HcmV?d00001
diff --git a/doc/guides/contributing/img/what_is_an_abi.png b/doc/guides/contributing/img/what_is_an_abi.png
new file mode 100644
index 0000000000000000000000000000000000000000..e2ef9e4d17bca6b7095813608cd46d56c5c7e9cd
GIT binary patch
literal 151683
zcmbSzWmHvN)b24*TDn6-y1Tm+={R(Ew{$2CM_TFbZt3m@>6Vagr0ed3Z+-V0_s`v9
zD7E)mbItY4xt<yO1k1@tAi?9ogFqmp_wPg%Kp>bs5a<~g_66`82;blg@a2hvf`l-r
zWSC$Z_~E&UkhBm8R346Ss|N%83}^dJ%>e{L?tuP#(qofn2m(E1zZVr!a?##PMu;O6
zi@zw8F^<a9)t?wiqw&N1Mp@4i31csWE=uNtR3iAosjoKmsrEZT47q8aLQM=fmuHUC
zaP*u{9O3E{T*!p%pZH0EBVSX(?MdEZNWGSfi%+^P*qPwUWtd!4o}Ba?j=y?H$=!5R
zX*qoGEF3YqZq&+60Ob1rz2FOoo`BYT>iBZc6)J2BR7=6s<21_5iU)ZjeW5tPRZspI
z*e5K2xk9^G5+~m}*Uu7rce12^So#9dI+{mtK&Xw#hpD$QMG=cp*jP>>zKxPA0gXMm
zWzl`o`kw(npf_bC;#(!^cB%Vl^IGVI^Ha;7r!qz$Nf3-Xh&wLRT_&FCQv{43*zGch
zvUg<NW@p=;;y;fQ>H1mb`0cOL?BeDZ&DWl3h?rv^A;O=798pfgdCjhmOs7A?lb&8l
z{Cawi)%~08-$&DwaP%4|Np^9_1Pn3jkrj6?p8-?O#a;2`N)wWIAW)K>Qt2Vcr@j35
zk^L8>h7@F$ga&(oH?V5Xn^X9|ZZBJjF!r99vUE~jkFCIkUNxMu-eYw&p8oq(+W3qN
zPMvj(l1he)%}YSEswbzLsqC(oeJf&^-7+gS17ZcZ|K9EsCO+sbOW9D;z-=~pT|B+7
zF!cHfbPAGFZO;=+g=AYiQcET>#{zRGKL5IO5!=DaR83Cf-`bQ5o`?zpx{u<j-;BZ4
z%Bsj>|AiDUZvjQ>-*<b+HJH(9iBaE_%-bGHCn~f2Ow)ZO_<nGot2*|_&Iwr7mDD>x
zv{(Cn1KS~ZKd*W!4~<!gWO73`n!K#(a`gG%1A(>(%zec=d_~p#hqIKqwX7s@$Rth!
z`>KrR+l8_%^ZK{M62BvtfRWblfE7^2x$@=G>Q+@GwajAW<)G<`Kt{D<BF{WLV`mN$
zR2RSBA0#?cKl^8J5C|^_4_vJfubm$i?^i*z#OjMOr!lYb!ZL{I<79U+M~M>B8X9!q
zAOmAMZB?V+jcD*^t1|PuIE5u9T3?3}4{#U27XMMxYhs76t;F}@eNvKr9+ESk(Lyfg
zaSOFswGl2TXP*6x=i>7WrH%>Cw}HPp()Zh2w^^vp5NrvLaB{Y8k_RUGS2W6`N1T)(
zG88c~r%u3MvP{yK_Tu3f#5zeQI}Tl+P46oY89<?NBr5OkCLR}O<HeR~$X7V(gd_Co
z-*M*ruOw1fJzc6Sgo=G+u}->nJucUZZNoqFfw)iw5HfChCvS(=op$i=^{T#=?R;5O
zq$tIcRYCn%6v53m8|@TT_J2@OVSWyp7pLO738O2VDhObL1MtSl?0s(*Es^43%%x`-
z-|~w1`<Gp<c{aOc>V?->|BB*BUGed|#i711P44Jiue)rSvex3wFi~MO3$O}X!9}@`
zJ}sYe3dv16N8BzusM$YhhHn4Il3P{Rwo(+09mo3hg=fq<JCZISWm++#1MG-M+64*S
zh=781KgC5`7-oII*x}Q}G!#)5XPB#UvaL4%R|b%-6wBcx#R&OizpLM<SOL{bOZ<GL
z8;%9{apTF+V*sT-B&TjS!zO-oBRqFVq=qa8>&zHC|7xSv#mIUjzbU37+r#O7F&Q>Z
zL%jq84#N046yPtaUxn}JE~V}=3Qx&{n@l^V@Tc?1|1*0xn|XcGFAp}A6>X{L4xIhN
z-$E6D+!5`KDP03NWU=e`$|GR(?h@g2qW?U_S77T+L~8jyMV)|jz5jp;ok6wbzP_m-
zKNGM+5Ynl;Od_{rD(+7k)Xlo8%fJrx_AMCv8ZV1?Z2!s)0x8H{MDT{ARY<X^k>+Nl
z2wrn6veZhmIYAYNlKycm6wcSfNxZlV`TkQ`G?P8~M&5rsQ|lKliQhw6H@zS-!|&!5
z!ZMK}+wqCt5la-L37-HEc#gvH?boZ=iSK(qiJ!V(@$B>;Y|7N<DSF%g$J_*Ym})j-
z$ozg_#*F3{^nJ6a&ynJ7P~33>Bxscvs76)U!zv)B>HN)M`IM^x-P`zoJgb=4?B)Wa
z<J8DRhhHF*=9w(lAkszzao;#gs_+@>f9=g8r6S#}dyW4e&Go0Ve{~XDaJJ7eXU<W~
z>5We#dV&)>7(ZDPC$O-05{)N5o4u$=p6t?-JpCZ~Khy=kI@WV=lPckSzeDp<#(R<l
zQf5Gzu4S%5zVMNmY(4p9^VZh};=7Jg_QH+t&q(*$YQ+Y<#qw+4m;~Y|%c@n|{IAUy
z3=T^C0X{F+ND`k{{J7msS4c0HXaxg*uB*vhntLdGzoS#a{zcEPX!n3<U&H%ih({Uy
zKWj;Y8BFz_TOV8OP&rkOt=m`mC>`%&+^`72f^la&h?}0LSn4#H_)e?4o`IzCl0ix1
zKhOHG3l3%pQEKV&?6Z*?T3y5hQ+!t(BLW_@wryGsptAkXy{0x37+k)7quP^(u=qRr
z(P3IGXZ2bI>Dd6nyJ0d^1uEPxvF<<X<Rw5_F*tz-1+NHLERKc2*_DNu6;}FAt8*W@
z>|S(z-Xwe+bzN){#E|Yst0<jgNo+J}aJb;M_OW`AS+4A*LCT|(OYg9_4E6kO&YY2K
z!w-dQ6^n!W4frUqI}b~8W*AAN@Or}d&Es*~`!2S&b?Gvd&TG5-fhx7kZnq{t2_47a
zr|TcEHph>T*90lB>#~T+G$i?gCrCjCXk19ho*V4D`h11wnW}(q>|%DFT4mX2x;88v
zF&Sn0ee%$Ru!BG{vE@oc7+?10S|HIzc9UYRCg@?i0Imub@CT}<s2mqHpb?(O6&(%b
z6#Ev!6%q`<vfNMRaK0p~a+@FH%N>-Hrh5YD)Hq+(w`1A5`xJc4alAxL=80dkRI30*
z7v&$)BVI~A8B$vV3qlj`eN&BR)V-xBt1&DB#|^`K_&7rvZ09>y_D{nrh27%|L8w<+
zRrZOOn0e;Zgcy5mabgR)-_!@y7)olDDY@((k7PJt8-(Q4wMwSkx@0jec6E74bIR5z
zdV(|a3vbA)_WW=$UZyxdjxF~VdTfzz<WTroDN2(M;s9$FPbw_fR{$6l<o(95x(A%M
zs=|D~_&slca#)kWKtXlE<M=_5@U*r<)xz=AOTNdD`Zm+m-dO5<efm=-*uTeST@|m;
zviw0H9M@GxwojPeK^LEa3aDfQz^606(Ql7xv-WdO$WXiV5D1h;ATC}%j1Zoe-4r)K
z1X~bedPv*>Tc?m@SSZcd-Fk}rC~t^3BE~iV!(W?m-M&s#Bnal?R<Q_x)kp!}1cRwS
z8&bmIhZLOJ^>g4g2oxh?MI`t!V9rvZI4O6kl~Y{mg9H`Nr&Lj!lIq^nV(l_J)1$pP
zcAU_^5o7c^<Q#ATDZL{cDup>xosRRse!rM7raFoPL6E}Yt6lWIMr!=n5JKt*=={od
zrf3OBMaEcDV*?{_rJBPo!YF=hk<0YdJH-Y)Q+^^x_a-2{&3BR;$KZ&8QL1?A|1!ra
z!U9+u{$zHeSnGuf$(sSWWnDi$9CRo%nl4e5{T|Nz@lD7X2qZ^TUH8D6xr;f3Klb!d
z55Rw2V{AsN@!;__Mz?$5#F%R{X$JPpk=Nc0rhh69YxgSAjv}Y+3CQOJckyfx*DQ7N
z_S99+s-NKR0n#|9N?WW;6Nq{)RhA%8ANp$_@ZitW)^KsR@v{7IFkJx3dfBl19Aozh
zC{43-%T3&DhxaTaj=}m}A^YEliBCI<m^+ged;X#b6*?EFm|M@*Ys5@ZS&;<r5Zno_
ze4g0i8@<wgsNZA31r)koK(s=cI=6Lw)TMk}R|I}m#=_YD_wgo(8?;>^fy?u$Gy(Y_
znMpF0>Nj~6<&1Bb*zI$Fd1F>IMGSxHn55d3r9}2Z=t)Wz$y%SnY%k>a)|bFYfLkwr
z?jN!UFE-fOhQlX6JgE0vOKIKXT#4`LsC-ar3gpKM9rS1;O9pK5@z6>HMv^p?kzLM!
zC#%Vm4z7)uTWmV~2_@KY*)&R(q6-#T?){y<T_fnqiXS2!Fu)YFKZB*oW6l_b@ddoI
zAVr%ditdHzy*ofOf_De0pjD5*SXgSFf79YK->hePq+7iN?sb5#sEKVkcnnU$2SP8e
zrK&ZOXl>Sdj$SDX2arbv@2Wd>xY?~(xJ)0l!;>_<=VDe^YzEk%k?b8uX-#-8x=&;%
zyCq?xoK-bpn?WD7C7K>9=c{e)`}<nOqN9t!osU;W8&JoymGQM}?80H|j0@_0uNEG4
zvL*U>G|HPhrb@9z_mqnvB8vlPd3)-Hz1*(T96Djl4M}d&+by7(1X!9+)Jp2C#-+)}
z(zzH$;gSVTR+sOTNniGeyy8+L{Bql<epd2;mEIcu0^sdcO{71J>d9jEHCY@-3vGZ7
z_4^n3iuwkNW#!$&X4hQvL1~JO<Xl}HaQQ==S|~QNxXjv1=g2-dS9JEy@aZ{E<&Rdl
z{>O7L3?D<JQx-n5=uKKQ^sP_UBOn%B-jvN1Mammu>uAFkFXXT%uhbQ<9o`vC8v8P*
zT+{&sKRt%X#+BCjq!Aq5-@NPLiCbCM%47mw*6-KJ<qh;NphAtO0cdjMdm0TVJ-~4R
zj%IueiSvV=J9Z-Hz(pN)PUjOsiG2!+fG)Ro@Z*V4ERwX>8G5Ggc(-MwuLXI52}52-
z>Juj{fpI!%Y!);&1?cP*_&oB4DsxkDEu+R~7=xdb6rBj`LY3-TTvUjg?XV)BFt_>i
z>V(n25$W(ozd^sr=R9PM41Q@fZa{|_!)FvzY;P_Wrh)hBR3Pz3e=Dr#29taicxVmF
z&MrVe<s#Hz)t0u)eH&3Bxh&XXL1BuIy|arW(!oDcC`H7$T>G^{wj+%95^;&=`5%9v
z!EQyUIRwj8Ooqjr$_H3+X5Ns7EWLc9n&qK>bQLKIV6QPE{uRH>*+5O?oKX!HJ6RT;
zy>iu5sQ$T;TJG3|LkcJl2I5;scNuB92}u!Mf|B15=(wosbfL?MpLZ~45QtyuROxc{
z8;fz9Ju;w2sPk00sUIbwSAye9A6R}_m-6yBDdbHxEo=A+0@M3E1htyHP?j%LehcZL
zJJ{cqL`Vjct1l7j!-I9mxqc0%UgWO|hmXojL-z#K`Web(e=f@-d24NZ^Sfj03H7{|
zPzO^W;)RD4?Co}pd;*-1Q;>Y|?K&B2y|x^&mX5?w9YMngY<5*u;z)@6bb(+3tJ_B&
z0X+?D?H@Mp-d1(q28xB*9!xY2h-1;4IiVkFsXtL7qPeeCkdl9c-44!Q`{q>P6;KN-
z0Y1-)LiCfX;l&YNv$>q9(R{*)nUy>b>!=w2$?n_EDuHSQ*6o&ljKjcQJ1NhSUF`3d
z?jtwvs6UNnL$s4^n`yK&E!Ik9@7&e5Fg)!ZeWNbzo94^|gURNiLm8*epZckrbGJOW
zpyW2dGhp7rSbXfFHlmjUmj3It`1kHmBg?aZbY+TyFE8{H;F#>$)Uyhh?K3XlXUq+D
zHTu>4v^Z4{&1Y`6T}$?>_?o3&$F$!aE$$$-dD$`%>{>HGaK5Vluw{LWKG~^5Y)Jvp
zQr^$7sI+dT(aFq2$2zV&R(x~ChC=Z>rW~EavWY3%r6iZV)Z^sc(e-HpyL0o_;Zbx)
zC^dv&gu@A2^LuzjL6X4IN8}q;{Zjdtbg3f^mldK^=%-=iw^5&i@vF6Y`US$H!m@9b
zO`O`PJe-(DKDY}(oZTL|<lNF0+qO%vKHoAtpUnxI*p|_)s%Algeh>m*rlAXNrX^L)
zmxri~3L6<z4v|MvkFz^7Xfh&u#BM`?mz2rw&^99Q#MFCTnOc1@hH;Maxwo&WjWG~d
zr|~MaS%%J8w&EmzRT7=ji8B(x4*(A-;Ma{}HWxw`@c5priC5!u$Rs;{ktUH@>^<de
zE+8ZvqU3Q{`=&gdCvCvZboh~KW>-fsY9FG^Fjsrl{ndOP-qm5*)6mDFXQR}_Z{G-g
ztEdP8BsAd<!-U7nPA$P(3W=)Zu)>Ah!IL4KHHJ{-zwvix?^MkeIKL+#g>fG{gp08r
z!z!sG_dZOrazK^4-$_3ibZwL!aNJC9T(h`x-GnnAhh0353Xc)ZK`|(-ufIsPa$_-#
zry<lv_tvUNFQJOwn9){!Gz3uoGL`<(A<kXNGSg>WNrF*ax$b2G@viIS#l=idb07u~
z3{9&lh;K+Kmcc)!%jrk$lBv#5YtqjuoI65Hl%~R0!biLJHe?ww_*Sb|XJz*RGO@_W
z>XLJ*>~o$Q4JkSrkCho}&|GIgVWpV5RW07tM>qM!6^Yg5!sfQ|dCLNvRH}4dF2waU
zl_awvb!8;?yJ|~GqO?byRk?Or%Cy&6;|u$&rrSGQw>t=CCsV2*jsCT$ba_hgPC|Ql
z_AiEyoxelyYl`Z9%0gV44Kk0fEG+d7%uFb`=+uig50BB=5sP<N=%x@vOKu$mZ$DE-
zQS1{go*`~jK5};3R&#!L^*T|+3^|QqfI-X=T}9vdrKU)v7nz@R0T}S{Hg~w_+WcZ%
zq<}A$;B$Gqk$Pb<X&hZZaHj=wG``el)aa7!4ja1h7R7iz`<y#)*PmV{NVSN21&gUV
z#N0kh@s$3R>!(jY3MY-9d1ZKJUt*?|x-VGb;5AkWXTsP$*<ZX^XA$3|uhuW!xD%eu
z=C}U}V-Ai-Z^l_$dr`FC(|fS;;nL*Yt;(xg)&5oIFw~7zeFK#X)4e+qREN3E26+e5
z9irpHZ#>_NLcFo?UJJaWzFGBEq2$sk{==CNI=Qm8J<e}69j{>trx3L(flT1&R8XWs
zOeCf`3+6G|g3&M(3ts6lUGO6MqQU@M;=az{*IGY)!Otz~rKL!Ic9Xy?3~41Yq6PPU
zrLZN{TA~+pG+z>XBT=<wqu@Q%o!(YCS<fM=$EI3^J@Ve^scYT)Hf7D{dNJE!yOLZ5
z^XRKh6dO}2iO_H;_8uEk<(m6j2F_9!IpKiv+BhM=5}FhI8|Bs*oxmteI!!Kw1=29d
zR5|tvf8y!Zr^ma>;@k0nxOOrL7dJG(fm19`n$0Zn`<Q`@r%P680-$F;SR@D$eqGid
z`*7-{igOfbb}DCLFk`?7G*|-<B#W6-9ASgPOM_8NL+=TBI(kgc09=}i_pz*=VH1^0
z_0E-0p*Lag?y`oQ>o-57w{hj1PKSoRK1M*W#2=a58t_4~mwRU5Pv{HIObiGWI9zQO
z*S?|kI%e#jfjdS8SE($-NLg{V2Y6^{ra>0-!1$xo1E(~PM&@HDZI#e}>8wl_)8(~q
z@eB~(Dqri1zWAwCNMiXFJyiW=6zL!?Ll&DV>li*#LwWfw{qEj)*WBp$8SJv$2txw&
zVIy0;ehZm)JZy*U!BTf5;?2&^;chpW!e|+$HZ08QOSSCGZ#2Vxlqw+%C@-w}Nc7!i
zf5=Tw4EVrz=)W*cY38TwU!f`BUBaBeZ54;4g?`a>=-#@eXO8qJBPXzq*^?x`XI?1+
z8Viaatkiu!^O@=l+ORfE{W6Be9k&WJc|8nOb6Dw}l<xPlL<z$}w*r*Ll2{F)aP&$P
zoi_{&f~pHKtTMS9TC_cb>WvXC>h0rRa`r~@*z6~AxMZP|=FhzJTfkB#8SDt36qq->
zBHvMI6o3;kZW{|x!I7cFRW;{$Re>!vAFn-R3!5cuqC%<c9;LH?{lZ#K{EHV~%H|6w
zMgJ~(n-%A6Hgr2%Qd`hP9>H{T3S{fH-inJ;&<^p`H-~w_6Fw+bDCMv8fss3ZQ$iv}
z`M{lacuQ=1N;BwC>~`pI9FLuiB(GadRH#t8H>g_(Qq_Tru>vm11b@2{KJy#Ky*dTF
z{HB&OQ&bIMDY!Td;_5AXy2+(fmh0!A6Y_@$ojXb=yHI%tAwz;laxOnMJ)r|OmyYS~
zeR*wx5w|QY0L#^g%o5x$59{HPyAuhR)o;9Aeiu+;g(ExTN`E!}Ot~TBanLC&kw2{m
z(aZ+pD^UhxT1szCG=}D|p9t{M!7po1hbe9lE;{rhrfB>YSDKf$R@!}b|7#`Cyn0QZ
z<2O8JhU<f*>k@@baxJ)#crwSY=VL-o^PE_dzy)vYDE50}TOG<rB}RmCjubJEw{=+i
z4g(1gD~bI8t#XEP?(c<#>c9=^gIgsg!ziS%6FGjU6_pk{Z`Zh*GPlG_nI&Z|ILHtd
zHNi|ODl=<M@x@@;Z9qUH!|8V(^{@J^l`X8go)4A6;vy}xq`kvA)giFPj8R(aiTZll
zIK33edTrs4`(3C1(%n)8_n91Jx{N>{;97x8Zvs>~D5Ru68b2^m0^QaQ7e_uLlW)pp
zz#k(ZlxZxI)f6P84S~~x`}++;M(Ht~8`)UcFr8gbs^eev!za`523>mpvZ}HGFvlpx
zviLH4@_S>Ox$uf(pc^fLT`Fw#+;JJaYoxkWYdv*#lLRHHbHU_`%$vN)36|TlvJb_N
zyXb@5_WMQE?cRi243x_jGZJf!fMd@pq1fBr5}!s8#n-`FSBp`CwX<vOp>*%xvaVLg
zN=L0K$<zYOF(cm;{yg)>bbitDyZ9~6Ni=XAGx8~ZY!v=^7&v1x(%Eln24gE;P`;+R
z$kDcgpypWOYEyPX|Mby8LpY>t1b3PHfrQv7I4-Qjr07QqN5nIb&oOLvdE}9pFkC$U
z?Rwcb@0buyA&`18^6L(<udW;12QY#V_@iS&-M3GXqQ;~xCCV50K9ZW|Duvfl3bHeG
zw`y>=$Ps(q$f6KTC?Q!8=k};|U4qv+J6MWs1w$Bn;<K5+q4y$j?95<0cL4+%?VI3l
z=-}|6^@TT>It7XAeU%)@RZf&5*T9wrR*Dw%u(Mj~Z8gwNC6I4(b!^}l)@KGY<n?VY
z9PaDw@fOZi^&`V-brF#v>5rSGycvV*ezph&+q3L-$La)l@#$wE!JD)PBaHcOShY>(
zV5~0F(t6dOQvP4v(?6N#SgN(0aqqTU0sF~%>x4GY_mTd%A`Q}+!?sg0dMWJ?noSq}
zg^rau<Glwi^(ppi<srL?FZF-&ZwApN`OzgS%VHH0vn=(Uj^-)=t=C|z*__b6_qRx&
zlX{W~O)Y4R-Z4vJOf}SB5H{GEj_$UvX=VzuGA4`D`Oj3ng+Zt5tq&{M2bAldDsd}R
z%$`>PXQdW1HSDgf|3u3}9!2p({!mzeJl2i<gz`aG(9BJ_Z~>4+hT<85J}5C8y}f#G
z0~}S*$py_Y9iS@?OlOE2w+M+7sBdup&AdqIZr7$_#n~!JqZ;y*Dijk{YZK)VuqZK(
z8t-hphM7^Uw^9~LbN+En4d-FO`b~n)=*HmHQp~&!YzCl?NVXgfoejuUM$J*)?B%23
zW^cwrDkR#7s*=>p{qdwMe8zw@&M1A}$bo8klJx880_D=6P*BXMvU%mFZ5SOwNCZrd
z!27|0qmmVW{Q~4_Dm}jv#X>ODF;<30xM;^DXH3n71WyKA7*PC%(92u+j~h8IP2w_&
zPXqQ76S+2No}P!GKBg07?ObabfA#B6x~z&pKZZ{L;maL0ZwQW&z{xsG{N^2C*Ha?1
zX6AI5lA%;k%$G^%P?{716#&(lUWM*0Wgte>TIcRhLF0BZoL=fOuavudD4RhFpe$S}
z02f1D;l^+=hxKM@RPS#1aMH9x|I_L*_KAZhf*gg?!JMum?m90FD4Dn~cAih@hV^2I
zIF8+t%2MB|JYV&sTnaZ=Y=>1V1`p{SZznx!P0%F$LAmyYILk<veA&x=AEB{NNQ>o0
z%9hVD0e5}!{@O@sZ2a3KyaO#5F|*1oMZG!@46$}e34AO=`1yr&+m}4#M~lE{&;Y^e
zXT7)la|tbi@B&~NxsRFZ7P@ZF?s*5@&-yA|-{yvDT1oA>5)&RUClIr~iun;%m-eSt
z8IpZY^C46IyfXZNi|j1r1E8gE_Jc)T2wg$U)$#AX$`5J**<vf<A6wVVIdtP%7`lF(
z;W?NTAH>32OZE(6^Cm&R6Xr(97a!fg*zSb-v*YVL%>4P#gA&ieOj6s+)B3U_o##q4
z2iUc97~Q6ODshmg@y&}QEWEpe=($sjrlYn1cFU%6x?ISgv#iU2yT3V9Ng=ljNbFT%
zbXRys4YimaE>t;<-fwmRs%uolhf9lps8<+jq-lQTn7P+QuuDp+H7xc{oW@s7Iu)7|
z^?HNFuf6(QCaE`H+5}L~oA+*>5j%cuZ$fLH+6``;ohO&Fv&df<1?l*pjaUfP?W9bX
z``o)yj7w~@JTo7sS@onyp{Sk(J(GIet8R|>7uunbOk!k!V>ZgecQW!e%d*T?2h43$
zpL|r(5+3%A_}3+OMc45xQwHQR7JEoVDnIiNOq%9jVp3EhluhbZFVdzt7h;&jp*{s#
z^_zg0$1@2B+Q{!Nq59iO(qwRY^Z`+_^a&2$D3Ig)s;~QbE(Sx{RK`oY=fyQu-Bw0j
zC6;?CUSP}Ni_X(5c1%myzoY|AN_pn%iO-e#06?M>S<JJjn3M8TUKOwbJdzxL^=Hmd
z%$9y4W%UNmR^jo(pa+UfQmuIBdD+is6}s9s%RaSnMG5vKA(>x24ZmzWESy`jkIf(N
z0EmBUw(hpCN=NkDsc7eU$Qc@u3HxHt_c$pl(j094n51I%XQN`!Iz`M;V;C;L$ADUr
zc_kbvjmOfU3NOPgmv$aI*~wG+Y`2`ha+GU0cCUuF(g^4#;es>w0C9_b!BROLvyIy+
zZB~zQ9s=t4a3_LSF|Kk!!@Jf86WVGjTjutU`IFoq#GM5Uc)7WQxhk)*LO;{#AT7Ro
zhjCVb-V3)|qmBK=LJUwXIex$khF|d(ULlg$rv}wzhvjHNW)_`YQM%>La9SpA4olJ+
zFI0}F55AgLJ|Q!l1p=G-6B+ln;&BqGDr&k!zLpCWJ5Rl6rwgQmF9!?c>_aK53dt!t
zPhWfSk)&WxPWS+s*ZdadKUm$D*{I^YP4ck&aRK%6A5e7MI^S_r$0HiyJlc(51Ei!>
zvEdyGp^<D(_<>uu%Qb@JmbuqAjK0+KAD))u(4wd1CMx#nP3Ti=AkVe<&bZq&H9T}Q
zIl+0>Q96EN1a@oz6f=wgvj<Ho*JE{bNuNpp`6>_P(?VqSi6Qdy*@YzMwb7rm^_o`}
zZs)zdem52lBl9Hmk^ouJ5W#KYzHAIZ;9uoZPBcKa+q>#Gp97=In_|N_&*P(nGRUu9
zhHeV+5*Kjw_9JPibkWy70^a8sp?Gf~2`ex)Q<iXvjo|v1F;I6A`oI|fT@Z4=QzFyf
zEDwO+ryVtQeoUX<;#j(^I&+VTZ^Ss}E}+;KTRe1mwKG_*Uf+-ZG>+2H<V!b}V(Q?o
zmd6L|V%|ZH>W5{V!t)zmFtE({pwF%xi7pw<D8o7fF|?bd7G6bIuR@+S&V3EyOv&gS
z3E%I4jIpggjhw?YyrDFpsK3P{A)C(E$9MqDrALaMIcT+lQcC3<RbDqtX0al)tiN1Y
z0ZD5Q??aG*Kz~8ciL%z17kj?UOlRupO&7zfAN(|({<PK(eZ2=JMCd&_%`N&s9bS=K
zpZZUc07Y1OErDw*UpW5bIjYwZyx>(3gXHb6*kcFB?`|EV1?ZE|h#ZR~L>-`n#9^3}
zxI7J$q#v#xqUmlQ-UGQ~C0s6f6~mP2`JM^$<gWlAZ@S=@o%y|MMT;oflDlHC-QkQ9
z;W8_!^b|A8GlQO*`A6|Hg{PZW<e;CX%+EXHdpAAzeqWQk=~1RGOBQZ}rl~l{LzLM6
z7I5zRC4E-6IQxcvAx=(7s-?}k^L{ra`#`mH#KA!OSQ~$lkJcJ7k-JO?(fsGgig6@`
z_d=IJs6#<~`GqNHmP1??#i1!wzp7qt+%Gb8-T(E=$6VJkkn76sqx9VPUNr57#NL^D
zcrEsAVof#B`BL(D0Jhr9KYY#Tb8Rary;Qz5oIp;a`y)!yOdU7%<=1tM*tdPQe25`s
zFSALT-<~kY#yj+Yajn@AK%jk}IAe14R<y1bC@YdrEcT6@c?f|w%BK+HW)I>Tp}E<x
zZR?arxy2d8f)VTAx>{vwK<N$WG~M9sx~7F@VNv}jsmU@V6V#<PdnBZ1C9qF>j81C%
z$m&!XSi8jD5Uu&~cDa4JR0es=+=xA?0mn>}dXdPDMB9X1<o2=@_cm5Q^Ei}Kh5Jn5
zmp{hYjL^JoBED9CV>Puot&?Gx3Jq0T)z7vl|5J~Fyb^!UcFL=iH2o@i<!2fu(A<&c
zNZ!j3w;jaLf}Sfwz~T>>{+3c_kPg!jWX@6$w_(sY>>vXL8);Qcb&e-D*Cuj~QT*1j
z+@e2A4N~zzIU4;9af#gqkms!Ir;PY$zGOsdeK5yowAz+2gXcXoLmG}vq7oupduZm{
zX}WuG;HmHEIvlERnOK`fO8?aHhqSM!d<&=@C%G$?X6O!aFJi>`qB+)J*zIu(BD#__
zY!-$`h`UOY71Lt|RSJ;+o|`RVMs_X?5r$KnNA`aW0^uC)xu4+LMLH!?euVi}ZN*fd
zLi(UmwgVG+JGCw2CWYb#O<tviN_(WlwP^iT-e>kH#^TSQ9AuN^<F|^tkUP5i$;2$H
z^lbfDS`5p!rlelsx3Xr^&$+_&ATyDeeM!k#IQ+mDuTV0JLaF^i$F2d;j}RD5`nvuM
zb#ctZe>gE9rx({$q?ssnbG5o3WQ?W&U%Uvnv&!$@@Zji*2J4M9)PlNF3k|xMR_0U^
zh(0{2!FJVro$~U^WAhf%|Mkqs+^W~!I7<DPsyX)K!YQBh$F&Was9EZ=D2Ib$x*M1H
znM`&dbKi*MIcI9m#C=m4p=8)NYX~Maptf&w%hrh8IYsC_+3iCpnq8Z&!}LU7Vrc#7
zLcXgr8no?ek2;YtQ{k&aTOf*<tXDfNLIM8wrI{a}bK}5e!qiP)yBDs73U+&+e^*gS
zdug}Mdw=4gB!!BZa*}tS1Qd2e){bKzD=nbbR}Stn``Iy8@`=GgpH$jTWdP5aZ)#(o
zBLvY$6Cjj_mfi%dH4&DJa}-bigrGp2HN3FY<t@`d@l9|6orZIW94OIb>oa@vneaf1
zq5oVyGU8G<#bEScneC9ze3Y1y+sEc?NQ0$+=KgcrmMM;bfKp+w=w>;<#$(_~4b#=7
z{mOUTs-Yd2|6GK9{F$Hns+@t+<rgytC~Qb<sv?+7qSM^<*-mrc{BtAl&FFBRSqn?<
zJrRjJcD^#OAXhmBwz!rQq~eyFjb!u3xa05px#I`r&tt%Nr!zoU*H3Ey42BRQH>d*;
z$$u^!^f>8<A-73*z`T@<3wlIxNaI@mu82bKVkBpb?@58#$pIMYkDm;UxaMr}^cB*7
zWfT95BPc<-QEuXwCG$iEE}3^Th+nM=y4-eXIS_?`w>sY+^}F*ppYLB`(O;8yqs_97
zjLlI(t1VQHDyAEboD=K*fUElVBJ_K|SScKrl>8jVT@Jn|fzO-oCD6DC0{OSWs$pxo
zj6bX9+n3JW_>;2&QuJ-eXK{$xfqgIQo(Q%<&)qg1%Ylg;B60e;GN&Wq>i;wbf<S~L
zNh!@@n3C4=iRw5ci5$pC!)>tca#-=Cdxe;Q5B*mT5QM2-Y9|HJVaE3)%^W$0)%0tI
z?*CAuqSS)T>c{E?STe^QAlgU>10a!fbt;=UsACU|*O2=W*n<BIh@)7f<GrYTAE8!7
zyeMG;<eK44Tt;j+w%mkg*w5PkE2j?=4C(CnDkFGXtc?g`#Pt03i~v&dKcrJ;-c7BN
zWFkYRB$6ypoHU1bj8*@W?)@_*Focn1AOJmiW*9(|yqMf-#boW2^ZDTa-lZ$m20K7d
zCIuG{1vJ62A~LX^8dvc@S&_elZ;kt`Of7zKC<?>`aI-3L|5+0#4Lx6}FrEbhG{FK3
z3JHW}Al{G*!2Nd;pK@r&>VGf6PY|>UtzvRvxDJBzG#4C_S%(??M&18Dq`M~!P}FQj
zHNHP|@6Y3{wFb=bmaJr)0CsIVWw|EI-%OTBZb<*}OMgv~h7L_3M1SV9=0@oEF~;7L
zD0H8s{_p${E*!mhHaP%=8jP-Q{s@PYyWqAeY`>c!RN*#e<x%Hf#BWzV_m^?uh{y@E
znBYRAA{d~?jH|c`9ZvVQ+-(fLeFm>Ri1G)+&w)3>ag<BTZ!Z3Ns2lTd^pC=K_PPmw
zn`GCgeN)rOXGZ)-7+uI`pb_%%D)uQKpG+W<lvT9&^9vulr++K0DnR=zPL!c;zpBst
zPg4_!5gFQh2fchg8cmmM|CPf1&+q??0s@&n1Dwm>KRkY&1O3zLDKjW=lC=%EOf_f=
zSi37<XzWmGIM;<0N{<Z?oLoV3xN>Zt*o7LuZ32C6g0uzsLUa7kQG6PK?fx@t*#@*R
z5(1H`P5sdiN;l+6ckBYFeV}|*DN||9p<Z(twC_|&5i%ADXK~p<1`tZh`|sG9vV4ml
zOn<S9=evXnXYD`gtQTmk`7<cUqMlMq1TMKBcD9eQAx6NO+)r8y+IHA)*vKMsyV7IY
zu<)Z5)@ng1PI*hJ8h*})Ix#Ip0L}gW$N3|S)&bQ28OjIqv9nlsOG<oZ>EK{Y4SL?{
z0<=8|o{WMz+HNR`^ct`0!um(D=7iBC!lh1gYVnTOLDtu5@lj-@f~<Q2YsGM+|2-_>
z4RxgjjpO2VgUynf2F3bgOX{TEkRNq4X<1k4t0|i%>sZSPph@ft4F@6;QKO!zGO<_u
zsIi=Vgo|#y&bz-C4Q*ombMa7e)t~nUtdqjZE03?(%8l;J;pBhG$c5}jAU*lt640Gd
z3CWdBkV@z(w#$_rYOagL7I<NJ{<TAza%ZrDdV{<B?r{PW61ZlwjI2S^j|WRPb|JP=
z3MpzgGC9T49F|zyV;>w@$3Q1{re%KZ3aySt7Y9+q3%`B9g-(90KErcxyT($6-7ZFB
zt}@&GX+D_*biQ|K>MZnrD_H`%($mLooalrLc7bGqU9xGk!?p@V(r`kL9fQ&Oa7l3H
z^z+;uru3<nrY67sZ7*a5Gqeh?7-f~ky0h-iO?~STst=U;6CRZAJP+doZ0x5VUX=5k
z_}`JW^S~lKEHfzqN4WXCm2u2&j;{rX!ZsxeXvR+v6n>Ypk{ljS**T6-vDDKzk{AA%
zEjp)(=*vjnwIcoMIc-&Yku#*PAt&$xqEr)_AFNbqcz-)i9)3XNrCy{$5fN)sZ&Jpj
z;yKsMy`!4aeUl}=8hutj%?jO#ZensL1&sGXD}RAJExiN&{2yk4H!})lloJ7l+J~ze
zmnvamkO&&!C`l<!K#!kqxExs(28BBaJM!PwHF?$?qj$sH#w9I^krxwWchI<;M$IiT
z0=>iS^y)#<tI+-wFfQTG>v831tfg=#Xw9|$Df=Of0DF~+RW}Nk6@mI0Z}lu^%GgK3
zidyMFhL!IhE{`L{O!+LGuoEXz*=?HBdJW7zhp!QeX)=<;l;ml>lgx1SxMWi8SYufK
zvdaJTuKJGcbLkmS-Acu5Y!RH}fAVnPz~;@F`F+xlxTgv7;~pB9|LGKlh)C^Z#i9M>
zF(Uf-D{Qd!*55SG(Jq8oRWo*%C=?5s>xfe-W*$YOify}<JL!<|Pr@R3l;dW1Wav`E
z54ImXZ7=AV!urTAYpzkerGpy>P1otA`UUm1up%*viyeQgr<Do4KfVWIwa9YwmOXqD
zd&BRP_ThjB(Fscn6SA=+2q#zgKp(S->(di;w~~;LZ751Rxy+(e?!}Y2r!b}xud{z8
zhvzs3OW5oUbON1Gg;rtM(Mlh=H(ES;w2I4NhoUC0bx+Ka$g5DVApT>V3|;!L@GYDD
zYB0||Z;El)b*r>5Da=)sr-KO{mcN^Fkx5~vKZO%PV5ftMo}nCMuXO)W38O$G2#6Bv
zFK#e8sz<uF+nX;+S?6DBy}4=oa$5ata9``Tic)Dud{E*&ejPzlfGCEx(Mm#jWv1d0
ztlxg`fu1~yc588slXN^Qjk+r41Z(Qb@s?D6UoVW7B866FC1#$P%!{iXMEN2hnoQ2d
zzoN@A+Rnf{UWk$h^KqJ~V{_fT%BFD%E95{d=9Xs*MDcg754VcE!-R1H@0ZHzp=Yz3
zCod>e4(-$|8<+{B_Gn-%b-}==q*C60kk%4-zUmq;NTcE~@4p)7Fa*<hs8|^DTTG&9
zIu+l#(iL!o1e3Jy=Q(O>4w)hmbeeoxIOvP3r%<q|FFT&z&w17DGC>=<1UG$A|MpX*
zN(rP80|)1qsHCaN>G2RHwSr<F<v0RN`jE=8&l^W6E8Vep=~Utd6qv^Ths|@n4#>$|
z6&9jAa~?p)3z@!v&f>7|2&$WzzY^1fD|RAf&h6}QWfCv%TsEQI66WR8Z`?0=vIB!(
zezRRukpBr}J1amV;eFhwJo%~&DA=MvPaY-wbcRl5L%@higD`aODOX)W^F5S4dGI6(
zULB8Jh`de-{UYK^5hTnrVh~9=6?zW8h%IoxQ@1jAJZ;t9Jh$7#-X#l^(JilEGwlyk
z77`^+nLK)7XhZp*%OL?~U|pZp#IZI5{>ZYlZ%BluNPa79Rwl1t`@>sH<tZdfW!`+n
z78Ddwsl)bD9rDomub!Sc9M3QvQUMQxUf*!h#A`-x4~`TwEC1#e`8mIx--wo-EHn*l
zi=8<ZjH9&_%f`?oh_O$V8QX`{FB2J#>OuSgJ%n><6&vO@o&Au%@O;<$zQ9-d^<8x+
z9$P(m+01V7{lXFEkRun|P0HP#meZnVCVJHcGo$MtQt{8R%$Orz_V!rcIKpj(yReM-
z=j-D@H0dYRk_rB+lFA$^UWh5V6Zy|QcSl3wQv8BezvJ}L%_E}vuRs#h`n}$MNh(!L
z6=^CcN6*~FCCeQ^0ynj;=M*}b`sd$|#@E;FH34$M19Ik6o1Z0A=@5zLW1i<X>bAnp
zXOEW%a6ps{Mx=l6GufxKO#SM5)x9;T4^LG0c*wUx-)Z@y;BHp<AzDvKf_8J#V=iOT
z)c3_WgQktGBa~moY@5FiW&EVcU0Z<*%y})(A)8DhF5)ar(3CT^^Qyp$D#9~8!%K)%
zyNuY&-Uam{_O!xTHo1tfFSXNeC#(B79|Hb`K_}!v^r%gH>xb*#ESCCP#?9omI%;LI
zUSSgsEJ7DF!Rww7WqNe%Czm@ApsA(}$rvix&wr>F2q6k#NQhFeA}xXKM<cQ_=|$Zg
zcorqiE7He+^=n9)_Goc8JxFOnbIY-EsMkr+<^D_)-TM;?f#;%hy+KdlMU6ypab@zU
zrgSnX+q0Gc)1Mcr339E+M9SFCop;EHIq)HEy~GphMFMS`<lI^^bNrv;b^@i-99(=z
zm?WBOxhiXCi)y(_m>(6^3&cc!6z7Ag=D+b8BB!=4osh{F6!$bh;GNaJwBY2HJH1=3
z2dl`=7&TJ3U(Q*+vu4PMcyFy)*S<BImYhP{tVc{Uk$4;}3dQ9O&o^=m+eR~gLOSC2
zp4Y@YIAn?w21Noxn*sY@v~}Ojt4K)I0ZE%51kLIXY;95U*MsCzk&k<*_z$VXiBvX4
zRv^|ZP0Zx$2Mk}FsnTM(a#ium1VT4v<+c+d1Ffp;VU$14doAKGDd2^Yf_vMlIwoPG
zw&aU>?jChywg>qyUP3)Tp4;>^cnSHI&W4wi$~Ty%Utx;g)s1NZVqlBjQtNk&{)|_a
zyf#-{AB~6hqt`P&222q^6#GXdAkqr>A-z0W8Sp!{$~^A$xwsr7_us;;(9lfg>(o1C
zj$D_d)^}T42?z+Uf&|Xmt7XM|--)nf))tE=_v9+ePu!?6(BiWwxEhzcU|=hg%O(@x
z!DzRZv!?s3X(h4;Tc{Hg+acEGCglWr<4zHJv-Px`e-@u>#)0<qzA3UVnoDA8v~WpR
ze>^wd{kvCGz+Qo*!t!#ys(<1{0$x;@+~Gc((RI`@ka4jG-m*hleZfH6DN(1`Ke4cH
z`RNdKL~7_&s`5fk5!iHUQCCGcp4Q&MZ8DQd^Tx5df|>iF>G09Z^Q4(+l>vn%?o6*S
zeI2s<-SBRQ%qydJbIotp8_)fUFn433t}0NW>Uj<GkIjisAbj&1a}(l1Qu}G7>?=PS
z)1<}A2-ae#c~<6Gjk@G9l%t#EqNgW3&9AnHDK-zh@iAbSNuuu-SC!!)hPj8dVNr@b
ze0F-HLSt|Jk*R5TAw}_9FZ(lSVP~Yt4Oh>-q2XLBI|2WaAApx#qkqm8>2wk|^qmJK
z2XZKhGKxvgW&h=IYR>j<pKD^-oy1D#&BA@4`Q-IeM5wUc#DB+4AgM^058?&0T^iuT
z?|yRiti`=rG_@gbSY~JWoKC1G7kr{ZLd^IJxc1vW*Q!wh+@RVu*i6V{sZchB@6!)D
zpr35^4SY)h7my&xub%kq;^`LWN2Ru&XL(<yU@ceQz8;Y6V0=br>_>;a+xURJWgW6}
z0~l!MF;+s%*ueS=(pVhARMKl>g?{V&@RCi_a3KJbi|!<5sl7JVHp}qUxPfR+(F=nJ
zTu{M5|HwU<hfTZfw}(0h9PrxIAx9og-9SjBVeSyffhy*~^d^bft$HR0qJ@>*E(UDx
zXALYKWj%Foz4V*9X!=!mBVx*8#KB#)M%wL+&y>H1ceQxm6`->Rqb@EOobXr3@+?Z<
zUA~oZ>X1L}9Y}27$uZ0y?;=m33rj7-^xw@AuAu~<uDLk*N4BXmB@{0~NxZpp+|#gZ
zI57g7<7>2pFKf|^KeQM8R5~+nX0V#6uHW3Wg<I{h5qMa?Q*rVB!9V+|f9lAf-V#`3
zf@#af@85K*o_UKZf~N>*(1rR?-Znn<(O=PdO|<dqL{uOcN#yqC(I8u0^v^N|9p1Lj
zw5@k)BqL<=qrwb~MV9n*x~W`jJ*h`skcYn2Kw{wdMHTzB{<M@>1b9dCRwF3RSZv|4
za_Qmgtw)(oabVp1d`)mFWeC>+!%z0J$hrJSmPM<ray~Q*ywqv$7EftE!EU5NV--)V
zqceSL<U?zHaq_i0h943!bfa3$V0M4u+7x<-eGaK$;jmV7`eKeuA|3|h-&GG2n2*)w
zLNVYYNwX*01d->((&LFcc&?Z{P<0NAHyQN)euarCg!>V~nLUB?q0?EM{J>GDc*#9&
zP$_n=4^Jp=Ca~;`7fxe+?}jAs!UVpB+iXO7j$b||Hzpr}yt>gfiTH#d%;r3{Xr7Z*
zwdDyNRCC=_KxpB04-za-(_MDMF=+CLmoZxlu+FeT^EUNrD`fJW&%#C)-2J)y+ng0V
z_>N6>)(j`dEN2SwPT%jAR*ZQ_5i#jRSw^P$iSRwLJ}P#Ta7c+rFQew`8gWqZHv<qw
z4I)LV=VaUM%U&f`WHrZ4bW{163eNQdhu-rhZv_oEBJLA-W`h%i39!Z0*<A~%iPsi0
zvz<Q2)$trd;gqG{a(1ki#7ADV(nHxqcK@XMOBeeeJmHFTb}}BDJ-z6%d*rFtu@{kR
zVSEbJ<fomf&e7-($C>tb-yUvx<Ij2JYmm@+j~lP%1xTJvdxr1Lo>rL$J8Y3KkbRJG
zr*R!(x{kMf8xu86{P2YNb`ZAi*5N~+yhM}2KJ4ju>Y_CL_3!E4s!dsv!WM@*b7^B2
z7tsyp4y2R%z;A0|JW-9O$)yy5EvrTaVj~&O9)7&Xm(~>?S2uljr99N9^J$z1p;(@x
zC8gSTFjpRvqnvmtL(3lH;eD;G#oIkMa$b!$*pbtXAHSKu7bhYvaqjTG6|C;$T9NLF
z7$3Qnh<|XGxqCiazQ)viCv#Hb+@hv7@#CRS_TVixU5hwBM~xT5Eh$Xhq2jgY=bM8Y
zI<AX78kd&)0`-K(?V(Z<V(lr3^ZCQsf-ax#V^-cWAq6ew-$flT1Oy6SS#Qm0x|#C>
zsCd3Sn~+DI@=Bw<bk?wwXK>TA3G%L4ES`c9S+F+AR%nob1=>2`p}8ra&z@XI(XW^?
zOxV3Kmfz~Uox1w`^j4Kbsj4DRH#;F`j}>u@Rna2Ai%Hn)+ne|esuwa>)D#rUAQJDF
z<w2p^Bx2JfhB)*Q8X}Q-YdEcabRB&vI!-S16D9??fixB4c#_D|$d2l5{9hp~bt{pm
zC%ZRZz|Pl4-(Cgh8fw^#q_1NJKG4O_EYj8NmvK4`G8P|t9i>b3r9Lz!E;y%hIu$eR
zUyYDS5MD{zxBlKN{DznnCuWTm)LvD4dHmBioIPZ*()f;$>!@{m+y6Ih(zvuO>je7U
z<th5A%Y#gH-kN6>=h7`3gVSCV{#=DL)>^8|>ias+pF%GqGEG`S_G*?Z*FVP9ek|&u
zZ#3))OEw(%cGH(i>>b4$ezAF^LL`jL_gk=fI!fFlQ1t(yYuKmVV6QYSRDbvT*E!z{
zu+OIRo%S68rvDTDFBRJzp(7nDp1|B5yLj2764JfuM*AZCO-zhtlGP|z5w_WQ1)Fqz
zH%9R3Kk8-j{?Hawx5Ohh&a6oQRd4#F9fq_0fbGeoETNTJ9z#V3swCoM2_*k>Q`nE;
z1H@-Wajw!TA($>8=4?WJp4>va-;+-H(TcQ_e9ej$)?bA?XZlq{I+lefC&@%cs5M(>
zNQPegs3HG}$1l?_6Z;y>pEmw;xv&eVMeO)K@b>16yZNn*8c@C{iJN$DH?9WahGlX~
zVc1kDoMgL|kKwjn8?eXN?cw~A0d`leNtj(9q*=F4|7w<?A3^bpxA}8iOSW#O=3#P%
z87@6Cc5iX|=niLyMgv?RB1Ml<#Y#I-=vh8r{^1~^iD68*E<H@(i3sb;8!U83ue0nS
zDmtx>*SN+n;5Xhem4@gs8$nl8VCOg$^4iHhB)h)Hc|(>jBSM8S+`GUgpiFte$1cT#
zjP26`^O8JnwT^x(>8-mHgU+{Ou|c78)`z90VDJ6r2!Yibc2JrUz$?>+Rw!Uulp^oF
zn(O=-Vvo>a?J{W@iEQr_B6eac#u>H8rHXN{yI`;CLR41?0(ds0nKo>waG0jub4}F)
zLOWpWEhwwF!&axPagp_F45+tl<>gyXvyu~xcZRv0GNnYEC1Les8S|^M(MNv~YRv^H
z{9s0m&^ON04Zt`pnCW~s#a!Z$fG1&6468tqt6Jdhk!uQ9tsWT8sd^(3aUb@^)c_wI
zb{(&BMMZ<@+j};tSu=vyp9_h%I44>;nFOuTANcPXb?8@?hwwSNUx0ivpdVMj?w9&p
z>em38m+))x(a_ZXpf`Lnh}~>h%ZQRUKQai4?l4FMK152LK7Tf?>7Q9y`sV*$lC56=
zxJQUro(AsO%RttOm5dblGvSP8)Ax&NJ-xkO>K#9k3<i;-X?Ek+8_4OXzgx-23CWGa
zaY-cYYNTfGi>ZR_CgpVZsnL9%=%tF;u}-rh<X??oxl3rte3zjSkm6M9<7-eLE%4!!
z(rHXGSdZ4>sR8`Gr{)6jfk(5Oz(Y#2Kf}PK^Z9ml6JR^o0G%`?y%f0Bz7O|~5a93T
zczFr;DK3);qJp;!H@7?RDc15N$ng4_Uz_N+KH9egREH+#kQ}30#)aAGJlo;`Q%97V
zkA1ebQly0RTXk}&LFGaPk@zr@iKtR(*};Be*<|0o3&*CZstcyCAayE!#0M7Q-<Z$F
z6jcS;j?ou>BcqO*CH}dYD!=47xg2YE6!oQrH=(xZQ^T!aaGNB`%S}V!{dsq2&&b+a
zazpbSz(<okmlw`VTKgGuK;rxe=)eX(sN$CR_UV22JyvtQ|HGQjU~FGCA4oOo`WJ~&
z?n-P}PP{tH?+-n~Z^v1?TB%fit*;jbU8#qN4W2DgC45_Hx82mde^SMbyxQJ1N7vQ%
ze#D0v{_HF{-;Nn+fiYx1QFrv`Dy|l6EEybn;%g!IMFrS;L4B%VVG(0klg{BIb_sY_
zdHCq~G^0VOrnjszGTf+mSL-MroW2}Fs#OES0Jyz0z2pwcsyEzH^LOZ_lN8DLu3r>q
z9#=8W2?E(&IzyL3r$505e*psN0D-?C0`OT3O#4Gtx5Rw+b-v_Z7?7Krg)VQSG6*yk
z5w6sN3=?v2IDZu+C3#>n>3k^v+lS#CBn=4o0PtFv6XVR{<jRUb!akGp`!K}oWI{+v
z%Ci%dViXPc;x`T}HNF-%k`{4BC<i>cOir2Yiwr9A7fZ$%Omo^1)Y~ZIRAOJEqr+YR
zt#P!IY|SiF`h?u8y8FawE4#0wcB0ZvFzI#oB66};OsFEv4-%v>ve!N!L*OniuC(uc
z+WA1O2mpf~eR?Ty?EguaFQ@5=o~G<Q!u=1FsEa6rHn?d!_{^k}ES%G7jL^8<CPh}d
z4*!+my4z=+j-tN4wbdU&d=22YZG~QC?kFmLSft9s&%&FP_sL4pef{c*ll_?yZw3$Y
z85y>AR|VDEm`}PP!JnMj6*@x|;-uaWxX5gqMcMus1Dse;c73;Vw|bF?1YnrYSUm_u
zPdcmLp!Blkyo590U5E7n`MZOBUO|18=%v6O?k({J-o30QE#-tSB5m`NtiMj?%HBST
zcs+E)ldRpzX$4ySk_(x!Dn#r6p3M^#*e`KmnPp!S*=^B#FrrpITPfyUh-wXI8YbyR
z5PZsDK7)x{JG*p6@9%cUdDa%QNTt2;?k?taNmjpFw}TAbswplzYJQ{$<^Rw!$}-{@
zFO!2jf+Yd}B%@KT@<S1%jq|Ns?Q$m$`Yuqfg4W?;Ko2kZ;u)|{etF%8YYf`G^hxQi
z^M=!TiYUT5G?8FoP9lI1DWy2=7xmN4^{qkkH~oKYmn*hKxNzk)t!B{zGlWp#U6)9@
zb_6BK$E+O_yqOg>dKo?+gNQVDWL#aJ1J%&3goUTZF0mshQZvabMqwBy853_6hu@?O
zmVW;~jJ;)8Rc+Tcx+tXt>2T4xq)WQHOX)^p5z<}KwWPbdQ(6U-Mv!hv=@5`^_FUf2
z^WE=v>~|l>{xg5Tk9Ey4&r#=y%O<>UP@VxWMN~<9q$?!0^;l$1br;x_ey6BA17i*7
zMC|U$|JYZj4_2Y*8Vbx~<;O>l4cFZ%Hcg*`I9@f)0Oj;uIOe|{2Ml9K=jYV2Za-Ez
zap}gv_6Pkda%&x)^LNGp5wjGhG7vF4Z>p&H%j^6{dE1<~0oV}eJ$q_un~MLmeuygc
zZEIS%nVy+mq~P9L(lpJ~6IvpA#>3~7rGkv_7f0n;x9z?RSw+wtj2v_JUs~TP@ek>!
zltZ(!T&d+gfF=HGh4`I6P#1&G{>CC*C@`{#B<?!{l!Bt2jS(D>aMFZD0qfUPI`}+0
zE>KH+AGG=y5j?pvto8#$Y~s|HYm@OfB-ihM>8P+M=^%-l;{E>;z1=1?OcM<5D*a{p
z+Rv=-pVO|Zzf>}u(-7L(TE)_ns@Y~vb$MlE!PSH7ZsnOyCGA`pBA?taya3?=lv4%y
zf5K+f<acGac#q=Z35SXx95T~RyJNwqdWglY0s+KolN-ZTO^6^dFktQ(K@ZrT9FI@G
zgg#34o$-E!^|kq<G2DkyWmww$;Ax7>4V5c_1_f&7d><UtUK`hDOU9*CwOh*?GR~iN
zeBSIV&q4SZ+{Mt{>rD6qS5sk8oni7T>=W#1IeSc3%)7h~Xd<OEqm>Qj0>H^b{+@h%
zr0{q4!~`Gzu2dW;e~5<Rx}Z4=zr^hyx99S%gJU1j3E17`)F+NBtBHqN`x+!rAasPp
z7-|Yu(ca=sdWI#yk|;`K0P&t7kCWNMkGn=c9ucrlP);WU&E)SjR`*{>j<8Yw7wsAY
zG$f~PN0rN5(xcR`!GJ|H#+iKnETwCP71EcI&o#(jVbp4~gy%6e$j#oJsFe1pEYxwV
zx9gcJ`^acpqUNh*+JBC1L4)E9=>Y^DLFZCgol)piH#0`b<Ba+rn<60#tM;V<dU<Fi
zr<!VO+Ft#Gk1KQpF~p>aL0?_<YApxTw4dal=fE-tkPwysL|>}%UG5<XfEq4?4&S1T
zt;ZA7czI4L9E@J2NqnTzp0>oqT<R@bDngEFpOPKjkf`o#mur`h0Htp6sqfM+8Y3mI
z;P6&I*VC}*9r1Od%XynIg~B35X7H7`Z})t4BPoWr4KqK)uuYPPV?hLGNM{BLJ*(#7
z{ukPnV>sqaE2tB*;*|e}sEmEM@Pww?`E{R`2xy?|p1N=De%j{EViq9?L<^YmRRkWC
z7{-~)3VP5m<7v?{yVhh?cZVn6W_p7QhX+P`MESqs&MiFmc;56MnC!1fu#mjT{WJ+y
z<>c8j3kwo=ktAYQaq_O#5iD9)eksjpsvQ+hqCwfBkf;QS=2L-4XK|i#h}~>Im;c9q
zhQ<)<U++0SA0TU>CH0!of=#kE1Y|oii~kgZWRDPW)ZSkM0%ae8t6`Q2I(YSIG}rBG
z&C!+T2AdN89XIrH?2g|x`T4J%&HQU5uyPLb`4;(8MP>JoC~Y<q%(7ZUsakcggx4zE
zR2Qk_mY_0*)TK&z7#v6SKZ(7l`nL;QH90u<e5H8zqj9wQc0Cgna1`c}f0>54jubzq
zfAgI>MpJ>b^owEQ1HTlr;c4-UKkfv!SNy27pWa&QhN2S-7BM=VE7D4jH93G<o#X*L
zce6`lf!5(WmmEVL@d!?*YxC0(pSDhf8NbS~<WIn8)7yEQK~KzA4EOJDI_N0FiNbgn
z8{g9bf3t0UPbO7upaDC*V2E%+c%xciTVXkZg?gEVtGs9SB#5xYUbQMB=VK$fA*=k5
zj+kqAE0dc!w%;XvGId-XleS#C%VAMwk%DfIlY;zuQj6_`Xt-QKrN3<1w18cVP%!sf
z@ENWlXJeHFVDvQXG+bV#$ngv(%YjFoP42`Xh6BW(Kd?;#3uR2e@RkRx2&Uy<xn);Y
z6&av`z+RH}p~23kSZC`>Hc}MYL|M)aTgcnDIVMRAR)GBgfYE|au+<0Xp_YKIV?MaI
zrsp-T2dajLW#XhTEMI47Sbq^0jj*wBNJnzEe|17W!u21aZ=@*T$o&<%ufGd~TR!<?
zgt_U0)VfJEjyN1Ct~_InST7t0;+<Jnq|kBP3ge?JR9Fq`PN0S6$O424P&RWVrFbq$
zV0TI%2>@r4$g9%wsPk3`repsmz)M0O*q&0MX!C31u>Db5Q>pobOzwBCfrfC>LjbrJ
ze!T!ib*5AN^)G&C)X`}+O9<7K)=E=KOX<ogm-KE_NE{9vij+EOSgd)GhgT8BOE{33
zRHZ7|<RD8TT<8I<2I!_5#mv8SN{gHZezNT5O45$=bkNKs(AfDKKSh{L)AC0!$NpnL
z*Hu)egQC{wpMbcIuTWklXt#!OnE3Q7ADn__e*Vb(rk?zfTinXzN;)5!vYshDFfG9_
z$^Gps<PAe9{eSgPe3V)VZub6)yNOekW8`P*69YzBbwy8`c04*f;o*VZ<sP-NcUyFp
z;>giI&QV$?<@+^@w2V19UMVrONGXtKY~kB-ZJeem=F3;Wq%rqhxv{)H)84y8&;s#P
zk6u-m3}E}*`2GjR=Zd!MnB$oUL7U=`<j#ZbA;f895HSPb*WXhGnhE=-JUJP^N-+6j
zPc=O?+kypziz6Y!IwFaK^POqAp=v#^kJ9t3aOkl-Mm9&sZ}g+|(o2cAe=8!fZj{k2
zHr7N6BFB6~F8}n37~Ly+JKdT}mykw$&9%#k2trb9VV4Gb8YvZs{M&%4xUj(^?KMd>
z@yxC@_{3fQFFHOPMKtMzegN#q6YHxsbRaM}A$f^$bgwvGkDd?nD@N&Ksj%)W(W^ES
zFaq`AzyTtbrxW|RDcILj6gv7?+@+{Mg>#e*GbfYa_f-5L!Dv=-d6uFBs!7+@+IN81
zSmc;6XRvhBQ%A?xEa6t`+q^3mY#3Zy=%Eb%6q)LoV|_{e$FsyR_eH#dpM38CWA-90
z1QG3XR=2#c)e$>d+jL!oX?9HCH$9r5vr(o3ni?RGLf?n<hrx<-N{@GugFGXwnB#4B
z+SYS6zg`7v2&9WH1{DZ{)}I1KWZ?buR#hiTulFiZG?zzeqOW8wknjwV4M|YKu|X#r
zSeZ+?vCmp`sWk3_)l0TCS(OhcBF`(LFtolDs>C&k80pHM1KdsAzsSHJcs@whPxB42
zuJ#u-W4OETp%iI)IWb9bqCT~>*kl${mhZc)_HNqmIz4qN(*LNSU{ukkUXLwBp4DvG
zfoa}xHp#J<MqlUXM#EI?dv&O_nehw|L3y$Szn}J+bm?J$OqhX*R1oAiAnat@>MoZC
zST=JF&1$$6@Vczh-13MT)<Jus!oPK(Lw@<bwh889aG1tm6*#k^F*PUD%kCh$Hy}}<
z*CMq~Pl|UAMncnYQ)b5>AnQl^hFHr?8^j?gC|pY&I?TN%!+-K)H)kg+Zy&6%$L|v^
zpUeR>&5r7$a_}hXEQ~1D9}rSq_VT5}LX2c3(Q9*Q%EKAbRvQ(tAje?0#0f@~_6uSw
zQw6d2!JP~8mFTy@3LQ~R9t1TJR))(k5u*;?5Gh|jyg$V>YI-#Z9HJOhG?U0Qv%c`e
zUjhycyBxE$7790K{L;ppNN2Nf^D1x+4<_;+ecP{%qHOfGJaEvc7gMeW&I03?`urS&
zL&!*m<u*H_@u+kK%oKAUAb!0*)7+Cu<1S&IRY}Oe$R_P?00bWw6O8?#gfg$;4r!+%
zIKI~q$ew;D6%DLsB<_aC(uj8yd1+6qnir5MDL&x&M04G}p7dS8&setkA<epBPPK}w
zsMc(Zc`33djCYoGx*0RlfnMENTvqQDWxDJy*BW82uHe}#)bzn^@t2%)t0nE5?yp5^
z3;8-23*lOw<Wh4t2~qLtocsoO**p+v#&G;-V1q~EYcxr}eQxeHeOoCwy%PEE4_Q57
zWn9d)v{N#qm(An=w{}fa-zgq7vA~3`26oDT{9bAplm|>{XMjOpiQ!g``*r1X<+Z5z
z;uwl6`R|8KDlRP`6pdMgsIEwK5K7=^wp#_|NT;zcj?ZfYD1J^*(zgW(BLq-P9U}lh
zf~B5DmQq%mX4~L~AL^#wOjbtmScG!C{uNt_$(TWAR61r6c9tmTG{e1(zJ9nw`=(3H
zI8C<{7f4!SDgVm(m!KJo78MdeHsTc328$IahH2#(*h70hqUu@tgFJ&Y7MPxpR44=I
zh40~{J`V6!yb|r?0>;*_|2*_knU>~%ktQ4rk@~{u?7NHVY~!UM&6Y@c2u>8-`k}q=
zKxIQ9`b~#hf0vdL<GsyNP|tCs9+o(AWS|ne<4AFCuojWApQdO`jUr)*dr;mxR}!Yi
zlk-F`X0B%wgPS?4Wl(o48nSGm?jm}V{9IlFw*Sc4Ytk-Ge?XreY^;YshMmxd8yN;c
z*Sdz7Gf{vhaDp*O%!&S<84)-$;&FgG*R^vK01nY*)$e|2H#ZX|RI8*z3v5;^X8wc{
z2%7Dwwi+k}%7fLT^R&c1{LBVD^*cr9xF6gFC!Abu6V|fyR|2&Nbx$=!jpP+e^M-_5
zU8$?R4K2tUl!9Hemhixu8QF3DKj%No7led1zab?x$6h*C6)IO#%n|u~(E_Ip1ar`8
z#PMtV6nj!*x*EX5m&{f$7PWoQdk%Qfp7BbQ-zia-!$Ei}5DCS^$@0({*I{RU^H!Y7
zm=jmZ(A?{^;kok%h~vd(%<{S(s$Q=O`$%E{66>$BLN~>!6OtbMwJhqm`k#W;FH1gY
zh>DC`e97DQ@!Lh#`{O+%B$EDQm+vz@pH0STsN==E1V+<*`{b}bK2_eL+Cs6z))Fb0
z;kx&w3xU$N;G+5vMP{a4#29Q4xNnZqxTF&P@!@-4@$V?I(pvn1xh%LKZ~U@1n6ekI
zby15lKG7yfMK@BHc#k3rZzr`CHn7R=+frE*2!7*JeR$Ry5gg+f*#>=q5($@QW&B!}
z;?F>Lwu#wTSyu>wA)>A#2DHJIlSDDp$7&W?R@o)U;?>09AeYjAq`5A%6xR`h_^*ON
zA@V~_0j#?`z6M!tK$7|$`~97oLTPkCJ1{c_LjVYl_uY?ekzV_$Wy&BS&0~_beBXhp
zVK%gy`l|b+;8agS>i?-)sfFiF&$(X_01kq14L2S|sOeqc44ryEhq`P_hIy1r-@Do=
zj3Ns{dF677@S9)BWv#m!_@8vX_0pgW`Q#tWT5LHDaLe_zVW7oiSe8ET8tnh0oD_#(
zQ>m*{oX^pG5=-evhJw5Fuc{%C`lleattX37rZ~-Yoid#Qz5K|<t_a<}7a$l3aQt=h
z2i0e%Gf0oES<$688-i^P-rHknZCgX7@<B0`zVrS#XPh~=jNsFa%D+cf<`_;4AnP|v
zm|;@?P8H_pr1&B6SDNB%0%vqS{nRbFfpb0{CTn(B4fZ!W1{68(!r@sP%N4TIfhlF`
zZ}O7|X(S%57j_9RP0Jkq#YKU>uWSy!WGlMY<>_?cdllBf43~#04Sa(hz?6w4ZBE;`
zH3LlpuIxa+y%V;%9m-bq)84gprN!A)B#VOtje&j%%IPV(c-qz!UjCMc-FDo@EEl-#
z1y|RG=G8-^Ompt&48G<^&<dFuEy#%fig5)S*u3p=nQ6*1x?--pN)MBxYTclInWA5Z
zG){(8zwxy&iRioUeM#7JK{6LdXT^w8s7l?i47nz-nzZ~dY(WcR4LfxezY##E1srq;
z5V`Sh5lVmeoJc6LZo3$zYY&SJb*U&<#8vmT2E`3b!#JjRK|79cfs;PO+Tj$t<}V<B
z#a`|trZw4_{8)=z6Igxke8CPcLU2_H3*}v=NuTldvU@xjd=;16Pk>jazOhV>>&^y!
zjLOoTy4}lCP@`;p|JJo%kr`{l^-_VPc6YeLl61$(Rnc{V`}2^X*DHB9-l=8<Dyt6l
zqAaT)%m%lag35{rvz~3IuHAf9ivC(6-&>{o5nzUl#qD>dy7m~Lv2wu4^|Ab_;40}t
z_#Io@a0NC-mk`N0jr{d;#Z0taDadoaZ6@+sAC)w-gm~+RF)E+6wIS{cKNNPV-f<2B
zCOyNl+ROS5okV>QZNll?mg}X=Py9nNK!S38Rgf3E8yoYq(@SbUUxr3FjhtoQ<K4rt
zx@>N_x-|NRT|um0`s13Wm<Xtm3Rb`An_lw@0Yruh>=In0euz|8_@jz>)w_uiIK<zX
zAG=bEhjKZHMn1r!H9V@uXjXEmFvf|A(?4G?GrS}*I>+71Zgr#aRh(vvniO8`z<O66
zGm=5r_)*RHDA(;OuP_lbAU(2vw6pagh3HVzgMNLd#BIX5P4QpJnPqYLSp`b9T*c@_
zI}JqS*Bc}6ZtvMez9A0aCt+#}aotVqtU;i}(R?s-qy}X_0}g7?_2~6u1N$nS*xKbJ
zyPdb)#g~M!hvC~-ffPk&`Bvy|yYG1|7N7&0J-#eGFo1bG-C&LDebw|Xf<3m4)<!1^
z1At}`pMWjtM1@2<s;)5%O+c4FXu{FgAH4KMO>0dAr+-f69hyqKWj&OIExW3{tLW1U
z8~x(v93zPxXKwg%uF*%1s^d#sF~up;j}om#Swp4Gzo>EOI}t(B5)i{4V*MhxSf9L!
z&6Cb@>G?YBS11*8QuBJ?%qBTP!Wrp-%IA4Nn)=T@rRD^;hVVhIa$aHn1aPNHHQ`9r
zx|LF9`_5SUQ}E4~$Xn4z-HIli9Pc~mNxy6kHTF__qLAmMc}0^imhUFUz;GKI{p6$j
zOYxQH{<5R;?Hm6N)Tx_?1ickwHI0?Wp4*3=#EtyVo-0Q$_q;|$M4on>gudPS;(Yr+
zdvu#i?xi8JjlDeIHG5ZSBxf{|dr8!8_tNX#yx;cdb+E^SQBz9nfndP0@!7}i<Q0#H
zto~c0<y7zaOY?^k6u*`1vC9){-4&1Hj_BDt%iE5Xw(XX;4sKHRyZqjdsqur!@0yCA
zj{20IFRd}9^%WQAx~*TkbQC=v)t4IZH2zvFIX)HWZjn+Ph4x<iRvkeiUtA30`^GYL
z9I4*<W0NckD&+Xvq#e8{Pk^#5tFmTen%fB$pKbciR|2$}{nszc9pTrKHr2DIoj7ql
z6FGq>=Rrb`bwU}uk<{(2Bm-&NorAwONKP6TCR}Lwjj9QT3S44oPgl;-o2E(qy?)Yc
zyx3kWY5tDnC%dALaud^$iUvGd(9g*H*zkad1Hq2qjS+qI&S%EC=nj#y=xpa(x^U%0
zq9?tfNPn&ZES0F(1Uwa>L@Fm;#DbqS$_>hauWtWd{+oZ0H*${70vEg6VKfm-hVXq)
zK#s8A__c3a5?iXKLh5sgl|c$3#|FtO&UafXk4W?47xC&%#oKfzari@3>)7-TqT3Wx
zTKmr*7cUP<s0975Wv2mZRq$q6Va(&Kof-Z}&O6%i(S<*l#8<{IKU%+8`OKgk*SUIq
zU14AJDR$79^lo>&bim>08tM9c^f7$#zI`R(XNMQX^;(nuAX273TY%S}R70OZml75F
z(S?8l4Jea0+n3|6S_9W{>0N0OkQfxqcmKvL>ae_tML4ZngkfkKTS>%c=4+QlJCpXY
z1xE4NC~RE3LlNw&p4G3c{4+$izh!h-Lbl`F!loKLc6kBvW5Ygc7ZzI?Lh^IK2H+yL
z@)PN-UFe^G#cEv`!g7eDZ<(7PFDVAwZ7Jw^;ql!pZ-6Xo@DOj@=Z@Pr=Cc;0q={Wv
zJv>g$lN9^5jKI<U%$U_+=CkwU;nEqjZ-;c58U1YCEGkpQqHm>Bt-6W)Ps1lp@&EZ#
z|LZCAmON4)qP&i!>y7Y#i=N+P4iq2hD0le%vE(m~OS35r&FUvsY_FvR<+TGp8&@oh
z6rFNj1Kz+WOj_<s-TdO^ZqK%Z{BV5yu4%@*NA%?X-%nE96Zp+Jo+3G`+{Jdf$NB!2
z6pcr@A7AtPUH}Hxg1#R?Lb=5s;Wsncnr~;SHJbT+!tsFMf#INM@DoWO-u^{E?TlV*
zYnKuIj*rj#j?||&%j9`Sn}HoIPj`I0@{WEZ+!&Gn^DkMZTUPYFONT4trcG?16?!H5
zSG(t8^LPLZAO1sFh$y$H+vNQ0e$dgcAAwJLRC`!>$tBVAMs8SA^odSDcdcT(ybA6w
zh4xl)<@A4{CI;Y5E2yLJ$RZGUR3iiWGgOf5_gKretiJ<3qZvI&r1@QYzp6TL!n-J?
zW$e#;9q=>yUOJH2Vw8e#rWqIu9zj0mtHI%o@jGvnKFR_VWK_HIjPyaAUZlrJ?I*^`
z$<vR)2Q<69v;k=RQq=_Et=+e59gl*X`OyhP<W~lqyYovns{9fB`-cz9l5aO<XH!K2
z$N=pv{Et!qic1dY3;Yj%54oOy^mzLf&w1lkSoA8}29jg-An=k+Y0+u@73W#u@3$+~
z$zOI>Aw@n3kIrP~c3aX!hOX5>0NNOVRNASq_|(nDUJ?6E>^iP`*-BG%mN-N>)}h%M
z|NgnL`yUI2N06UbGviBwk8OH%KGXD%R?_Z9PO6Wun&xr-nSCi*{^i#_W@1_^?XT49
zYma@pO(Ob;=x@4L-<F_U0PVt(+$|w7<~l>eIjP6W5DN-)^I!b7Ad7R-@Ek2|4+yrH
zCAims>QNTJIK|aNgYs=ha1ZTvDY#St(j`keD!57O$#9DaviUXQ31h46yQJJHoJT*Z
z`J2=;$dkkCTMYS87p&L%x{Yh)epC>WC_+EJvTabgb4q=crlMm6_vb-9A0B{<oN7cT
z<8W8jzv1U&_|yZBx&Sh@`VLV}RB>DC6q9V79YJzvigu!OYX_H8sSC^8G?26T1VJ|f
z1FL|NLJDRRaH6^vf7aQ)jeVltQp=pv^eDx7RMb6StLn!;|9CyBQN5fd4I$uR1UCAy
zmnX&y1t2K;Jn4A@qoX6*la4pU-<Um>RnH=#i2GT)u+W(;8UxictgF?k{8cbK4J?Qd
z<=zt@M9FzQ8It|vL>aHiautte!<;eBCZ%ptfOpg6cYowHaTN0K89ACj*okTDf2tCe
zZ!`Wfx&MkRD7YyGL7H&?L$UC-nn~&~G@6za?6I;vaKkX!Z+drj92<W<nqA&}GWMkO
zep!<ppZVi{NAyLI6hKYSl+*D^Wh}j|ZqK0oL~5-E@N?U8ZCm@4aXYvxI5r$u1RUHK
z7-Prvt}5KB;~ZEx?2Sr_6UXJ%#PYPH#hx$FX4QdWMII{oO|J?+U7Qa2^>prA98tY1
zHgt~f{)d9tH2vqD6Z&g89PZF!jW#iRM35c`{Q~E8zCTH^D<nYaepg&YO5C3%-_EaC
zV#aMD!gHlj{F^^sz=#!);oXguUF+Zn;40&}6D#okxXQ>^jfjhN8$-s$tU6uCnh}YL
z%gvYTVLnXGGiRg-6Xd^!m~{}E$&{eUcsUb}q&#rAR0kNT<KbpyHELLU<uAM3A%0F&
z8A6<8M;jgW{mwGCZ#}LZDA8pZLAW$MA@ftPr`}nfJ%(G~ii)i2f04;D4i^^|0Xz3a
zB;`GGHr@{|S6DYPCW~gGTy938;s86EAS@K+``q`ZNs){T(bBMiTB7sljj|Rpi5SuL
za3!BV{uRB?`sPLyik;uXbrKjJBmlN--^%Q2Jdo)uiZw?_v@?#?Y8GzK8a6dih_a!N
zeGd{2Q0d`aytUN_c^0Tg)H9+MXV4#T$2nw_@gU_AKcm3dhJ$FP&FQHvh4jFv1g%L0
z5Z-=79j8<^eYItiZvlr4jm0h;GP5_^1V5;Izv(drrx5N7_p{b9d84E1NKnMwHi>_G
zAV5R8?Hc)|_~5-NY!GG=zw}~Xb3-a)k@o)r$owu({9k<314YB;6O4akq3K#B!Dv3z
zNc${zAu3*~Ks~A@-jU40ZWBo_lyT1C=T+wiM1S?k-H5LCw3pjJ_@T*i!QQ7*_WGsm
zG~f^4_W6A?f|b?2-toG-txX$VxxCwyyfZ(P(WBbeTKw6m+l26pg>}w&fFfnK8FVaF
z2+(6c<hhmSOc|HB*KNZ@s3b&Y#PP<6b+$`@Mgu+$3d#8Q>B}_Y{c0kZ9?mE0J$K9@
zFeN;g!D+IimEvPs+l*+fH6a?-i~tENS;8-H33VN!2EC?FtTf3DZQ-iK?PT?QmQ1Df
zwq?PsvE5CNG+Zt*v#3csQA!(0xv~z&d^QQ7id+g5ePzpf`Cj*o&9RAf_fP2B=H#1#
zsZFfYQ>1RDz=jfG@^hxl%xZ8wF?{Z_y3(1i_4RSne)qD{X>A$<n?8C(cmi4huo~YL
ziYID;xQmXf2IB&-C4}W(Lw+Obo!PkowFLz3OP;w~ePLRJ2bI$mt@Cy^cAzR$0#D3J
z;c`nf!DmI8^fo`fGRSoIckxeio0hu0`^W5Z;YX(Rw&S{DZgx8|Fo?#?jH(R}Fn=51
zF-G%fK8sDay+4hzj9)dWP83(~y;0JQb;8&K2FAtFN5<bgoJ$iPBo6rXz3G~LT1&;*
z6SapGbulRS)V7vBvgS!fpD9mR4Mb7&>9$nraZFmeQ`!6cdR63ItKBT?oC0$K2@?Gh
zI_TAu7K|46gT|K+At(rv<`iQw5igeCU9WY&U2h3kcYE-0a?#`Hymz{<)nKt?@4L`Y
z$n8&U2g-?R)axHjIvcG_!d_0XRW)tMD0wtw1mqcdOwT;HHPM?$R-#|r@#}x3cvoqU
zuo@}zrmsLK(K@59TkR9HZx+)}BE+{U*hX#+nw{xfcd;Jg%Z!Xz;?1U98S0C7xUo?h
zaZMUU(mzGKHK@&}f73tQ^x!2uDaX-_nW<1pw1=V48EO7rWjCj*$mB6GDX6_AvD82U
zB_NxzGEv&C+iFb&UUqa-_%AH*sw?wzoaQ1-V_a=Jun_f9J<ZE{Un+f_`>s0vf7?%L
zKbsg<s&3i?c0xi%fEyeeiHN`S2smolSZ>1h$-I1!eyU1!ba!RvJRqv`LHl=UQq60C
zlEdXFPt=++?{DyZ__|@yj<kJ>4dyUzi#5BqQP5pUf*ibO3QsvJ)LDs-0C@fC;=Bq~
z%^6wsA+^H14r`~#%?#tOs%+|({hATQYl5|iH&6(4ruRz$l$JQ!a;UBxU%hH{cwCb~
zXt1XoL;s1RE1r~L@ISO7@%oQ8&=+|tRIg~fZ`-%RJhIVzGY}}4T3LklbX}j@fgzeY
zN3N`wN0q3PGd9wx>V3Y8KTm(xC+#$LAgUaNK{kF+`14$+9zD`rug=D%x=eVeX5?Lb
z`V$`Rky!iXD11YdwKZGlymfx@8diU8LjW2JgGC12!!x1J9+NB~V~SUnIc<Brqk&o{
z*fqhmU_IwwQ7reBta>Z4xu|Z8cV0q9)rhM=A52qh!vXR$CaA>$P4?o|PV9R@&)wL3
zxf0#K4EEq)EFQpFa&juOJO9u&(JhM7wOwN24lRF!j@G9|N)lnN@w#3Fr+!jWNtM%d
zuYj;ioJg{zOwo@7d*_CQkj7cit1r_oObwyjC*X(yDBwOX(Skr>G|F1eK|=ks<I`f3
zFYS8fcr13eSF?5!@-4vMsY)3&x-=NZ7bQi!phYI}iuw2Iq#{LPKdEGTE)BV34Saoa
zvtl3G*y?g)MIwls0tO>12UVZ?m+NVK1IYq+OvSLy<@#)VJp;npr;uW^g+!r5pQwVn
zWW%M8YHTu-4z)bNDxn;&k~>)~0hg&KDb6s{W+90fW>f4dKY-Iytcn9`&)R59)Yl{Z
zKeZ8*6zR0IrkN#)Nr6k`r?%PnQ7@-Tzi+{GXGbA;{Sw@1!GVfXi(f4aez|5<!BA$f
zEIkvIagGaW^OvG75Sd&)I28?Y3V76ASSRe*<Ut8unH^+&Jach;<?5u&w6V+DHMnUP
z=7ehO_p(l#4$L~n@2Z?7w$-^$;5_BG4A_6R$IcE<wze~}%E1`p2lb}D{G<^R*g3$C
zi)D`2que%FJ!1xw;S}x-{?d!a`^-p}S|M>ipSrTtso4}f@>DxI(3q5AT4baKkGrZU
zxjxp_dA(3Hit`pPq8MA>k`u)w$v;Rvr>0PcLI#Yz((6r?`Z66Fr=sE^?52)nDyxa)
z5*IXG-^fX*Lz0=O&nZdS((xoW7cG)A<wk3ZbX_jd!u8PP_c;86a>>qJP%a0mDW>1M
zL?-WOTK+=g)~FjO9(!PXMxl{cjYfFbJ8%n6>wnkFDkZvXir!PYH40ZQU19?$c76Np
z;Xs#(%@kI%R;lGNg<2OKz;4)+8mw)n>x_VYZjSx1XeRP%Ob=Zke}v@)F8gam9!0=n
zl{HBw##QF4{Kaj?;$Pfy;d62-0=Ru+qU##@7q`_es*ZIt@?SlG#<7<W_pONZNnsVk
zF|nyJZtE0NbZ0ZSst?alwE8SJQih%%r`aIk|L$(|8y7L#$*2LomI~1K!wG~lu5z`J
z#V!S2Q87*t@AED6)SW*uXacJ1Dz#=bY=2bf$R_hM0G`~@9P#A6Lln!LP)G6MK^dX<
z7xCz@`6vS`BMQB#Y`!oaNHwe=feAYn6JHfNasFPf5)kiJ&?C8(E6sM?jSV+9VP3<J
z7Hr!hK6N4ZNU0hT^Ku5m{DY)K16~KvBi**CBvk5?_9-neFAzFL+l!g>{*R04+t>Yb
zW3WYM2x40k&lU56Z839Vvx2NS82T{*4_~A4QhpihWl?T$Lw>Za9+iRA7vy&|r1$kZ
zItAsi!G+wua)B`gMy;Z|cn#U5-`fJZAL*6jgFm5x>=pX)4^l5MnMuj=FHhghdJWFL
zp|4Ag#yMx4inZ6VfAAjeeWE8T&M@(hHo?7YHC~gu+*jFL<c^cIR)yZb0R@jtO`F0&
zmidgH2ZnOxG2DzmB7zb}&D!G>l811ZG=DyktpB7NRUU|DjRebKprHw@&nIas*2oer
zf&95nk7qQ`cSw%v<C}O(#{lfM%5&oZTJ}hQ++}uJRWU5hiP>0QcaJusbf@)fkh7YR
ze<=~7L=5BE>NsV0eh6Sq(A;W0-pI{~QI3~{WPfLm6$aVM|0Fs{9i>S7QsuFU{vhs*
zv67`{+hoR2Ty^w4pRWIJMmPPW?^UY;=G{17<b~6#-|FU7*^8gg7&g}3#o+d0$@zPz
zy4;GFlQ70$TKNobtO@DgSG!`K!qC|4QR(;OH&`c*m{!&p##CAeH~IJ>^wCMT$Yope
zNZ!T)21AySvD`@b?s$=8hIH+HELH#u?RfJNu3GI`$>9i3ygMXG*Pc&}^|lKls1o5^
ze*G<9C@Lbb1)PJ-%4E3VBA3xY;qyVJ12xyfLukb`7KikR_VT6V$<(^)kbRgJ&@rvJ
z@W})iSV!XWI4a4!+WV*%O=feGnOZwH(W$@__~)Avu(83)#Bn0~_KOg$Nm<I@%t{^5
z>%?F&W<MR{!HpelfDIn_8xCo&S+Bj|n!ee>QKI=ar$B>Z-1{mpP9aHgDP09?mfbBM
zvmO&qGT9l8o`AXPX=+`(9=cT6YCxl+UnjdAHcW%_`@GyB%zzUTlQUsUnWISo)aG~e
ziwihnxaWs-y}0FFxyHD#NH|=Tnzpw0z;rcuHh&2EkW>BXfG<lNj!LXoLeM)PY=sF@
z;@f${MypuslgPXrP2ItQeQ(Cl=KJSzgJzq39u;KGw9sr4a8dcxQu}hytbJ(Ih5h~q
zUl>PN0N+<^R))n$5%SUH{$Zj(Fd14K>E7E6I^x!%-Cr~j*Cgm9>2$<xTQ9*ZzUC!*
zD?&kl@7ISGtyT`u%O!uu=~s-Bpp`6pmtc}o{~1gwe=Is)EX>(R8Rt+XYNL%&UK*Ds
zQzA}AQ)rh($N9@gt~$_{%$riZPh#aI0o)-NjeIYZFUSncfz#jS6)OIbA|z&g^@w4t
z*E|E8iz1%CWrETaVI0n$;vzj~d_;>M<wjf_qSL5l7h=be41z?F*6E69IF!Oj0$
zC6z#~wi&ZwBZ>KW?RUGKGWNl*;7cC6?@VRv6-UGJ#T{yUf&SG<jXm#LBv1yoN(jqH
z1m53VadtSVK3_FBuKmqdWL)=!Ejn<O-+wa#$w?bVGI*}&Wab|xA!A8OGb@BbC|#UZ
zJTpyAQuzfBR?Urj{6+__NMSQ#olY7l9v_bl(mH{<Ji(0Xb$_2rv!ClLX1{wqI3y~w
zB|5vUgY5Qwy2}BfJpaSVNt`chB)%1PIZxi8S!c(i$6HU&B%!A5+KraZdx~j2^P!oQ
zvZl9KX%C3b5B(~d`XoB%@^D+0*zggn5K#TU7*T!$&=$6azdqy>uCwDvns<UG`X@NX
zg{1rWq`r=ER+k97p%c-YhQCpP=bA9LX8Sv1&Q{-G=ANGbe|uX|eXk97rQd*gmY;_-
z{s1Y4tFZfiq0#ikW8CgK1*fFVi$*jS#<@q`MImW>#MK!4e1>Ue)6ziK$&gx-xqj4Z
zV)UX6x`W2=$|U;Nj1nSYhzMNmvOzKVK;&%lyYL?@L4i~XKTl_r$2WpeqZqvIDbFTJ
zyK-n2C$5h>A31}6;C4J&N9fBmfSJBR`NAp^zdsB5EEX1L{CHz}GL=(0=n!xz`gWTl
zhcfo}9p3e6ZZ54LdHf{N#@74YQHIKQtvdE3CesXXXsNLFBTRoH-608w5q~Oq`8W4o
z7nES>Ger#R6G0#qAsmS;fspo{AV@|hk;0Slc%Q!sXq^}d>CfOlL20P~F?zIe+*#2J
z<|uAVE0zGL2=t<Lba<kbqOPD_ru;&Sqi+PbukN>eI*QMO!1K@>Q6SMA5HDOdSHJov
z(KO`J><bflCUK#KfCuCinW>I-7i&Gfn9_)rPO-?(BzK?JNH3;k=EubA2F06KH2cAH
zt$1n83G~eEHghE)z25!AMjAuU<>!CXeSv&1DV8}`xWzvxOftq)c$S<V9xjzlhYMyo
z3u}G|W5Z4e_rGNH+87soOSpOY^GJ6iF17u24TblB;-LSiSBe3VP<U<hE0(}4qd!r>
zgFT1uIxOSMra7XBOC2oN^%_<GkHMKPkBwwqBg<#`|G<*WeDK9~DN<(=Z>pQgCXh%B
zN6%ayz4-}xkGNCzHwedo6c0_8K++IPkr*3HjH|V*PG{p=FA+f3bu7rm08qH&-enx~
zAl(vEQAh^9ZV1WNYJ+z@NDuU@QJR!ojx~}hg#Tsz{@d7Wo72Ui>OKE#P$$jiL-n2J
z@tZ2uWG7g>U=wCw(o&mbCN)wrVYTBziwQ+&Zb|Z4H5HGNB^H08tGlX`(wy5Y+aZnU
z=ilTfrynHWj^<pQp|lGJ$mWRtZrk|UnXY2Q?U#|&C4;{!T`xkk;L9xV&J9N0x0|^u
zXU-H$ANh{N`vPsoQK?~Td$r0fk~d5b&f<LVL*({l&cajH<c4h`g%Lt{;zW34X)dJ-
z@f~OYc96UI>G}t<&O4gxKK_<ArT7N-@A$y|UKb#IS$HIIUW2c>%3N+w5)5$^2uYBg
zi=K@iN>TuEL#Kk6qa-3r&;&7yZ7sjgjg^WIEWb2CL=&_>#RWL5thm^GvHx&bD<Oh9
zw~YVVrO?7qhvC=zwv5&|+O}{EeJ0#@DNUw|`zYwK!~FJrS79S;1Tyy&QYx49GM7Zq
z%30+Qq7rCWpce4Evzi_oWh+~B=j-kUaGB~xalk3H^Ujcw0+!eON|`XuiPAGXDJJuL
zd%Fq_7`Lv}n?5aa<;HhU^!nDQ!(yN_cW{EMThPq%KT1{<TWtPsjHv7b`}j2b+obBV
zA&4`)1Xa<SM%6c0pVkXZG6Q@lkf|!_hy<9Nd|Xccwe_*N*P8=n1dP8Q7&b*D1PWDC
zV}aVGl<Cbftq)j~5Jy`OFAgY~Z!Gd2{^N#8(e-+!N%gN8DACVDiS~Q3!in1^zDK0j
zeO}J`m5vP=c^oh|(!pPac^mDGs*9_}M-`;1!n@!p$rxsn(kgV7{7%O)I+&x+Crgy6
z?vK1MuD5to5*;3HolhPvS*aPmIH@?-8;yAyZ%8qj=G*@7G7vB&t{hJK)%eOA2Y;rp
z-lI$m!)xXNZ_uZfFNIm)Qf@*R8f3#$pgn9**JuSgCjjY5DPmlF_Rv(XGJgG&Q*se?
z9f-Hg-U`>81ro#RAwXiw`z42#AwsuTt*`Bb(^Uls_IDGs-vL^q#FZ-L>PW_>(Q!j}
zJk1*#YVgm8pMLhCJ~;qUB+49-Dv@#XIn5ikXw2=9B@(0SxL<d%Ld&7j2%&j7ylrLl
zqrhHn*IzLL=}!jXeKmi!*bMy|4E%2OjTIl(S5gz{9?vYD#}rG4UNQOlvO?*YfqQm%
z!s6+p%ToI2iBU=W&*HL@j;9AP$x{8>ingX1q=)%p6eXg<Dnn%opfN%Po=J%2iPuNo
zq0EcXh#e?gkm44W-z3j3(U3t&N<-)uHl$ghO)Ol*=2q)9)09YeTnIVrLZ>V{$owsK
z`Z=wF^qcu?PLas!`#Ou?cto{3m3+^YP1T8`bZrcuomZVq1I?iftA&j8a!ELXV6zPc
zre1!k?2xTHInxJO@@)y~_Y>-)r3SAPWkZUzUOCBTg9rKA_V6lHgHQ9hR?-7ly(w~4
z_0TX3%{4!m77bFr<Zb@=E@5Y@EH$*U>a4-_)j_7M0#%mu7pY6dktFo}_<F||<eFp6
z!&$euC*Nu*9P}G=senc}cg&_|AjZfP{Da<Sme;}4+<dV52Y7x1uZrgV_~V3F&Fw5t
zB2WrgKG#7eZDZt%dk(C0L0$~A`h#1GP?wcqvLBz`mRTj&0<4}W0!1&ocmXW9!_i3v
zcq4ePy`V<YV|@SYXYc(okA8QO$z!ZnIkug&<Q+X^K~vd4SJYlMy}MgwQ`Qw<^Z$II
z{*h0Cg_&)}H7c9k>{<iEO~%cwtqeCs5y|7ZVYXNwo``UR*x}r2qV(UCw263}$+BN;
zocz5rq}`;`ZssdV(!{3CWC(ESEnW>!me8<AihHu9gLLRAjG+2eOu*x;S?F?GzHQ@;
zoUw1Dzy%~MKO#Q8Z-8I+aA&XboD07ps+`092Y*B@wGRB1TofRYv<C`*9A%>j3k<;L
zuF5IY<Gqz^*%IxJNBrA8-Epmwai6%z7+R1^!O#$5b7>#*qTIrUnIB?9d|Bp3;zc1(
z$s*aZL97Wzw;S<U5Rx{c1$(;XuyDdgh8XGFP(YbD6rX_dXcbuI*nX4fND%`Y$TRAq
zwdU%Xn$KzhF9jMCNpdpFVhYQ5snG2j;k+U5?hx&R>RpDzZK=UM=O1P}s#JSDsZyQ{
zkuH3#JKPAelp8EoWeqUTXXnrfDwJyqFQxXEw}m+w?{<-0y=GIJLm0FIls$*l<)E7y
z=6v*V#LOS?8Tz>;@GsgnY1$ZfGqiBpOSPo}_-j5+j84pt3B9!hcX1L?(1s}u7Q-Hr
zoEX1HAoQ^m=mZ>Aw0!^7juJh3Xb9CDMFMT)H3r?!H#{NKP<T3Bg7%ZdyuYTkC7@y9
z$yelXK@Wu0%n&ZSN<4@D2H;Dy{rwUVs?peL%wJ4JYM_F*>4KIXgVbmM!t<rmq|lkX
z(ZG$_lQ=PGBN>K}nRjyHM-|uQrT7MYoc1fmIcf-@x5Nb;95Q<&uHdD{t1%n5@#UAk
zJ$!y#WV8N)pVDtPcVFYYrCm+PZBQM;mH}@kF@y1xe;#|jcM41!GJ-Yc{<&m(%`$(i
zy-#8jWedcVNg8$A4m(DDZ|j#`N0@Z0I&Ewp<ABrS4_7`}w9aCBk;6^o`i?Atg;6GA
ziWp6BXn-lqPt6?L7J9h4NlY9?oDRf|E{%--J<NvY=rBK?Ft1-Tkg7Slx+%gfmE9sf
z`U7Y5#1L5g^D?_VC^FGlNq;B<l@V`_v;2Y9FD&uBk+*-}1ajaqtj4Q)6;Yd<WZT*F
z*KdJO(W&W}-7tSFeLNjw(8rZ5+0c;52qX>-<s#Gyvp9PJgX~G)rjGhV+6>AlIkh)h
zecj7R0)c$c+2&4pKehse+YW)InS?>szPU(k``;@y%?Neh&fK1eiPL65^s~hTbL_(x
z7dBzp+iM95DirOXOQz{!m_mC`Aa*qb4O5>;_ejF0iAiru#@_XMGvI=y8!WXY8eV7&
zR~87LABUFUJo{zN@oi=~A<tKwYIybA&jQDz_f7gi!RA-b$ut{hiDfci#kB4-yH``_
zYH?MB_Em5enNHM-f~cPwbz?0nA=JD8`WcMPPQIJL7x-m6VK(Z`IzmIz1Oopi4=9fv
zon<civs@+AQK=L%A02;AiG6n-79?C(>J+oMkhrOKmXh1TKs%Q3#TWTJnQzu9J_;x@
z{>>3L7?1a?7%jn?<~tb<aT3w`DGMs)MLpx(r8+;ULa%bQ2=?|(fmXFBnN)YNRBgvU
zVh9QD$9W%Swfu2E$1~>c#K$x#N`S-1IMDA6xW4jz=^Nv5JMt$9>ia_-#_FX&K_p8i
z;_AC;CT=!BxPeRiyrVbW@%58xRb*F6vm!rjUf|wPQtYfZ(zhf)b#ygd0`5pB6|cGv
z7PRPnAjvfQ>Qzrcjl7{bsDe@9i<ms<qWRkh24))vK)cmq8v)Fg*-F%<-%e4IWj(GY
z`#YtBx?-y>fCfCbk8+L*V=m}8(l!jpj2Yf@i}}G4-z=Eb#5#o(bZLxiMWRN*)e1J9
z;9n(sboow>6OpTb=0Z89xUsU|Sx+O)J_vl`zinH^9U5Z*^qj$DCo40$5XItLTEoN`
zT|@~q)pmFIkZlHHF<#QO9ei6soX7NJufiVmUKX8ct}?fuxzgUETa@@l%Wl^ge)Vb|
zl00AGE57l=2$w%G_g$+@fGbPW54`46%mlQ%Uu}r*ZBTxF=M#$1Dk%>|!&2@L&7i^(
z{@w*Fui-C@>6+N-Om=FtscPZ4zXM{-SH-)~D2{Yh%=RKDBFT4kKGX8?g~Y^fx6*wg
zwIL0UAue&SG_hI|+utf9pwSLT#7|N!9YcV=C{DAU${)ob){H1GCjr_VJf`(H)2pBC
zAp*Ura^H-8=!@pP;##9Xy}t`K1=^H>n>?(uUcAVmCGuvi?s4fo9>=E0-XLY=gopg^
z)1Bn@7h8jyuo<?@855ve2}-yILp&_G(7CaSRyW*refq7bSY(EA@O&}Uxm&Y-D$V<s
z3RUpP2=zJ$yB`u>$t6vA2z0uG&Gh~`zq;IbhY7XQhsn4w?|osP9C}Gf?1A1a*noQ^
z561-trQZSgQmFU#i$$h^>!R;^>;SN9gs*KY!=aKeMln}mFE$7NnfW!?i`!s{`$Fhk
z`{QQL?PfxRy{NP6QxL~KN^w<e?O}D(+mD#1^GK@{73A$y{#Xq@yzL)qNmW9-SzDdU
z9yOW}`Y3@Lgn_L*<t~}_#H42@dT+r^=<i8BXv5YN8OE7S!;`BxU!6gpe{4Zddv954
zuhR8xAsp|710OZ5R~VNYvpCZcsLHAg_lxx~#`F!)i3axeTMlUKTb5aXHO#FKGR2ky
z@FEXWKB+)y`Jfru8->K=Oc%3OQ+iCt6O$?RUZW0!>ofuC`rejDmz*behudY<ACDgh
z@-4=;iyk$Me@^xs3W8JL58d(li;2`cj%#lg=7bPq<x7rbA?kT3gf)wZrR2DTE&{jH
z_HF=qb?QD9Owtw;Nm)yH>`WVfZ&c5{&7jk7X?D{b)BbtzfXtvY4CYHMN8uu?UM9NV
zotwDd;&2t_TXi~T9-FASe68Lj4HOS=Y5DJtQ>S0SB#|){cVeY%a|%qEDS+;Pa;R!`
zccjF9_{yVhn}5GB(9(v6-rUu90h-DKH@)Nsdalg+HiRL{bG^iwbTT`Y_8>8IV2RHW
z%>m1HwJ~t-Bq+~<drzpE<tn6LTcLN1e|osTe)WIB**bNBWa0YD-+EQEX8%YtrzXdp
zaRx(kpBp!e`v+QzyIlO@OB9VG3&HKVUF6t5b0Q-P<n-=>PO_XctiaffsX6}1wN)7u
zEU+|I@wDcC`V+rRZ^TaPfC7c+wNl2EGW|*Mh|5Jlv)($Kp=)jwg;NatC#bDAJIW&e
z*e*VJ6k-O;u71_<73h=p?6dv0?7VzquUS&ZH7x+_o)Hbrxe|y<b>dvEXSn;F)N0CF
zTw+JgY?;24*UwkfH#Wx`=nk3DPh2g6x+h>$BX2l;uYL_W|Du<r&;HC<QFhD;O~3HB
zu9*i}OmMPdP`%{b^H4u5Qj>!h97-$0l&W1RtrxYSN|_f1AMd)j2GkpjrCG0fL=%It
zgv5SBUB2^$jBubU)%Y0+n1G#t$|=)z@b1}k+9)|F){|*Ye?-5tg)=bQbNt4ClD{6<
z0RBOqFZdN-{e{P&eOZ#t7R?LU4{@1(#;`)Y(ta|4MPn6_u?{gbC%Nez{`lgIc)J|v
z5K+#ni!UbpqynMklg?n_iUfvsEAhpIOL0%p!=_lOB~le#As<qZSy1e2aX;(2J~qv7
zU<>9%UDUMM+#B?AI>sl9_@8+Fur*d@eeB}j2;n88X_2hxqe2WDfnM*ZMujOFJDU_G
zSZ^>q1^H4P@()rdm45mL7k8Kbw;T;`o`5vHp9<lfYolL&<@n!cI{@#2NxI!-@2X*!
zsX$WxxyL$Mau0&W#u-MPZb^<^W&5qq9Nn86;iQkbO2;A44ZTZzy$ITz5;Nf*dgCpE
zvW<_4XCRQCa#lKJQjD{&Y(_M+^I1mJVL*%Ux1rOA^cJ8R-UyTQ8W_}X@g=8_HPB$B
zvHwYI@5q>0_IBYV!&tdNQ&Rv@b=pdZ*3=UY9xPxkMWaqzdkp9q<J28oIc9=m!}Uod
z!xuEcDwQ}zflQTW>oiZ{vd)+JI?Pw$<~@VOW3UL8Y`abuw9qHDAxY<VqtA|EnXw(?
z)>nLrGh&3H>lHUxR!kX0g^oGtE_G{Yy`qSwLAqc>>;B{8=uEgAd@d|M?y@tmq;F&P
z@=>K4MSiy8Xz9<6N>fUhq%jgJz`G^taebvqiX08b7EuTbmz~lg8!`#oqH4g;74~^h
z$ni95SwIK4>FJ_f&lNv_*3{PsVmMd;b~-g}jretXN9^Aj+3SMeOHGE55l(=v-uAZN
zX99CBWj~lp%x%D+lf>KQwO`*!0=S|+|5T}MDx%)1&J8Vh>?5N05M5^H-q{qnxXVRV
zF&BH4uN}FjNDsHEfp-7Ukk(hke{-zH@6j*x=b8S}kPWS}L1Fo>&)(Tt)JFV`U7(l-
zCwM9~5DVTN4(>@((pwvAxB6v>Z;)ouyu3ifc0lq$?PwW~{@vDi9_2js+?VZ^jX&d^
zWh}xCmp&w3{<AJqTH>Dk7?bL0bsT5&i9L>@O`9^qg6gcoBX{efloYd%$CW4J569Nv
zDjw<${#T1`*82_v6D~DznU(LZnE6d^FgCoi!^ypqe0M8(ht4Z&_+&+}0HTvebWyJ3
zG&?t*ES4MN5KcstvWnz<x&K{Dhxd7(SMtL~Kkwy%SB63TFYgmzT^CCxy~+W7dwY0%
z;>xvv^Hh9olD*sEr(5lf8;5(fnn#OK<H2Adiom*Bmo7j}b_$K;S?|tfH!nhSy>o{*
zw;UcrstiA?-@rFL4eXY727kG?#~66FqNjNH<y3oPOcI-x;$aLj!4R?)SEJhyShc1X
zPTu7wVsIIGSq*&62bSx@y`NqbN3`9e+!*JRIijIp&sBsCp8m6h7B;=Sccd?7+V)3`
z6ZUJ3X=WckZnJ%yW82B*(G)>f@tic^e>U=L_VN8<+qL$_&U`KHk@sch^m0VO!gC@_
ztrh1N^Pe^mYOYbe4(b;|G2;{Y2qhjjG?y83W36X{P*>OtuUXn2dTX=TC_*o={f}Kv
ze!mD%a<_9J94i$Lc<D9kvn)l}zV61eL>bAU5magDlXJs#-DSZvsOsOS`!0H&Z$Bb6
zG12oucJ$h3xd0QdD96*6sBP`jQDSZ_aJ@~qXvGZaQ?fg=lV&_)+2ymD*@XLG-jipJ
z+a-_mv$c0Q21G<6<h~~121NW*{A<;<Yow>sTJMt%L(zbM{{0`!&9Z4JplS9?s3@_r
zS2uB{8(>VSn$SR~CK?Psgu5OF8Frl?Of;%ihF0qQ7-x_`3s1W%RPuW%WdCrYYiM!!
zo^`9ym{@|RIodS_hJTo!)M$j>LNQNWeX8*HrQu-x6f-qcyc+0TDzvR0chH(&x2~O5
zNA}AtmnJmL)-07dn0AjAnB2i6p&7Lbn&DY&-DV4*lhmh(ixz{GJa!&EVz!y}wnzY%
z$PicGZ`LkLv9=!wOygROYN{qa-gc<+Mx7(7AGGh+7{|MaUPzx5*Dlhsp-y?-{b3j!
zdmcNQ>RoT7=72XelC2tN)hgqsUls5x(U{Ko=H$b(+_iXh=PC83XoT3~FQM6tLbl`J
z)NDZt2FZf4d$pig32<=3$c+&H=~?t(29HNb)OipC`N|Ks=6E(gC;|WV#_7Nh-g}+j
z(394WIEw8$Y-KHx7Te~rL(Q`8#~sJ^e~4+PP&%k&@^k-mjK;AU^2ojGykC|9?n-z5
z<Z{Y3*rdIpwETHPvuVA+&wlE8?+MH0yF~bV2OL3jsf}9?GV_6Z{qXywogNcLr&vou
z<Rat4A|97L0zRr&5k;)~ZudRXZ?m>%`9gc2{V?XTHoIHxe8+Gkuq)&|xw@cJZ&bIw
zwvxzZ=p@{7oyI)8?|G;Cc_78-F!lc&_Q-Zr9dq4Zkem{vVM2I1UkPnM0L5&4n(>Ag
zAuDfp2s<+PMF|4TbE^;8pY@Rtn_@4-^i}p8Zzfo$)OCwBLkngweI>XLo{?$(z;)N#
z?D<fAK=3zw)?Jh=G-dn=XMY{dMBq+}6e;yCB+*u3{+Sb|IrP~FKOWcrho`R&X!8By
zMG-+M5d@?=ltxJzA|+A+O2d%uMw$tzG@}G01(cBPX3`}gH9AMffRP(x_vQP$_x>sW
z`R+XDInSq#aV4l-ZrqXIL|ID8zo64<W}J(CPo`I}P}*u-jD2tX5F}DPpOfYOIgp5H
zNtmG2C|5T?+3|DWgW`;5J-XL)49c^E6lcu$Vk;nR#^u}`K)e#g1ukfxyK$ceL3Fuh
z>doOhXJk4~yDc!=hRw`QpuY<rFyF>3d9(TIfdhLct(~*4|H7>*O}ixbYw8n#1VL@;
zPGo~cxG|;{my}l_2-_?9dO!(TT;NfXy_i$(PiYHCG{&ud+Z<4Ck9s#;z*jbO#_|k;
z9&hen34=vv45NIyS0JiPYr<qw6=F*J395Pn6ezYHnsRiZV-$sC20s4MH&tXwq2~Bo
zs$tpd2*E)MR*8dC<#<Z!VRk{O<;xdmx2n|DMo`t+i;8W{et{OhnS^g+PsW#c;RLx`
zxbU<MMXEj6Nw*x(S_A{tK=(~O{}A`x3TQ-BFR?fTxrQ+~LE5>m4fad5M)tU68xsRo
zP)kYd?F3O8FOp9VgBUzVZyozYnYYn;!Te60st);;inD`yR1k=6m!vZ-g`DdWa?>Yp
z*wK(?Vl~LMXd(zpAIs&kHd}3KLT@E-e^-3zg54bpxk{Ok!Qykrd*jB95N&tk*XB9g
z2phM+ZBL?3b8)H{C=6PBIGKi}%l6U%=AQT$=YEY&_3C<RPj;&)xP0MN+tDd5X&1AP
zbFA88hO#RU1wE2HL6Wg)Mu0_M91=C+;4sc&7*^*yY<X>PNl<x?kb;J7hlX3B{W?m|
zSP%|7snzULZudN5XuZG{mJ;<ZMExL%*>CshvHaY+yr)P0*ZRF5J$NXqAgfBWe0~fP
zGkY74=RWfX@Pe}5Az@;=ilY*+SZb8I#v;B~tUv^9_NY^sm8Sla=Hsw-#?C#Fp=Wo?
z7`@ebn^_6zN?#mX)aVO}mHyg&t!#s|j7}60`<pMYu0lq;ifyPzs~{>FI~_zdC#p!y
zt;&FgM$?H&O=-yDnIb2LCX9gns4pkDuLZaF?KD)`^D_+GXN~$K2Q|LUfiY-)cPjsw
z{HaWD>>E(fVQ4`dPzm33HW@C)wyPQ-r+OpYG?0zIqn_g9S>lS$QR|}sX;wRx-&G{|
z&n{r68(pj$pm0DVywmMx5lwD0^p<LAx7$yP2!@`8Caw=^aw{`p2mb3if$-)<h3mDi
zBXNluL7|S#E3!c>Q3qaKo*;8Sx=IE=+aDvp)%kMuGr#mX_Oe6cRRylS4c^My_S{0G
zl|vE&5P3KSQye;P{h0(C9Nu;|iRe_t^oi@5ayTitVO8}4Rfl)8z5r3`U<Ko0=;^U$
zwu*JDq1v2|G*>RmA`U_m4-MFR))=_2=fY)l&<ZXS7Xl5dixq1jnWZ*O5b7?zF)dpx
z=p*NAbn^y%HX9AZ+0NEJ%`bWmzA(-+1-rhOL3~XYWG{dmTG~Bzms$QqlK>Zx2+6n(
zPV5)IK(R+&ghUCV>3k2xhqO*^3r|81DF<`HCim`~id8ZkuG0M0C6{2}#!GmMngQpF
zjeqdGY*Z%G5BhW)@6IrRe-!;tWRdGIAWXy?<)Y1M<Vw4gK=E(yNj{G*U~nI#@_bQh
z)flS!#OkuaTt7U*0GQ6)_S1{A&G`Oz*%^w3zu;=5TBTL>i!@0&Vj+03e0piSPC2-?
zaq&0Ym+ND<$k%U;J<}q_BuGV!P%bSthde_eR_fSwvLoU%mEwgwS<$2K$mxdnpeeI4
zFkkK;zZ0!nq_J_5SL+b^^v=a^s1c)w*`f>7QJt6DQEglpu8mL_I!!aK6MQ66s|@Yi
z?_@SrLt~aSV$-m1jwAK1OJ^;@#p-?5Xh7%c2A^nrBSKml;^IH^_`##jZm*<&X?VaA
ziB%p6dS5D+2v1+UqI2I`V&s7%8%(YmmhNENY4R4k*_8f`BzsJTNnU*mE0o+@D|y$A
z?gV=gH7@HQQxwlR6e_W4-v;RwF~Q}N0sH^X*^4A&D_QY$)iU37jaaJX_k3@2QP9XE
z#jEHlG81%u<{->G`MlxZ9XzvnGgngvk$x{GlTj6)=BzowV|zH2!^MC`-Wtb#D%3$d
z8f%PC^-1he)Ky7yuA7j9h`a<^phab9)?TXTYySkN`#?0~?8!?X|E!kq2G&LH*qmDD
zjhKG@?+(%YosU!{9&3d|nh&_r;xoYBd$N7K{PM7Ps8<(HKxH_uM;FgUF8YbmWUtZ$
zjgN8nJBjm!bEC%VBjUxU7~3~E|0N{>+|^8u)<xUJHW2V?<l`Iyg{)@di{I8zGJbv0
z*H}w&WG;Y7cfN`@)_fH`2KHRtozN92_s9M_78dI;H9MLQk3R$gcv(X{rg!b`mtT)$
zClEq(nsI-Whmj*~shuWYh7!(U^3IX}F3oS5m9QMB^X9kerWYO7cRn_!V*~~<A>fm_
zxaG}c0&nkG%TGXI+TXvbvVvNOm|~XYqa7OhJ_o#5r|33JmTvf|9S)7Mz5P{t#+u1v
z%x*n?L?>ga$#BacYzi<$bmiANGWzmmw4T!T`-U66-1))L^1{2+@77^zY&Pp`bglM(
zIzfv;SJoCtmYAg1(UCnrqi}c3as+o)5>!1kW8CBr7{F*#MGBO%oCzah7d5%rf6Hmt
z2f4HcQ0Ywe(a$w`DxmeH?1$20yyvy(hXj;h%z-s@w_N^|=rMC`=FvF57f)7Soq+gN
z8xsH69t<XI)dxkU6Q$?LGMRU^y_`4Jou8llEIqaKlzE*>%)SP8uiWHtr-ujAEVk?N
z(ufY-dFzuXfMJ7&n7b106|df{!J9TWsh~k=9-$hnfPKosk765+^Kaltb)gp)>d#cC
zKd|f~jU`=}rJ`TT&NizOSZ}g9>HGY5c7;X8X&(<b9GMzdjaHMDx`jV8P@STP^^N^P
zR%BqL-qXkPy~&^K-l&p~FUaU5<N=bow?bIFH1=1+{|+$`GQ0RWnf4bS{#!8YB$0<3
zYJ#WWtW|5#RacF<`XoazPF2u|*M-lhR|1s;(g4tmT58tb4=QUpgbwWW<g)YbhUp=`
z3JX;4=XvUbSA=~U)5bRSc2e+F!tFfOrXi5iG@pQ!fWMv18vfN8onW;XkE!n|ajGEf
zDb<2unD=$t$58N*&(vd@bp8T(GJ4358G~hB&y%@FxNvc*KT1r47fWa5qK(>pKMw4G
zBdHXahQ^MfE<*cbtl__(Is;kLp3_;fDQ;~}z{Qsnmk!midsgyahx>A2OJcm*ASv1W
z{qk}rJ=Y@gv_pUG0U%E#VPni}T_Y+|lVs1cbZ-hJV>$PjkEFQ=-J})S*kJd1N*d#q
z#c-czRueiu|NN`mYd6Z7M|v4X^bx}b)gRe?M`Qyd=dbp&`<$dtZsm#RkhYtwj^|i%
zWL07+GogdM^0&pipG#5;|Aa<aP;nJ7KGc~dBJ)gj-gJFla7_tCCVx5N7;_17_(wxL
zZa1EvP+WE`A?fGrREB20f{*k!Yk;!npIyi|GAc4eG0s5*;mrPfQ1^FjL)wB9@Pdh3
zW_&k*(f$>wPyDRf7wx}R<ex}&3H-ghH%0A@ik6kC@jjj9y0IbGqSQ>^46xpG{(-GG
zEJ`ZM!|`H+!$O&kqWC$wo~?P2r*v#Xx(RX!H`g;V3yyE1rW)R(!HA&Y!JvPK`mx!9
z!sQB&i35w@-OQ;J#{t9kLTTv>EQnj<1@Zrl1Ahx-U*@Z)vnkI(E}x3WL$l5mMP7)Q
zFIx2;sQa6C5M5=cWlaA9+=M*Za0>AZQpP8tYjsOOBf1VBN_pg(AH<FTyDs7y-nw2!
zdi2}K!jjbd+u!+W$*IMA98?WjTt;3V?6#Z-B3ZyK8*3YbwWsTph}@B&w?#F#y%<^;
z_3+ZvzP~C%kZ+^FlYe-C9H@#X)9Ju&`Dhc}<}n~SnDNc(J*N(`P|D>vl`_f=<Kce&
zeyNRRBn!_ox~Z!9cFu3F0-oK~a5K)35EfTv<j+EL9h#QZ{~3_Y8Hu+pNctI@SdgX0
z`?#WR=aq)H(A_>P9nYmA$0C?)#atkQj(U~M_jBJGavvAL;-VZPbev}4*B=u~lXyM8
z_{8X-X;%+HnO&2?HIoBD6O%udvOm90yX}R&TQBWBm2YCV?A~xwBDDW{(Oc)rA1CsA
zD&?YqM-gzp6|QC_e94oKh7Zez%(nnQMk`WU%Xz8s2-e>#`|+D`dG*ru2pi#0;ZV+!
zOp)rag-c6IQ{&=birPa=Tt(+)!|i2qVN$VQ*5W_9f$#EHSm1`GO&wrXQ-UKiq`n#d
zo&uI_e0F_NedhP-K&2#F@ZLUJks~=pF89}I_lviEW6x#TWn<zUT3!jM^*phu21zsC
z1)TqnmHRB;NcQ}8kZuYUescg<_%6u9G{|cF_lx+8lTln@3=8#YPd?c_nlECHV3+E9
z2V6IK=$BVkHvLXoHL%?Suz}J&BX65%&vKeQW?xTWi^ocYEZjNyH@((%WTykF6?YZO
zdM%#NxYTLY%^Ei4T=pc^E;_np2P7tL8?#c1+<s*qnRk7j`C0(BW8BCedFJhWe~G0I
z1iyC4EU2|xp}hC<UI`#IO_ohni-^%W!=p79?FhODj#h%N-JFcNT({U4u^P#9mBl$3
zdz8`s7h4?gL)J9{bvA*<$7sv+$3%??Zr4*vA7&GmLYqgrfC}(ox^9A(B`)*Dn&ki@
ze4STO=xhJc@xZp@J?PqDqt5yCkt~EIC1}O(c!aq{0eB$um&ZfZ`D$)t9A_rBt5^>>
zjrvhuYyhVuB5s!9ko<QPdruDgS9b-U`(X^)PhUl8Ay&3^64*f}qj|?iaq`znpu4`^
zgXGCjc_Stka*%%Y;1{*dhVzJ)R||K}H|v+4y5Kqf=85emZ-ayx%U*q}jiN}n;=SaK
z8~^8V-R2g|Dz$VK?_tWx>;i^BCu3Z?D<6nwF<#8_7kON!gzh*>gUHR){-hompA{6p
z2atXx0?ay!kP9XLEtQO$Sh(q$u9gfBFH>bT!tv(#EX76aN4N{v$d_0W5400n{iKfi
z@Zj0Fg2nO#4L5g8dR{Cw)X<_(AE}UaO-Lb9qzy=25UvDkNyg=m@om86%AM!gshWXX
zFo=AcnCnyD5e!KEdK(v%R&sg7)u>6eE9;4tUiYJo;<))_!0lB&!mJ`}cv@r?xiKnl
z`ef6KX5Met?{b=Bv7V6GFKOO;$z|Qo1*vvPi)dJjWeY3TI#W#vDs!ub-{lQ+7)q1w
ztAz(~du93PX-Hj?x?47;c4CRMaryQ`8?(hQenIPSixVLEUZr-Uef2Z}_3M2$nrk5a
z!e5UH)m_<eF%H%%ogoRpQv0sW?3QD)T0WF&`9&z*lXgFQj0XlVWbl<*pJfuZ18{7&
z0_zInK54ZF!388AX>2@xpajRqdr8Ncsa5dxC@HiwT6Ox9=KO-PY{OUb?;krVWKEl#
zIC6#ENkLgrrycz^m7qtx5+c2`Pw+Glexz0sXg{aO^PDj{rpMpDTxOTS!b)1M2@zlr
z_IcGsjb?m(3(B}^&q6QWW#NMTxGxV9H(tOO(8+L&1UG)lK;?8>ntSWL0#U2brQ4J6
zcXZjh{WG0^do~Dejd*diIeWDw*YzdgW`^7)Y^%AfeLj-&X05>jngoRhVP#KsdUzB}
zQ%sLZ8wKuT%_G?IEhTXwK_YJ2^)=;>_N$d=m;836-21!n5qE+2%?kSY@As^jMC4_H
zZ{TXHyGe8vu48?PW8{t?i7kTg8msG03E3wGUH3y`HE&s_Yqts9DZCd!Ob;l}G+$8A
za+aHiQyiWz8a2CY1dH11EKgAMu6EkMjLM>TJ=#3GNx##mD8yf*zbz&1a%VRQEuFl?
z^CYV25YQ<uG86E}tLo>QHi|9-Tn^uA-ve79De1V;i^CU2ujK+O5>DggrB6EJ^+`g1
zeCMLWFKA1Z>@KOIU#0yrM*H~EjuzSNCq5!kBw}75&!8-h_s}g!-AcmR7Ie~#W0l8Y
z&1^5vcIGUFP5GEHDdh=Pn<m{UPy0i@Fmt#B>}ngN;^3-n?dOo}tAI7#Dik?1F@m!B
zHF62ZD0!MBp72z*KIsZxFUY+KK`bDw1f$fx<PP=buJz>8$wvtZ4+zW>s<oY+8ubk=
zUGrXUo(Jj<p9@6Jg#c*v=YW%|Vnhu;p&A4vi7xE4@@LF`441@Wk#X#O-Ql}X$Z;`(
zSZ|(1c?@o5*~>27KO~Mzy+X-?Cv0WIU}PL=pS#NOLGHuN_==dpzFIJb%(o<n-Ef^f
zvP%}bZG!kKD!>J9xRSM;14t<q^b$s_F^gE;)Et2e8`%M9#YJ~|-;p9_?&3>~`AF8r
z382aYG=lz`MjCBL$lCT=ZWJ8%PZ^#c&%5#XH7n<QEx0jazGf%vCvG7-VSH|)pol_e
zIiq>c{kSXpV}JVoa2D#kN~W)b-8#WruxMfVBEP*BxX;iWje#aQ@+|p-m;F$&MDJ`*
zrpB{D@%}pT*7aJ%33(PD+z2Wq!70mWc2%*ucs+C_7hN-APN6h|A4m2c8o`(T-1Y8<
z3B*T?o~dL_*Vgc;S-^j*IIo5B>VDY(X7~+<e|LJcit#3FTi8gzbP<44gv2tPD`4mk
zNTF|WRPu(?c&{-(Jzz?4Ec}2db*uDw>D?;zhRScPY#SCV33j={t^-4Vc`YA#2$MKk
z<_n7?L08cK%DhA;s_qv74g!Pz$#@NotPdib4CZ%?r^f+0!)$Ov2F3}6UMo6ne{Wq{
zO4#(?FR1Hb)vyUy=SG*PAR@~F@&SX*E~b{fAqU-;m(;k&!6yN2YV3lTxs<-#-C#fv
zYL&x1#ps0Nle$rX)(}yKh%tr%>SI1^-y`?H$<yp_1Spnv>@MVYy5#e$uF)w%fcMw|
z!Nt`6TWm21Gv<$4k?P{vx|!0p_hm|^9>h@>15|xn_3(Abd>gi|yuVipx*w2{G3Fg?
z40D_0Y+)9R+qDaS0XnNJPE$O+qd;*Tg8Fac$eUW%)rDlDAyvW0wrV*#%2dPNdl!Lg
zY>JG+>uM0(oaK@i38@piP2dH}9eXBi&yEc>c0UsDOJkmw1NaAFL<kfn&MF9S-j3`v
z0x%^Fuf(skT96f376s1~-0r^1ilI&5p?2dnO5J)5`*VaaX=Gy;Vev$z*SlBaI09;-
z^3f5ZR{NP^WO7|EOVyh3$bItRmw{I}&g^T$!ncq=w*)z$vX!c)e~NCVBCZ0+mSxoz
zWA7xLrABR6$aVy;2Pgn-vyNtjHJF0lJVeJeu>S)^{wEwJ7|<$id?5a9UAw+;=$S51
zNI$`)_(ElXQr22CqI1j6o<FA5WWicC@D`A|6%m%3evk7@C0kM=^v-iSs}pJJP8Qqz
zh=R5Ts$CP|hes2Fa!(^qbuG^)d4KQKhXv>Wh2ist7|$ZZ)Z;?|?2ZyX#tgRkLX~<W
zTh<9;P?CL}yBUu?FRrzs*o~}nnPL=snYT^Hb=b&vzXniFbEl{zHobF5Tn-v<J2+=p
z@kll17(o0;(clJSk1r9Qzj#>C+y!LjFizarDa4L?mz^kLGu-$hC^M$7warEQ)sb{$
z!%Gwrcg8C@YpS<Q)4fEgk*ZvFz0=zXun}ka6VJ-q(6`;u+ql<$HjF*crE1G&o@IE{
zDO++X_*&NaBDzKu@PcJkFL^W!t2J&(#Pw2Qy7D+yBp-^wA-m?sW#=sh#hlIp&0Ad-
z{iMYLQ&hVI>K~5MQcrijy9@u?K)E*}<o@<UhG6`FyEH2PM2-wH$%>$dL%4qFgV$<N
z(72E0h?YZ&@XKug@DnBn_IqWkJ2zZbSy#5nUExJ_w1t0vMDs~w1u##n7Km}>?a@R%
zmYI;kV|lgnsKz6fHq~H38L5|UXXYvM`VRY^+@Ng>9iJ)nf4c6U4qOAHz+vxP92K5a
zs~&&C4A|Z~@6FIl5UQO_rRn0&JP=@h#7ZNNQxlB|_Ob^a1evMO;BYO5nE3eYuP>Ol
z4fW(@)+*>aUnf?+djQ4_Xj?Iub2QYTh%nhsg@NFJg_h$WFJKK`?PYqk&p+a9<*aF(
z-7%tVc0W4{bA|0oqUll|!_a=f?c<s<uhz>~=F~Dj9@9@660W$`z0Q7AP7q$?@bP-(
zUg@F^#5e8gtR9zFN91b`*FnT9JFqu>fgY-_Jra~nWjL;av6f5hvA9pjGHB$>ARFy9
zQ}u0~mPn)>1B{=2?P;SkCbY0$AgN*F0stN5k9*Nn%j`<Q>Y+r>f96zrky;5T9TAgB
zB3>F;YIF;AWy$#W%ab3lEF?D3ix<>(ocWHW_0lu@B7_Zo0#hhHDdwS26`&e%St)!@
zDjwOm*KO<X%lb#(@bTTf?j)&@3Ev(1vkAU;xmq1$?Ej?x=XVK+sh~-Xt+I(e1p_Gw
zIu4<3VJex=PD%ThENPC#4W@=zy0*}vg0N;VZcahy00<7PiS8+GKeURsO@8-F8_N4k
z{UR+mFg&g`$0OIEgRD3M=*2SJ)-Eg$gd&U_u1GmLT?vh{Q(J>y=127W5fcnv={5cP
zq?t8Vt1vu19(tE*ccDM2B0B!VVi^I3`=g8s8su7Oj9c`|E?U^Dz7rd`*=Rq+q*al9
zG>{VaI1HA0wO?$i#^tGgs9izR`(YF3|H^5yVSg^AxR$JIwM0uTB5~Wym->mb1ZEe*
z%mu|=svK_tBUi8P-(cL^AERJpDkNAOM^tr4T5Zvd(bsok`r7~aTn<nW6}KDxTYTBe
z^RA00Q1o$1ke!K$XLN@`w4+YqS+a&9%GJVl@HNQX5<&TL)<}XgE#H>5pD>zlEdDhn
zEzzm)CbB|-U|@8bDVi{VdXHexPtSRU+<={H3g}UYJGEqg8d%ShB(AXbB8<a^RJ*tE
zSV+kUBq$v_^=B|!p}|Ujx|9|88@~__{=HRgY_1FNRnf1$uj}y4QjwrUG{6?Lb|T65
zwS6j0_iCb7SbymXH~29=W@7p_=nZMms;v`m#?^#5nw;Y`o<6OQcNoEF9xo@O)%oOz
zcHve%4olja##h@|jC{Wv3d*9oh~;v>mPB&l&el==O|{^7@w7kyeAF-%tyi9nWV?p9
zxji)qn1%JF!|z6XncVo0n&HNLQ?!caai6uaDDqB*sQ&UtaoY$GcyV$#P<1M^QPUBe
zO3ON>)8Yl?>>1W3_jGtb8Qyxh-@*&wp#IMwx=da)31mb6T`m*E!+R@Yv18nzE;WAq
zEP&&!Hn&J8fc|KsR@d0f!;5PiRty1Y*SAd}g@DYTZva^j%U_wF6HkC@L|zvf{A6zN
z%4B%7pmODPK3?UD`>tBf?SK6+Yxu5fhG`xz8ORiQ6*!MMhbU{vT*e^4vZq1Ec}NpZ
zX2tXd*P3HAGO?U={&%!PSI0(w=d_;179D(dvsRF8RiLuv(vqmL%dJEC$5_JT`0Anh
zEEfgdES)ZA@rC@xy}x}z1|_<;@NPXMK6~^=Q>)l+4|toGyuShOztU#C;|7H6LUE+u
zY4iX6DQ+!I<#uPplW$=vxMS#fQ6;SUR~4<qmPM8X?eM~v=P!<<SUXV{Jw7vND0rj4
zv#D0Vcrn8$U-i;(GeTB&MmTRTkAzf4VHmRqmp%xQx6_B2>M(SbwUp!jU<{(WXxR6=
z3o#>s8=+PEbM;qhAmE4k>-@TBdg7HRu}k*cT`8tl-&K8HD1qEm2B!)t$_PIjee}_}
zO%s^m*qm*fRM9EHY-Q_6{z>LWpxezXvis$$s6U?2!6@}!cV^n9O@06SviCHl$lf?$
zhwJN~>LZ^^$DS0C2MZmAgWP#k*l%8w328LU_!M~(bnGct$IEmh1A+_FyadeMXoW5f
z;y_i@M@6c-?r=2E5~km}R#!lIC{0Dz*fam5f#<A4|9t&vToyrCd<s+YRn#FEXHD3+
z_tg9c-KaTscF4640~&29hoZyu`g)hY{A8J7<xg-Ql68SO{p%5Z-N7>w9<RD|^K(7!
zc2nVsgv5*{ll!vs-Ts!G$bVCXra4Iqg}UQgGMnMS-+?FJ58(fm9gx7cO0MHz)Ndxm
z04zpK6O+tw=|w)WLQJfHPYjdW9L(UWwNpdh$mf2+T)Va%Ifv%01d=*-++DBzgmXay
z>^Kyzj}hR^oWAwe6k!q<JHh!Tv7j?og9=DzGnA_!;3NhSn9>&7;=agg`Re@6`hvoh
z{f|eN-c(cNYgAz^$a_cEbrVG29Rg8Yj}-GZ={j~dy^m7EoU}p=lHzrxKejr#+mFKu
z@SJ))vQ>3aVSnY?N+JPNMByLx-eb`;GMl;gMwl?*KE<mckMXeWOx#we{>*r{ij}hS
z8J$mU{iCSWEoOJo(Y|zRt`{#4XhG;4Vb}h_sQ+%s6w7Bdx%w`EVsb0BOabudF8^#Y
zvL)8peCIAn^J;7hEd^gVVXvLaQuA#OviyKEK3;?bN`F7ubDo_2xqxyZE&(U=_B4VQ
zjai^yr{Hf#CCUJG@@5MSD?RNN?UzW}#-E!*-12=qv8p1se)?;~@Ht{uGBg&0eqPMI
z*{l$ESAX_q`a=fvW|aHG^qgIJL-Woh49W@kd47aHx@i*O@u$QNMmNI&9^TLKe49UY
zcHib>d?eIwTOr=L!-j-a`7Gb-gP(Qq%Q}C(N4xobK^Y@3=l}V&SN-VlH>F_^RoBSD
zbS|R_cr#}oh78x*sqMfD+ZRXl;o#&4Z;e<U6{pxnx*eo6Wxo7txa$mv7H0*Nk%;X)
z{#ynpB7P7^B`H2xOsS#rB+r7r;^dCH-CP&mVdP4U#Q2TL9mw>>i*^3o>XWEeR?0O#
zsjmAQc*x>?8O`B7&Z(KpA%77zZ%0dHnFe&T<ADX5VCc22>rm-l5mhwy8DJ6Gr8<s%
zsqRT5%?G#*Dx!5nmM*VH0Ku@V@i}qr)FOk_?k~#?Ezt?x#5pm?_YN$S#VPg!4$?@~
z#p6Kc>M0;}vT5UBZ*yyF)f%xj8;`f>KKS5Cp6r-R!B?8wk~Q*D8lH};cu505sK;tg
zwX}oLO}OaGPx^qmPYPU`UwL-ij8R{71^U(Ebe04CbeP)xQSx|um#5U6<uKM@Ax_1=
z`A!cb%#-!|o@|)^_(0Xem!j4}?HzE+9~@p@`!1*3PyTa}2OY48j2{;sk21gG7Y96~
zCk!bwO`4C#I!%90rR`loG=YcJ=Tiqzy9CmqH7>W6RjCcI2dDbV?$Xf${(JrkyR4!{
ztD+he!<=Vt26<h6P(h5g8P1f-6gTl-9$9KcS?EXIF`JKj)&Y9dGfh7E1Sof!7|Mwj
zo8P5pf<$vQhKr=?w|||t_sSiPk_SWIMdtpUuc-{+e(+Cq>2JPC<f#Uca!hr#sqof@
zH)M`}Gr=O(@7JvT7_Ub=E;~g-$3gvVg?HD^X_}ewu>*)&pqZ7Mu2lqMY|Uq6>^_dZ
z*s|0hH9GG=$KwkI+7cdfZS=#Opyaeqg?gOd7u)?c1wmLm>d5hWh;u@)0G+G0N#0sp
z`Yr*v0jwMSgI@{?_U53fV;sI_baz#0jxM944v^Osroku86S8i!Ty>(#XT5O;Zl}+9
z22_GVY~81xe*LM1>J_{oY*e5)TO?jm>HW1>W+dWs=8^pi_@tM`4G?<Qj^$Kw6`{JZ
zK21L3kD*x!sI&7+=N}eSPt2*NpAFYU;|v5?Gsd1Y2r3=l$_p9ijg3`j_@w<^>YL*R
zxdFFOqW3;qH6WN=%X$kO?`ec}wI%%jb(2qcIpZJaJT7wgOL?}5U$dya$mI%47_(nx
zH}l2vY##dTKpj@(cddU|F$gS-O6>vyN@DJqyhPoj0A%l~zZtD?)pgusLFX4rm&3q8
zbGSP}t~{%a2MIj3tDUl(oxo1gxpwR{z7ZP61Q-PgSQ^pqJO&&rdt*+k$Cm|OZt8zB
zI@qYDvR9!hM4>@R8S!i`6aN)OQO(Bmt+!0TMMTr>mlzC7?PbDKEHrbGu%WsE?q(Bv
zX3AFVT!8U2Rzo(uZb&vIP?~`+Oa8H4(!);H*MGZTcK#ByI)TNCC65Hz0fOxw5wBya
z-7gNg%{k7$mFVcWZAw=u*mse@sz;~ilKTkp-?fK+naFEfc3`Yev{%(a+IZSgKxONU
zN}tCQV8jM0*(;^2?|t=!`e)^~PEEcY0UVc>JV=5ReF3a{q)taL532&d(&OMNB7|AG
z#=kSgHGbS@#vb&*s2DRIpYV6XbZzv<+WUVRxTV4Z&nEdIFK?#7%odv#ptd7kra2{6
zo+~^HA@Hipu-M{_tPfFcZyja_r<Uhb=mf0%bn_g0LcS1i@=xouN$ae$Gtcrc*=xwT
z{(~$M+mE#`?GF~bl(px7Rl)2uol@6zA@PfLLvpJ0_<4uUCymRNO)6@+r+boyM`69;
zZ|R&8bi{lqooZ$A7(@G)mlgqhROhBb(v;e>V7%6cgq1%#ep{9r0%g5SJm+!xYGi+I
zB`TCFYz!J7{PJ<wz^`cv58gsS4Um8afXaEm!aty`rN*WQIv>cZ7mU(z|9ht({(?;j
z9(B2ux1}@zJB{&}Fifo7Jz(o(&$BOX^>H1{L*Qi4Hno!o<|WT)!@(cdS6+Wm)7sHc
z^~fC5(RwU(oACJFT^VX&c?pc9L{pc@aRjYox0_p4*Dt}k=_ZGLRSD|vRH>mU7I*L-
z6c*KbPu6L-*GS;GhOqCcFH+52raLwRN!Fi+e|@4nj8kdOte_cPdf){24PzL1-TZ@g
zo#BWYR6aX;_mZ|C12C$1*g39KaQ&NnJI-6v!>dd)VCi}~0(3axgf`(c(-V1qVj2dO
zn-^VG<8T&i@0-qA10~U@zUS6nSFX)@_E}sl{*mNaw;(~jI_dc>AC}9aa5G2STkx8=
zA+hIFq5uPIfY`}i4W*&ZiC){F%KN2R$QBpsB&9e<6XOJ*<A>nw^)pzxNW2%vv&d3v
zXF>sGE|u1E&v8+7=I8+YF!<TpV+ZtGzj~6})@c{7XMwck)jUDE3?MGu^J0v`w&u3F
zZv0QcsPxc;^m*nhhYWYm9JQ`1)a<`g?$$~`&n(OJ(L27azPGo4WU)!WA}P>Bt^Q&&
zBD1+)Jp~v30tKg-)V5gd5;$)2&in&^(am2+pBZ-K_FCw#7eiz97f9c`-IS-_iLz>X
zW_kPtdpYHS6Q-C)O5;r8^erPx8mK27h=qYwsyHd0h*@;0Ua_6oXBoVVWg^VXjxa^1
zI1R9>nT_|v(vL5r2nCA%CdhCX%TrJs&tt^|hU;5lRNR5ANjiJOFlsV7Zimb%;Hi_D
z5e9@h83-!na39r)568q*K~_gJe`}qcT0As~%uCmJ>rnd&3oVBw2COC44^W(GU0ZE8
z_XP`<jewDonwlbvP&XVUi3cN$VYsZ!3e)%?>V1G~9+4FjKK>aX9VT&_;i7qJDSPat
z)WvjhZnR!yGXj2J%nWctrXq(f#p@PQPG_GzABZnhd~MlfmVHxpdHiHjn*oZwh63nF
zfU<zuytxRGPrlzBGh<4=X`5lNY)+J;ZZR+%-2U9gH63KbgO*nMcZt{uNMDw5iuvJ1
z686(v;4U56hYTJM&sMj>|4xoCI%Uhg(2mh?jU}zb%P@@Aq&$5<DYi^gF1iXbx4UNX
zE~w#v#(qdvb1x`?WZ`_S=eU|w9?_H@#Fvs?a!qh~_Fk3~EamW)dZ)h3|E2Gj%jJan
zU#8)OT9AjKQTUR^bAT+BHk})22_LDm*@XvzgQaA>trWF{RT06F+E@#g%jQ+yz?K8e
zc*bZ;pN0;7DZ8P}+r2(p?Tm<$U)kI?nWw&!+Dw`<r!*pmF_Z4TJG}6az-5U$y5jR~
zUjo*a^agts9LYa4&k**lywUL66><Mdfn-~1IFo&^Dwg{R5D*$`etdD|?~WC3Q$4_}
zhuUij?@<w89ANA;LLlLc-k@ViWs)DZ<UKM~X64Qj(gr0373Y!uXnw&NfT_9nY}lDp
z(}NO_ej+6O@4DUY6%Dd|PB?dVu^H4xa>u<_=!cWJLi9jiu|`hiSa<0w=oj-q?sgPo
zOLvfGAgR(Kx@QJh)XkzIUVhqc0?>6fU>|jhOZZb4mZvV)@y1h&Ro@+P>$skLSP^QE
zcZ6+FHjvJDGaZ~~=t!Yiwnk~}?d9nr$S3{5G0QA<Gj*MJwzn_w^<9EDz2V&=M5anX
zPL%96Y;^~B&sY;Pw!I#9cCsUGAoLOd$LDT2@H{@F1gQ<1pncu3UQ3CX8Y<T-`JdC8
z|JNrIgpsu|mI42;3eOFEQj#X~gZ5KpEZ?-=6D3(mIY2syr`_-Wb#D85|C2m(i?<QG
zEo0|jRsiv?Br{|XAaE*;3XXNC#}Cqd2rPbzX#1KwRr#FE_gIiV{wcUC0ObH*c<(^$
zdKChB>mZk;W5IfJHTTag3MrQnHXdv+7uTMd0ne7>rf%hg6vpF+lVh*a!pcJ{_dZqn
zhJ46cvs$4S@rtoS6bMH?J3YH%+=h%8Ri!}uhz%}!{?6#DvWx})$^XY1pKA1`W+R!-
zK%(DwIENFlFAu`?GGW+#k<GFyZ8fWRZ){}!Sp|SI;$mFD{YFy^54z-XKQPEnaq3UP
zF{t@pzjI&dW@b|!*=1#h9byqb(4=#>cplilH7d~+<|o6eG8@4?Pc?3xEn@<h@r#bL
z`Igdsr_4EuP*Bppp}GhPVfX&0BU`8kccprAUYw}1U>Pzg7f4P^I_()4bOagWZCLb#
zAKf(Y_eRgVT29P^&~#p52(mMNDP8G->oQhs(z8;tr9RLx7`~`6jN7WU+;3_Yb&lSB
zb><fN!EP}~pUVZPfUSU%hP+~z{{<i|*iHeAlEJa2%dqJp%G$l-==8jpRRAlp=O@tf
zCc2JgRBH}YAB^P&`Eky)<DlUbyXBdXJB?<Sm_B6z{a2=tE5{kG$cvN!w8KDCpwE6K
z?q%EIFKE1Yt5ci(tsnZdAEYPj*&e+i*D1z}HnR7auHF;J{{<+d@o4d>Ttk>dDkqs;
z?!;<dy|p9D+COsaU31Ay{4Ng+^Q&twFGgK|j*e0soeYrx=44ONiE1)>w+HD{boVaC
z{^j)RP?Zw8#4MGUAoq#;x}4qc^{PK7D9u2Sz5CwH`AazD8ltLvUSaY4-y;N^+&ZY&
zOTnXJyyBS)Z{-VqN^vnwAy1ITM(j;bfl%>e#z2=TV*cbeHd!%&#?E4#G;Yqr$GvLh
z8$SV{mW1}5Dk3Ff1I|!`ed&wGhK+4kqRbr3l8}h`df#!#Zb8pI2If5Z=RHtv+Bu<Q
z6C;G+k?GCY^ebrGAe7?FvDgs6_|J0OI?~SnKDy3?-kgWIP_xIHUsOrBnZcRDfQ9y(
zv*ou^hb3U+5b(n*F))lzvDF6LfAZNfq=eQ4*Oi_9&ggl*j0@zrPP``yX|R<)N`ab=
zia!HUc<c&FZpX}8twdC-jD&RdsgkE_kon9Ie+$b}zCOprlLuW5T6V*STzG6XvCH(J
za_W43&g49+v7U{4U(0&?Rz0}p#$FEcWH{KcH3WQZ`m783FUP1iPuhW=$ro5yp18?X
z-!bl1YQcNnMcKTbB%r7@j!T?<l3>`2Y(}zHS@c@CFOb!5t{nBI<5En|zvfLECZgYo
zc8)~H5ke9_u1unKf5cXnh}68!+oGSGWjm&Tc^(Eg@Cx$WNt}@u3X`Ede|uzOfE>VF
z%I`2}MOPZ^#17->|BrNwLkBm1uzlM`jp_gw5M-`@s{3^mO%FQ#AM5u2fq03xx6^<W
z8dr;_{u(zNoEi1`s4eZl)N+sRSw>uIu*VIsD^=hama16T4rJ5-|9UC*H6<iE_^VrL
z_y@Cb{Z}k8r7!F`nrQ6}!lET4ewHZiJnNno1x3Ekh`s_4y*6@FhNsgI*Z#Q-mjSe3
zHca4SwSn8(ECdU%IfGdin&?8q;!4-{cti_%R!LolV;~yRha<rmI89D}JpXw7$AmnQ
zu>Qqw!_{zeoE<kz&dF;0JXp1FSQeC1XFvZDwK-`>O<iDXI=pf9id*xYLxz4(@;&<W
zT~eBsZTVxjg*3rIWzmLY#WdFO)a;zNc!P1<BY%(z6Q*;`J;+aZJw`Yw;l?>jV{+wE
z6(?D&tX71guGJaQgkwVj&i(5LPd$b~-xyBDwgdRH4guM|dfW30dH3G0tjE*Cz3&}j
zLdgIO1f2{|05sN|nEi9UNxISSJ_2!MGtL{)ywsX`bz<WkcK?4yp*|1aA>mfUzDv3$
zmxkkFpXiP8NdBN@7v#Q`BglDUJo;U$(ZU?^+(9hvxr4ZSHhzDY?EFb%6y&1`q?4}k
zIz<M;2}n>df44FxuM8m__dG7#qi64X?3V8KjUcCAF|7t^zGyS5edvDi6s$yjY~j(7
z9&2~9RzeQZ00mu?fW9{?{Cax-sGtR9;g0Qbfn+hb{vk%7Z}>s6s{rCg?3w?1of!<9
zrA~%z#_ty{|E@@#$&=4D&RDSF)_|zSX#^c&vlGRSMlOg!mm|NQ$2PFIc8bF^gN7{>
z1xXm5t>GQlDfYgJ!2+I8`}elwjn}@f?aMqm+-&+Jca|`1-f@6nh!)T${Ce8IgS>jD
z5##X`g6lng@NCacw~&q^FLBYYj!y`7j>WxR^hZm-bNF=UAE1v=J(xdHbwA0_i}yZ<
z#nki2Umq^xOZU+T;>a5>f0aYDL6LAQd0TVV{1sqX?(H1vxTv8~<3ySs>sD?3gmbrD
z*I%u|5uHd(v@y(LupW^YYc`L)tVAC+B3TJr^a20yRnrPO7#W|vbQYx_jQtS`R$IJ=
z_V#ZF)%0etU!`P&F=}mFW90q5SeI2~@$)MWB?Rl$u!MuK!O1XOHNEo2{`AV=CeY_*
zQ$g)_B8xrW8vyo~*ezNeAZKiseNWV7OUUa<eizW)cp7)WjTzE$FCl9ta0ZgOHJzG3
zE)aS-F6lhT)-~ea$H0j2=AMKg5an}#;^vKJCu`GAmTK#wO7rxZuU)?e>^OhraZSDl
zur$#APoIBAREK%p`y$_c5hTZF<V>m{*KQT}lh)X8;<(qt4q&oV?!ILek!KE%E9dzr
z#l%j1A2cKv8~+`#F=blD0VaqmFMvIeH-gCDZMy^`Ht7?3!y+h-Q-Thp?EFXN#@|<h
zhSQjxIgkcb8S__}**$=Ji4nTT0lwZ*AKNHvM(UK_dKRW)j3=?XCm1Ix>srK@rK~&|
z6>>g0ijPnQx5a17d*5ojoRih^?|5<C$NbZzwSh?qceRHIj02HR<{KBG8p0dbZ#CXI
z1=wi^K56DCNb<f9LfC!dF-@YN@ijA|YzTJ$(JsL`cBkHb>3ac0Y4-k(A7lna)5!lX
zf6yL1uSYc(+U?h#1)LJ2g9W8F&q)?vLD_3^5bZ%CaqbUXnT`O9IH1+p=ecH_V`c|8
z9UCD`q6AsIov&RccV70)ZJzEZ4ZtBN|0`P1L<NlpJPL43sl_M0lORi=7<dZZo8~3n
zUKJhy#v8Wt2t^j9@^jRh&!r8F0-$EQNIxpZV^`PVh1B8)^9|nuE&_2+EDo-)%qqBT
zT&jKv4Go!w%#pEy89e4IjbvnoMqqEFwqCF3Nc1x_&y;M?@S`zhfV5q04LX@;Y9zaI
z1sun2?74rlU%cm9?x{8l(2?8>zm+K7u5p@w8{oi0_h;O{+rU{~Erz-*FL8~y96S>t
znEjde5ImyV6mJ4$ghKPuy9V?uGPplJC|XmEX8#$Du>T5QxLTXF{uHrbf&iDd1ZM=U
zJl7c1(+$$2#^wcVV!R7$r;heEK~XI$3(AR{NRz^$qg|j+Wlf(Z%|0RS<Q}?cbh<U$
z!k6Hr0{a_kZA54L_zTLLrHj-SDqSb0IQc6ZreS2s%=DNQ@It}*SHIJ;;s9sD&oiR5
z#*&r;n57mtdY*;`f6;5_mHXtQ%{V0F{7u!45l!Mq09tK^>nuWRhFJy~RfU%gEJ7yi
zYsPKsRM`qh9C#@w49a>_UqW4i(&mHa;0X7%WlqG&IV|)FayESDy5I>8=K|x!+KiP)
zSm%DX<+9KHLOuIhE9Z6O{crq6b-AliaqGperjBbVd9@2|Szi(B{Ch>L{!i_h;&VG{
zv^ow0I-eieJlM==4gj48fsmVda|@=(bKh(a<mT@D`DVaysrIE$GG1?2puxnr-b`AH
zX{)H&@z3TNb0%LGhR$CqqPv5E@Un&5nbI+z_UB}+@><<aunlhB9ls}aJ`Bxn3;9NF
z6pswOEj4^}#`zLjHuJNUM8Xi(yPu~&#tfklvURo5FRmTwfp1h<ovM!i@Lo!v5qu~V
zzi5JjJ?UDSYphO|-!7cli^5}QUAnF-{&38W|KW5H+L2(>)44^6w(l+bpyJM;BJ*R(
zCwUnf8f#)$rL}S45Kj@x{2^~&*Z}N)uJDnmpnH325DV8OnS{RcWR~L<3SPj)66!x}
zaZ4ng3HN8<gbotZTgw$V!>j%6hlpJ&v@mk|*AsNQck4`I#(&Grgitu{_hf7jn*FU1
z$?x8sYttd7rtGOYxl|ES!pyG;f82*%#$I;cfiYX@WQ~GGT;xit?S;VQj#=a$**vFL
z9iNDRg?6#okbK|0h}8{9&~^F5{M%=b$mAZ}Q>yCS;U%71<A3EJr80d?Oto%(&z~`l
zwoLIsN+X>Kb4?SMG$)PR&KI9Fl?xr>xN`M7Br~_sW<j~G`SC~GY)?<pjINiVxUWug
zvZS>PQYVz>$yV!wE(q4g2gB<z!@-1anhsm}2-~o?s0G2j=k75G#iXQl26!sf?QaQ6
z#%^C=veZ_iLbs>)G4`_bSb=KGjl|8<reYWALhZ(fsx?cp)?(7u);%Tb6}#iGsv<F^
z*e$YksHa9_a2{$?z$e+sLK|D@%l^Y*bN9#DOhM%x8FVkNL(|7bGV3AQ`;*1HI?ZMw
zkQAqrCN0R-vDx9MvcB<!VtXEx<nsr4m$0VI><^DuplwG{cxtQl8#*?89d^%6ePW!N
zAD+y*KGA(aZ4Z}>;eY;HZ(aG6*Daa=_EPANAWsj}=+DD}RDDL#CK{3_+eBWwneDTL
zbJR|T^^Lcg_~|?HyW0Jh#&I7G<o<@Qj2j%coW|&NIF@3E#&=j!-?`*vrh_bq(xnJ3
z+EYtVLxayrVi&VT{Uzs|0~BTSPb_&ocmL7tWT3c{e=6cg1!Te))2q!m(jy6Z86}^h
zLiD{uOlULS{IV>(uBTff6#B6wFZ2T`&zEETYrthWryrlsX+^z{zH<=t(}>|T7MvQB
z{ZWuaaG%s`xVV4B7XID=sj*NVc8fD}`ue0_(^6@DP$Y1G(h&g~TdqR=P{Z(jNN);O
zIhC{;oE8i6J-iTI`{9pF*H_z{&Ay=Dz=wY5Sa^-L2w9?)MBWXWG2__%M`J3uhQ(K?
zNlHxZz23xyU`7gwa$jF$;|2nWav}2Mat{>rJI-uS5%zm3<HhF3NWOtzhYf3blf)5!
zaz00m{!%bSN<=)-PtX40%wC`9N?W5AT5(2%3dTiseE*t6Z{zEnx4U*H@#0eTt8|gc
zE*0DteH2BIF((Uoy-2SOm+5Wj`T3wQX+oZ=Mux4Lc&9IRa%3lE@lzW<Q?eno?9s#e
zvG&hLfuYjXcV=Q4A;G@HS!*_*)Qv*N=U7an>rXQ7_f`}cprAsBS#v*D-FmA6zm^W9
znn65gTfhHoZG0hm4$bIKm`Y^;{=!n|g=W5dOj9Ew6qJ$PW)VisM`2pM!F%<SPs@Zp
zwtpdYOtM5ksfuaM<Iky&%-wIEAr9rue{oiob*{p<me4&<r+EVLqdp_eL;}Yq{6$qT
z6zA$T{br9ghQ<<4YljQTLce-`fcG&$ZzYQcP6dAdiEWt~si;j^qzrCb{9+@=odGko
zz+js77laICiyR*q72uny7?~VJg@N#$k20C=ZbphHtkJ_^LsM*rP2|ka2&jkuDX*_F
zTu4YQA_rp{Ul>69PZKB>WGFF2Tu9+dmoBSmKP{2mbOg;)1=?#2Y6PvB9KfSjmaz?$
zkfUP8=^Q|oaYs&?1K;vK%g8<W%&01>I~vTr%Hg=CI?a8AmNJiIdz}tt5l93zEq+dm
zBD(*VWkt17B3+JZMTA*PV(g{(E4>3Y1_Gp56Uf=rH_#|6aYfhJ1y1sW$(v+pyj_rb
zPjG06{2`70s|1gZ9(F5ct6%GZirV%`Ihc0bLZL63IcGs#uhJ=Fgh_dIZT{#NIlo!&
z4%nF95WE!bG-qLSv{zi=kOG4G<Jz1|l$Yn^VvZ8#V$$m>X)4m%8&{47@hwOH%;`Tg
zN{(er>uexIS7}|wvw5j!<i^m~S6%K<j93PKEeS*}idSwsShg2=HhIB<k(U-6`$xog
zG6sV#MUTM=0+5Ywf3ME5$j0|i+9T7*GEXhhU-gzgs7@@`NYjIBxqeG~I`L+gh?__i
zsb<Oxt?;8(CzbR}`ldwK_l{KY0`Bq0Iw~I$-?B(*qmgim6@J51TLov@>`->~1PM4V
z7y8j@Ws6#xa_wT(N#v+*NpQwi&>St{uCJKeR36;XVnCFnFx34Bn4>InDx5EoY#=LY
zn$RL}Kv~Sbf5JLe<jN7iOJgHSL|aC%!G2$vPTRRfaZp!sQ@rddJ<@T)Hz_JG%O}I#
zdR^7;A?v;60Bi~vkuiK~lJ!>Y?n4^s+_@(r3`ab7c{sf(2%N(D@_X*leabA!&et;~
zN@QxH<cgsb6kqb>tgi~WcQR{}Jy53{okbcN_N}tW*!ASZ1Y4C?x9_+rnAElH60L<I
zGyWZ|gP=vr&OMBNnmzYts@5U9C;v~pe5HxWx|T^gU(XfhxqSx5>nWpl1sn0Kt)v{4
z>Zo523-3D4rrq7F*!Ql#PdINqU899_Cn$imHy&p;)U#^DTL6-4)W-SDEaR!er|~^c
zq$k9>iIH7JHfZm~^p#S0$^x1KTe_;!9Ls-^Ec@QKgB~2YN7%+uH|KQTes=zG2Zl~{
zrKz!lCuN`Z&^AK*y~;257yj_NlVGW#yIRQH>I-TLaU4qaC9;o$=oYRM&Su$$Wg`lC
zEs(`HTOqT+aF%4HMar9G+A+=H_S<BS)7UWFD4UqWyy`Urn&*iXilWPkBX5^8;PoR5
zN&+IlQX*}|_PgNm2K#O0tyfz;;Lq&43rca&mQ67qAD{ErDb#vfdAEo7Oe+Pn^7b^*
znI1{j+8eVYY;G%@9FpA>s<9!&hH91{AI4G*{0?}<ne12|m|V9!YvZQoV1Yijn2sCf
zlY>V1S$xJ+w9g8osKmp0>&h>p>|y8~`YnI>onPy{p*YJr|MQgzL4$&xaIE2T+@C3J
z^_)>tN%<(d-jA(}f-dh;N&_n=bNoo<v?qR+8zI%&th~%;sxM&;L5y{EZSzxHRh3Z9
z@z7n6cXX|Q43>srGR86=eI)oMa-VrG1SHcVl${#DB-X^uXN<<fxZ@Rcx6_RL=G^bR
zyZ(W>qjawiWf6P%$Y?Ff6uad!&T@9!C_u|)<_cr*%H(SxI4IL#Z7bj#r^kLHx%3kl
z7i$Ml_-}?*Sy653Aw2rm1;&l;B4AbRjx3gN%lK_qrP0CR=f7DCDk2-%9oJq@_Xf4P
zy9Rr_RY;3-CgUO*^~vIT*p?Jo9fAMZ{?`6?XhrO^&jKP}E?338v7BBbpLT?=EU_EO
zDUGpJkPq1GOVew&;`4T9Tz7rK*RJQ(OSc<4DWYDvYHO2g{SX_C=Ged8I3k*w>f&$l
z@eWAiOQ=eaAQ>Zwr1^ZSkvTwRug+Wc%(PCDrbh1LKKiK<J#DY%5nUxR!n@c)1RlJj
zm@sCfD1E8wRjgauF=bwLCh=tHxigJxSIZNL7_ur1s#O~`op-mEh^w!@4He&eoU?1r
zv4aJ_*bObk{sZ^RBpQg&f_<a{&xC9Px(W7D?j|xb{#BYKRGr<VG*!_taTB=U5uPOm
zA3l-R&N9J_obVnCZO4uu6!IzMF79?$Yr*4Lh}cM~*!S**un<uuw>4RX@d&UrTs#!G
z;?$b|q&30*i99UgGkKA@V=i9K_8l=#*56H!72Ye8Rrlh)cyg@2DkCZXcREzw@X19{
zwHwU5CN*$9ld^~~x>U9{JwUKFENY*t#kzLoJkTHBeNgd-#fwDy9Si*e5jeVO@4Mc#
zSk>jo+=t$<MIBj2wC0(f5Ti@$T`u*8r=ub47_XA{bXjlh$zzK5FAgA=$>A)dw9Jp5
zY~|%EGDvxbyAa5WeQ+5q@=#5E;q=49SD}T3c_!2UUT-)nwBjEkH6ZKCR>gXN@eWiy
ztqkT$op{zssOb@)CFI>B>gdL<;(cq1(_Y9<)_30Gdpwe@f|OLst2v7k6UxW_#@|g)
zf$1`%9Q#9ZFCzCY<(Y^}NBpG_#T_}SyOO(%VgHA-w~neR{MJU55H{W2-Jx`+w16NW
zU6Pv)X#}>UfOK~^n-1v|P>|e!ba!`s3xDU_bMF1?j_)tWkTKwW=bCHA^E`7db|%bA
z0$h-HTyKdxG|4eV7MFF0cXfIzP=As-?BXp=sM4J5SdIPxu$D2w5*I0Q>dS11TQ0S0
z0NjnUq5hE%Gg%Z@mEIjsJE5q3-hNxVz#x+M<K^e}guX|bIddAPgAIpB52t6djS2OF
z2`AgW%BK%<U;HTmy9&3j*WWUCSc2xDK)$2hA-8hM`R&oX7qbzKkUn78WaTgSm*CBZ
zW>vG>1SU04rwVejQtaT>i(+*HRT`z4zRjmE#$wJv2KTbnLRyT8q7%Ol7jCUQ$ub-s
zOgr0Ri%Z;V#8{pN^P<epVBVZXcN~vRNe&i!^OW6}SD&E=l2;4%3e%*Q$>mv^g^UN>
z+zUl_R*7{32VI|4A%btREQU{uW?r|+KoKQ>hT4PMZt6ehTpiFB#9m$(NY;yQckuKW
z`7VoGN)poGt955G*>IUWJ)NrrQia0x)vl2?$fXBS)Z1V??V*Mfu~hB91`$#P5;^m>
z7^0+6PE_iGUao?45$T1~1guhCjfj=wqUWI2zjiddOyCqYy7toBW4+cA@rbPXoR-Q+
z+0<s-&|MYC_`)T6SuAu}>l=a`8A<?Rg*270CvC*I2?39lFrWC#zm<L`ZVgmzUUk8D
zK$WEtu^i8j%fMkVuuwd-)%$eLDRAV&b5Zi)EE;V0V@wPEis&2UA(In4!G8zwS)TW=
zuHR?6V*6Oc>uJDeKi5C)ynQ*};MY6-djD_9=v`H}?LA`6A!oE(5lPnEmc}q40)wVQ
zz?WO?X|RQ@@+5!LxqidJD~3I}=)D1)-*3m;y^FA`13Dz?C3{?QpleGMKeI(XQ7~%m
zk>LK))kj$?S{TMT5|BmzZGFVs_=DJt<G~=SN!1qj)oe*t=<)sS%zGD@MrACMP5xWh
z(<*WaCN5BF+Yh?(;Eu0pQh%k!HI#AWrq|b@!|+Ap*F@tWM_`s7V#r;ly@Bhd%1tKX
zGcqxFbBDZo<`%#>HLUZ)hn!5v+;^5Ws&e3FZ>`@p{-NO@>$L6ui;-#5uOk1GoBC@l
zU97*jyNyaB0;{2~xiF0+!;w#~hVVh+0jq1api95{tjJ2aR6{pxz2l9#&lhbe<UKNv
zx=BWh#>87$Rnf@{O*;kx#}$9A91-LW)*>voe)&hK^u=vh3OuOZ4qG^7-We3i)M+M&
z$V}bHe<p)+WIoD%HvMr=;BSY8M{5%O^Yh-YK_A<P>u%?r*reIvz)xK_OZ;Gu5Ib#c
zd03Wzz@x*<xzhL$yC}}g3}M^ma~?K7qP)GmbXKB@Dg#C3UDZr6fw;P*iig%f*OoT?
zZ+`Nsn~bO>!lHN=971Vrjs$yAKfRRaynk^zs9ojO!n}G~8tX=5?#l>H=AO^1Y_#cl
zz^aw|a-rF@w~YJ=*d_Em1sy_#LQsNz`E-Rl=#z{bNEIG<-CFe;5=lc`kNnjJe|4HQ
zF36UOjV6$YMuDJjl7OLcoL+_`98!Ca(7*8YJZ!U_Bk*Q)X4?zCO8s(J%WhTgrt3fn
z?cOxUy+Y&KaPGKS9CD@A{vAzGk-!8o2!d~Nq7^9I?mt47$W!GkMbFo=_yYIpdiA4u
zcdu+pQdg?=+`3e;p+S~d(Rml*9JVCcJrBb!H+6^A32L=NmRn2m+sNOebS-y_yc}@?
zNhTMlRnoH~pQ7@8B6k@j`xFOi4)9rxKG}-9+kCR$G9s9+JY#J5ojo=gNiy{dwlv|N
zI~|Opv1awiZx0(H_;?%<S|gWqBI?ieaj9_z9&G5Gj#0G0{_C2U`&VV<=-c(YgJkvE
zYjyteNS@D^&X=pRU!z5nJa526EMzV`dPjHf_fe0&5@ya@DG-?D-o^$m(w;@U^jX>e
z%cwYuY6P`&^h*CF)?w2Xb7K`W<9A;D+D~&Nz2?(?UoGlZMLfw*)pdj$2YuvR<QU%_
zHs9~S6Zos<p;Lnc;}X)8uHL5qrh-#6*Js%MiGUu8YI0zp=Ej^8O|9tftZw}Mbv_uJ
z+vx@}-nOAxVRZaPl4I8+Fou<ibkzZ#dMjs2vSmHmx9t^FMv^(uSOWw!fpIa)AE$GC
z9=kV@KAfn2P_;+-y{ebvUzf!|o@uYUuR|B6W$}tfz61!3(A^_2D|TBf25y-QM0Hf}
zI2@Ax{mDIZ*V*b2#FD@Yg*-l(SqN{|a{BkijI~;lHne>jc2B+?e19c)q+%TF_h*f>
z`e2%$CvbO4)pY*Vb<J$RmAf&wNwe?K2*bfcY2E6qcFuW#XSJ33bVthDq>Rz&Pj9lO
zO?lfWdp7Zg$@GLm?96g03EgVmA))UM7T$N7`*tj1g5oYo`R%`sbJxZfg1Yt^yrvkK
z$SeGBOsnFdmUMfO`8U;ft+np%rZxO_xecE?8o=IXq(?pRVbQ|vWKeu`R6-FX)GhIW
zhWWp)33lzCoZxq0BC$oEv$hfad;8_|iv?kf5c)aMkmWj#%^jzG&fJS)_pZR7w>5~g
zIy4mX=3@r9#t8$2x*i92XiZ=3*6y#a$8`tl8=;PeiaG+8^z-%;mCURm@(A&kbjXzX
zIG1sY2lAA3z?%iXW9Wp@GFmE&Cmk)uy`BRlmqjrhlnqs&FmRA+fyftiV`rNKnV22d
zq``#D>N<7p=kg;a`6I~CG39i1)0T5huer<iHp6G+)6=D1RwsP4$j_^q<!!I@cE$(f
z!<6MlTsar4{*WH6u8(8RK`@2K2NaT6G`*#H&>4ldp<d>i?L<vy3)qKUIM96{#6O@p
z6@C1$zbt!m$KQ1l3AZb`PL2!ubiGPA+mHgZNhk+%hJ6i59D1k*Em7k`v;DXOLIMWA
z=iK~*^@K5kh~H$H45ackBLlcql%3wq$E?^QBVna+zy6UT^1S<#;g{$0Yf(@xzit}$
z^2$s8e6x^E@Vs(kWd)4Mn~Ylj^*J`t(=iQS?Ba{9J);&^bk{>4|IKTIhxuAJqEbyU
zidX2n@d|TS+hVV@fDMz@l;*>mn8uDjH5-<f*165RPTrXIe>?c~cGzsZ-FF^G2kK+6
zNeDOC74!t5S8YdFXpJ1A+w;C^Bep@jF2HKnY<Q^9uC+U#-D$Xk;R)&lzkHWeySede
z?6ELF?J`ts0VnGBk;N^dT<{CZD1r(O%Y(3)8rA6cu={@R`)|Le8$Wl3Qcq-S+;yeo
z`_$T!bf(MZu}!QsF&up0>Wkv+&d|85I|`imNr#>uWid38qjt1fdtvkrjWviqLhGzn
z>$m`VAY5-eUkvo<)$cZqIG0@oZ4G_k=jusbOfZUfMXhY#AGSG|&Lc=-DHQ2<%$<o%
zOf&Aa9xDHx&LLZ(L;l23>05hD;cYj+Dt`#g;=N{2AFM88^DGesGB4fVT^x@1G0SSJ
z0@TyajeJEmCarN>opYRihA@o-Q_hvwLA!Db)Pyk9g;wNvsqtPMuZZ3pLIJGXMi9h{
zkVe4~73lg*aBq%UdSu=!vQ{$S_~l2`fZ!p?0kgqQ*(;`xwU@@f8n1-}1j2phM11km
zUq)3LiB!;S(f)Mec{uniK5|vVmgY&B#y1c#th@B9(KVGvgz~UWn)1eWYr+2k|Iuq(
zuibLv&H<x)ID)Td>*y(x66$O@qgAPK)#T(d`W|+5@v^<B)_zCYhgIR)6$$<lZ}`(&
zBwLK`b3XEh*x$-E!N0s}FK$0*z=R?qs;Hoeac`W9dy&$U>G&63$Nv(qlA2)UgamwE
z8j9^RXo<GCOm@uTIeb?+=2$ZN197xV-%gStvLj+vXH1z(LHx4oCQ*hb>i3fWGG$^+
z(>1-s%zRalZJN<gbW_ei^<9K7*JXg`{xIB|Yg65H+{Mgu`<Vui()3<lf5t^;{=Mjx
zGES89fcbHD0C~=z4YwO?LY^z2s6z9--ObuTC*9ZR)frxTTq)vmAJHEUbp%H!CivM2
z1{<-UQBLFAWVs0OH09$8^LY-5MwNJSo%osUdBZN_R7D4;uP`}-4k)iXuHYY%(00%+
zb~AoP3ow(5l4^-|gwf9;p&#x~$FnII2{|Xyt4AA=?28|ZD&3P9JO9udogO=F83xXx
zfA38;`R5(G|1Ca|xI2Y-b`b5+0q8n>;EAMV;SVsag3i?}Cedn7#MmFAa6dBmqzv7-
zrx_e;>B|_`XychUyotp&n7vG1c>}X@SGGt(n~5d{`|e3^FSy~&FV<Z+H4p?${9e{S
z30>$vN`LA1b|5k{j@TqR%(IG4m|<rCEzMqGa?R@D;jY6)|L)M&J}{A=XH%}mIIX_D
zg>&BU_MXnds62jRIZZIYIYx$?HjeS34>3XSa$>sOVaCH{=@i>1jw-%H_eOa|^s=uV
zlcsqf*!EZ@R}GYfNfXGE+b7e#_+K!Fg*r|v*7PFcha?;?e+4u*HG0CIx5(c)sl#un
z{y%>F>Cyjb>XC4~@m171nb->V^-Krm39V3uy$zh@X_ELRj`EOyi#O-~&HU00cE;1=
zRfk&RrM1s!m@l5+MVvoU(xjU|_rA9H1N`HO|5Grh*do#s%pdFVv9IxXF<8}jt>&}w
z+JxRpYsXV(g4VIpm*2XrgEjj%Dvrll(-6}TKJC+r{FO5>#WJ+*FLuYWUg##(NM1*e
z);Mv}ueCf`;X0SJviUyk?>a$r2jmPmV#=}Pl>V;<D7?`$EA<t;N&rZ4x%^FO&ofM1
z1&}D@q<l@d_oqn>Gi9^8L7-dCc34q@c0dKEog;>L&!6?FQmx=&PGL&F`;#TMm4}O&
zr5ecX6SU*Q?akS5QS$z%cKF?EpTyeuAS2rigNc2dP0R;lXQHA*sil5A!|MWx2?oL*
zz{O#8{2kYb9as95pj&BnhM|u8`I$uXH-Q`ao*Q2sM;97xpu3!t7cwqSL0TQ6y55?V
z<W^Ufkj)45U2{{(9Jz&4uk0Sp=lN7oy#+AMb+pMWT8gX4;wK<6VPGr%JgdD)hun
z+Br-7F*H|Qj!DVHe1HiSD}LdnCMSc<r=c<q^Q+!Qr1lVaq{prVq2yx+fOQ}e%N4#q
z!R+7<*?iF<(b>_=YnuUQxsvpTW6nnRTO^OaxQ(6KWAW+n#S&)q<%s@KTv_MM+|n{q
z{}RaUH}(&*X<mY?7xfzc<A)nwgqfhDjysUgQ^sPjKvIWf#hWteGh5Q`L1^2G>U{?C
zt7NjbcU4c1?04PkJ(%v+?Y~WD#P!PcyZu5eyvr|-2yW{*I^4+0+zO{N-Oj!JuPps1
z9*LibZ8Bm+0Xp?qbdFad&4Rtgi08TsEcUtaMA-7n)u)Hy>pIuDhnoxS>rZE+YHK`}
zhGlb<?;vmI+7g>DQwfF#^}P>7+Ow~_etxc1zp*D1%U|da7Bl1KwRfhzKh^Hm79n7x
z<+-+am#t5yJ~FtjU=0z)7Ut>B42>VbPy8{N`8paGu5or^E>U8yGdGFrGTtNodz2im
zqOBl78w9Gbs@V!tY8uH6OE@Qa{*S8OjeKVjQ}~GP|4J7SKEC%{Ervs6CQ{aJ1lqU(
zl=a!=2*@#A`++1bJ|q5*(Y#$0@pO@u{EnobqBZ^)D;wtRAJFdS720P^EC<AL_^3vm
zf2-y{-XUPQB&wy|N9RG3^v<YskNwS)a)s>O+ozf>z)O4cy+!~`myS%&7XVR0iZzlj
zcT7kc9ualOD$2yJrnsi>ENZ>%jF}VFLM=Kuk*dMCKGn<jiY#S|ap)TeiT92hl<jcY
zB0OH}qO7ke6CAeiDv@&4&!2DHJ7Vl4?4uzFi?@xFm+EjS5^Kpw7&w3b^?Fd`9Smcw
z3jNaR$$o$7N%Sr^dzyHvK93>BoQOEtkRK=23q|A|GC7H3PMuiC{=Emk-h{}-?ijLn
z;2`;H&-b%Qzi|M#MU49F<7uy&+HX@;>QXygZ#{-%(+wIrIbsEykQdOV)Lxw;AHF1q
z**}Pv`@;^W6>LdHn!;6_AKcQ0DdciF4BMlQHWD!^sx;qC7vW~zu|TuFRyPS?Y8r}>
z4o0yj>O(fwUoq?	dH#_PQU;72PoI5j*3{88r#SUBJ;;&5-cEGo@Wh7*0gkD&~*9
zXnrw_qG>qT&|Bi+2mwl!-7|C1*a%W1TDd+ta^*>l8UTru`hs%_oE!j#XR1Z#$j(Dc
zCJt1_=ic&CfZAkN#~8C;I-i&^l+b`_CMsPalBKIQv@1+jYLccr#tkpYvr~#njubxe
z+1+$<UnsvfEd39tk*ICB)L(e~6aTuetEZ>6$IdPtipL>#rBghw`UVYLwo;wo%0Tae
zjTpu&FONYAVw5!1pvQh%gdr3flx%C3WtVExTc6~oq}Qpyq(t_2S^(qHkto7&5hT|G
zdtJlf5lp#bs`(YtZ3w+VNgv~TIo9XJkdUS8K+fs~GW>c`fpHuR-FrP?c%frnRNQ#s
zOV1^2pPbUAVFMRLhG8dem*sg%<BbT1Z50zEnpHx%M>v~FL6Bl_T~G?<0;t|Lqk9yO
zR4kIDsg1XXYi7uVwErU+9XkbXGcx<h3?s71-gjwl?UIY^PpU)pQa6>5`sh#yL=p=)
zC1q+)W+*_=WQKklnM(LE@_+Nvy%$N<;YA2BT$b2gV!+z9!<+njF+RlhT#Bs+t=hi(
zf<-*Skqxa%Ia?on5*%p^3^?nZ->Q@gzO+pWg@|f-4mU)boS^LS2oTMWwI5s?CJM_b
zZ|Rw0JYI<(XFZ(~YGpc$kU8~oE{D?kzhumChts@SbAX-H`rUX7><q5K++Xz>P7EqU
zz>?4`9DDs*j!Wv6m<Px%EqHpeMb$zwVCp?(51ZBeYpbmC>9G^odYTxxAWEq)6{?kS
z$?fKS#7n6wz5r`&t8lKE`1b=JcFi~`AyHCHXocEb&I!7hx|jL-X)^D<aFsc&*O|`*
zs5hqRV$p(P{_K)UX{ZJafO4eaIpU{rts$G$jG=$2;}{E`Ls@(^BMq&K8$D-j(a0L4
z)9$jafqqPAaGM@o^b-eTyy>4_MKAP-{~<Lj)E+{G*WtW#ywfDO$x^P_Q9|^MIbDxv
zu;yUMzUWv1nE&zXc){2*NSlv@&o2-bMbIoF@ANT`n>aR~NtS>VPX^1;<6YJ9jP%l}
z=x2T0OZV`?qC<7{tlK-ree7QXR(FE?_vGr`ix7w&Y7XvYXlF<@a$08{D~~MBeL)Z&
z%So$Z1rodW`$`oW<~Gnj19-udS~sM=auF$4k%Cl25h)lU80bxEx~loufRD=@X8(&O
zgESf0KP4F$Lw@E8IBE41m@2xV&vV^0s8V^ByvO@G=acMS(56wdNUo}Jg-Z>fDmvI^
z3bfWh6&Q^YT-s<sHo0iRxFgTxf{hTSetMKHyTJu|U!B3^!xS4rv&IN6aJD*PVIPc$
z&9hM;c%QAJ4DfT`0F;v|l!g{lbhg(uv*#6!i7F<F7@ou$f`W5d?4*TrBx|AZ+0?7d
zjh3uI?c|sMe6B>PT(!M~t~}G5QU~xxi@>{G-e0o%&DU3q?kG7Je@v{=OA494Uk)qr
zZZFaT6cj1&6p?eem<Ro=DOf%zKX^~I^k+B9lX(Te=lF-c$bDOn@gDZ?Zd)uTw3Jvz
zA&_R!_VC1rCtNfF8~qbeS5X68I3Cznlm)LTGu6c0j&6sk<`jX>E#4Q8BLN$a<=f9z
zjQ3M9I4J<9WdS!;pG+c;V}k>Lw!Kmy??9??$wKLrtjvOf$xdpzZ0+<fSa5kH-l==3
zsPXe}kkC^nZ8u4hsC0H(U=YdqFSCd|mpm?+$Rvh~U7Z@FJ~DXvq#l|Kj^w;O*$o)N
z3UX2@5mZ*+oZ=#yG@r;7ZS?37r!Ey-yx-z@Kax`UW9Q$j0#Pd;_h0a$15G_TrrwPz
zLPIL7<_<LE**-J~@*XG>CaqiJ2L|`=!vO~oRBcj(CrHEP;xnVJ!P9K@(4=u(0A9QX
z1^dz7Z#9iE>wFQB)d}xnd{e6&*q}(Rsrl~8XT0=)ZHb?48<_puk|%lY&uqTz;vmk{
z|CrX45lnsxvDH_fqAyiMI$yJ9*x@Wa&U>|{XaiQrqnaDagA3Th{}3d9i#?=|p#GJJ
zS8_Mj1%bPGx~nb?GDlBo%lh$h{O=cq@C<tyz?^~qQU}&J!uCtX!U;0I%lM1O4?U53
zSv>K3X`LuYAZGQ8>OLHvI<i^8b;;pK+;7ZbP6Yu{cCFcN;UgT9ktFL!MuFX<*}BpB
zQ<A)y`8#J(%*zr`#{6#Us0=FPz(LSIUpYSBA1!RoBbD5j)qAMMS9R&x0LjJ4{s~?b
zGi{Zsj6y6x1Tk14CL5@kai%|#0g3cA8<Fa`sUmBM-EeWx^gW^*aql~;H5z1+tQY=`
z>UcJ1*czfJY1NSg-D1CHSYu(R)i1~O>7QZ!;OA_JWnEw;1hcLx7Vz``+G)7G$bc0R
z=k1KGk-JMS$u%M?A)aaR8#jvfWyH70{5}n){2Bw%Kn#7tUtF>7@@i(AH1P-@W=JK!
zcMEx-u9s$ozdfZsI8k~}H$?}ada)pSfWeX%OTlpQ^ZK3m9zH~Exy?`GBAwT*#5*=s
znl~|jXN>`S0uz>=>b?;XkxKzb87|8e2!nLhifn-fV)O@gk@{ovJy2pa8=!KX&6RTB
zPa%99a2w9+XlyJYVa_?Gng$GZx?~&IH@b;mZiA4X5P@!VB=N@zN$W+n&BeG%=UnQ@
z>5Soz_0<+*;JThc|8A{7-GJfvVt^nn{$gZDlEws?L!P5jJNGaQInaay?)rqsDbtY*
zq^1w1V@@=sayvM{cFgv&dK)t-E*VKA&BVVTgvZfVYU$oQ?ysQfjmOpMei$I;OKBRx
zsnA099!huqm@}UWJl0UG=a)hKul(oA62eS=DPCXLUZUf3l%@g;D0FHf-Rv@H{r$jk
zO#K0(l24?S_#!xy;o05+Yu|{z(K18vNK0(JqmALGsEzx`EpYrGID`St9?q`_5}b)a
zz3NDk1rRrkd8V5NOi79ecPyqvx(^cgKVcL4vH)`!DES-&!XYB(`n0UC;p9pVnu_kj
zq-lI2=&YwyR$T>3=_*uua9n)J4&%oCzPgi7C_)WGq{=>T-!=&s`%agPK_8}Wog=Pt
z&CYU2;&wY!oy<@eO=3L1M={YdDok;VD@g!_z~H|>gMzwyzlm>o)~f%V$B4G20}q==
z=iQtJ6ieF^@1i;kug8=t?fG_aMv|uANWM{noUlH=cTo`uM8j*f!0Y88AO&tFC<%wF
zu~kkowvt$OQ6Os6$_5GEr4QYT{VG9CrER^+k}vVHKXa4inebzJeq(6p*NYfaV2`%3
za7r$WSrlqGQ3r4-?gbnM@2eb^o8XANQUz4$Dw}Z#u}rA(X9j2oC7?*<j_>CcXgB~c
zNY$nf`Rx!#^ix}++%_K&P^eB3{`o8);vefIF6DiA$)QY58g%~hH_)~YDIlXd;D+`<
zalC}crr_47z~uiDD~(kIvc9>KkPBH@-H#XRtv*swn)*e^K_ef0ziu=k$D0j*_;g?0
zw~R9m0Cj=zz=F{$Fv9g`o1^vzA0%v;i)jCH)C7fDpDf2~hp7PgCEHK_Jy$=zv&x1e
z8*BtmB7$j*_Ze#Mr!te$H#dPp%7Zp;r!ucn8pJ*>xmm6@U~F77NHVzqi?Y97!EEOV
z<eZ})%Dec(q^MnF={YNkMm8Xj;c}Vz*fG7q?F1C(u8n+GHhrdwnQAL~wExE1gVhww
zKzc@D!wLzGbf%3=WX=I?V{Ix_lk=g%CY&3Kbk0lXjIRH=urf{GccKx7FF8eF7ZB+}
z&3Fc}bKIq<7X>vFqpeSmtoIjVjXt9p|6OEV8wM(DPe?WAMfE=I3A@Afl@lXEC%`0z
z3n6xu{(dNWcQkB0ve_isf`af7L!Y(^^+LKAt1fGs_sBICkt3>6Fdkfqv_x957k!U#
zVfA}eBb@J{t5CIv_$BFsS{sSIJ<gC*xmC9*#}ask5xH-vzZyc&chK;og$WV?B$xBY
z`K$fg^D4ERZq6Lrq`l?2O>`Y%4T_ILb#V-T`=(nvU`P>!9mr)yJ{bqJA6T>lTiiyC
z-L0|;$dq<D{Z<im9=i7&x5rG}>wIo#SLO(kqz2|w=g;|+PpWj9DfrqiD+wkD9f7Z_
zD~0RUzYAc{Hh7_uW;k+B<9i!rwzt^k5J>XO2Mi{%Q+9UCmd##9i$CJJAKnMr%~ZOO
z=+UaA>G%5VB+sdORDk6d#nJw|+*Y6~*fOt+eOvjOQjfV&pN?4SRpCq|q%adM8doah
z74QTZ$E5_(+f#6v2o-=yThI3-CaPEn)N~TBm=zR?Ci`QnM+~~VnSG@Ano66?43}j}
zzkR<z4!4A=PyVD@ZP_R(2@RD$zc@qYaNt(LV~cM>*4)EwdCVc#>6Gz|Uc~I4luN4V
zt&b^}bL_oI6n6+Au1%6oIyBQ+U5I$P-(rMa9x{Tv4ybzy-(&^<%8?eATsX6Z9t4R$
zYE1Z!5SaJXR!JpQ=CS?sLWnzM)oMo(^^W`mato^8dqit=Z;H-%6MKPd&*^J}{@3@2
zPQX>b`)}lecb(S~H$lIu4uT$^zp=tJ?R=yE(C>?F^`W}K20Pu6xzn{S{~F(*htvwE
z-}1nZ8*B!yYer95gSce8A3z`J<})~+Rg9%M&-yo6iB#Z2Dc)c@ArVQ(ZkQe1R11Tt
ztFMmp`%n@|DN{c6slFXX?uz(iiBL7VTcNktf+(vjF?Ml5*$~<7!1}$Vz)WAt5xOa-
z^E*J2*t5SAWYU#x!e!&}*=k|enXfyYzUfW`St-j;_RSTi@#emz@tW;?w#6^;?);O_
zUBhKXo#n(qv!R~C#c5n8P7MpUw{N*x!N@;u6^TX8B`B?YrSp_$ucA}?Vvm!{JN(y`
z53aGnbPxN)(koMLZ<ceA?yCP>3|;~{NM?<sYCHbv_EXDkpSuS(W~0Yq+8pb+2$R1-
zz9v&HaVEc%qC3~a;Ilthk>=>u4^;zOUB~hUl9k67BfTzufB6}AXn{IMOf=wVJ$_E(
z#}m?ISV;EQ-(KcDSO&UP06LvK1QH|!=U*Y>$dveK8M|v8PJd(~OCq6TX?pjgQ9G53
zSL<+2MJ-m7%CiLkoQ2F4SKjJ0ityrM<65_7dPZpH8t2gQcj+mfFgq_TVRklLi8*oj
z)U`rkT`?DC1MJ2wecENZ#1t7m{C}smEUYGvMflL))yJ_&KSYcM8;a`ol<&|h2hqZu
zChwy#pSs(cYTXy~L_Yd6PKw=W@+Dy0y1?U}3~t|fOH^grWedII&6^^WTU+xBnXfe2
z;x>tzSW@=zz*f2+O?Aq0uWYWIDfWIOcAUvL$hMu)0#nK@IozBff#=cQExhsTHgnt-
z5BXT*H!_&XWo3-ScY*OPFUSaMveK8Nu04laM5qEne+G>_Z1%>v`l7>${Ei&L*ECtI
zo<KQO)XSa7k`TQmXScrTekyY*HY}(RPX8{vAQP}5bHKkMgF@zP`TORgL#7$#2cMxi
z(O-=UY!EQkhco>;|L?DpBXe_#{(5y&Yq0dwY1Oz;my5<I<9Hj0!f8Zlzmc~g|FHJ{
zK{g?%{NbNC_OmJ;hdYC^;Rw!gGC1?X3N$J55Pkm*SGj>y!Q1>kzYlyc0ML}betRCl
z_kRiVG;1LfKCrj%j(r-0giu=kJX62As*alG=Vjcrb$Zxj7lzG^Bty?qFlj#mYBWx+
zfast1wpDgaC$Dw!rSzYw8{etpR-Y(aOx^<*`ilN9DZWHpq*r-=QG9h2Wz&iWd~u@2
z&XhxD$T!v=PLfSA=dDf^9WOSvXjogj0*W$CBHUAYxxtgW#pFZw%f{_6H^~h3AlhT>
zbSWewrai={(kl{mhoZ;6`dLi@zsWcBXHIjZW}0n;fbwu?TA=7h%0ESDFHNp_TgqbF
zoCLsgpSGC}xaQ^Wn`PLev<CtQ3;S1>(-H1s2GjjQCvw>jvnToQ*Zi%(+-n@=!+a6e
znWQTAUTq}nXOriPcpQKeyU4%R(vM5SRo1#94?%5Sg*8PR1gb?UNjb!LY^ND|QY(8E
zXE}18&5_Q-NaX5gL`QwiTS=2c9b9KU!FLN+=@a`zO;L$gp--z3qnmns%D1@hiU%8C
zynCAX=`-B=vh0)-6=Tj2!`4SZJ?~aw90-6N<=gIeZa$nKj~UUMe8EcuiC=P92-vLu
zUN?c5L-SYLkqjm9N>BtUaCS+Fm6U?Vol<i&O>prp^)YT;GS`ai2VNR-x>GMXd`&z*
zI-lOf;6He3H;wo?mmiB-I4vy+-?U!&^i9`~mUd0Eq(2skS6^IWid)+9{}b-wm??jz
zMP--|sxb=@yL#M6%Bp%Ws(2_%+bmSYhb|4E<O552^XR7HfVxpF@Sg*R(Tlc$E%ODV
zL_cBDB|2&lc(25LLB*j%P)J+8%2Xhu6AR*Mkn#F?=9a`ECZ8VGR5WqZCFFX|$X9@F
z`b|bVXL{!r`3fGX-O{D2b>$}T1EKk5GrtCO{OOdPv~YOE?UVcAZD2*vBxwgzr{__v
zjK_nEAL~cd5zmtxdF?vj`O36pKL>HdcL8pNahe&ue0G`eN@wgvLgZ1>9M#%(q;ZBr
zJdh_CvgTXOfR7IdS2i_hxVoBCG~)`HRUNh0#c(LgjD&VMy1|e|ny}8NoF}qPy%z~A
zb8owquEkrrtF8&!xh}<hSOpT31{Bp!nnE{Ywe6H5Q3_i^7$sgbQz@9x*M$XYIN}46
zQYHZ=Aic_%NId)w{pk`kF0Cs8?kYTV!RA{|W@XzDGe?sgF|b33${z-OqznrEi^a_7
z1Waa1NSdM<aDtGajRyOO0>x@8NopGiq6+QaqhbwOEA#MA#<u5q=^~@PZtO%8JNxrj
z=z6R<eSXGIS3ZoR>&@i2SP~|ZzW_;b4AJx7M_36g=pbo2V=*BDzIyOw4Q8rN&mFM-
zQgR`p+1gpE7d<_D4_|>vGGokFDf#Ex!}nr6?o;FO8_6=ip`+Q`QHLXATl@ycwA~_)
zQF^2Hdu$(8x`s`+sD5o*f3x=c$f~yCX%_JOCHImd3aI%C?rY2_3_j8BwUYcW(A{Hk
zOFZ`!$hl3S7dvnF|K`e{#U<M8Q=h*t6#O<o`k{y~xj7e8ySX#O6`LW)R0l#1maysl
zK?Qav3^*+qN0Tx#1R<4v>IGmBnq&Xlo!baAZYSSmz4>A%%BK(v%HS!^_MZ*5Q<J_;
zPWqY?)5SDaX{jO>%I!ywaQ+5VOUX)Ey2Fj3y}7rufS*z!|ASa)Qp|-~CfCrF7gK*1
zEDM;0O@5ZQ^t3%gR?FyEl$<-DWsqE1=Z%LLBzklzh7-V7t;xQnD9bF?QvJ*R{qBoy
zV+&d72ho=p<hLLOW!;hJ@Dy%aLGM{`U-*F3JfHRslJt{sqA-bpyV!GTm~Y?8RH**E
z<k!G%x>i^Lc8ZN{VPqaT|H}b_kVFH*P5b{mCWSA*t)S;dDi`@2-tVgoKrLM1>&;y{
zC*j;pLx^Je5nLero*}GBjuYd)5mf9@eZGupHlPLaNtNW-TUT6!S1Y`ak?ZVy*;sQB
zS=E7x%fM-oxAl<Rxa_HOd(H~>9$RRv7Tkwlv8w#ru$1jBEOB!xR|Dup{MPE~pL{Np
zi#f@K35O`%=)`ew1v?_k5ettvqP|EZypvGE-{=GUc@X_y?75bg-Kq)o?#wBraBAH+
ze~pa7I<6;GItQTo5n?HXgE$GUp$sNtHuSZEQeW_2C;EIpIyIw7n&M7K_^TBks?2*$
zbgtfL^h^BVr5B*Xx2y87hyOcgjZp642Tvw)&FPM2qOlB=;I4;aB%5NFWWvFDXM362
z5q4V$CVf-7ueQ1<dAQQ*j+_Q`bfC^{1&jac<YhoCx-qtrWeg<ed*ygYQ>pqAmc{Bh
zIVcR13!fT5$+6!3vvUlA^Ylt#rYLo%Qv~uM@su9RL8Ec@HGZW2hNn#vZS=8@UrsyO
zN~6_+R>}sUI{Sul#PH1%s6+Y2T~z;S=JnF2g$^-j0(&Uy;OcZ#WyHMOO2qsH9*6x^
z<a!_{?k_euFV68MAHR6<_b<hQJa1udgKh39yp-Ejy^1N*RxOUS?`F>hv>RJcIofiO
z^eU~Uk^05zUby?-YV+kj=8PMTmC4!KXBax#q$}XwN!OPqtaLsencPEU$M#djz|9*C
zIb+!?2VBdU3Sd`XAjx$D0}oOQEtZb9QjBtxxpn?pfC@NNGw0$5@GlDf?nkA#<ic>l
z?>Y++8G3kO)-I7hM(;=)?tSJ%=n7Za?F)vvKtcU|`*1{esq(j|B(;dHt1E*7_zHrt
z*NKGcT;m)*kbv&7Z-K!M&^TAqSVVR$d2rMx5b2i|-(M4|!E=@wD2(Zj=?#QG?w=Jg
z(Z5<S=N4S?=NmL?Hz`+u=4lx!*z-ef+V&}v+-(wjgo5zzWnTB6jv=d0#hN=hd{-j<
zG%~i~iV2U}#Z|!kjmc$s&KOksY~;tKz>wdSy$>;aRIfNii<=!H>Q?kf`T#G7oc1;T
z{Ac)4;+E^9VHoOnT56MgcpEn0-gFw3MhBi*)_ZXm2A>)1_tc^sa3HS<XWBn{M7K5A
zIniCgMoE0)<b_jMnVl3w=_wfV4V-fzV$yF}0leKskeVEcP;rl!O&28nU!vfWJWxSB
zS%jhSzW3Zl{(H7modS;#tv*=PL4Icf{FHbGM)N-)jOzeJ1v}x|pg`=>`Q<UM95vai
z309(J#SA2Nx%`DR$T%OvYrdD?L{;c~5yNHzLxRTq+ICJG2aLNK9RU2?;^B~>>7ohs
znqr`K2Cqh#-tLeZO;dy)#ncuW<7yn+tk*=A;98LvCVGxuwbL`bmz6d$O-$w$2Iou9
z!AP%f!Nf?QhxhL+Vyv)-J~08GYsO+AhTmX==vEuEaXO^-(IzU>ODgA@_fW4-mc?_t
zE=m_&&Rd1fnPoX%kmpRlnouHX9x8u^<m~}SUX(E}fyX()6Q(%O^>y4U@>@fCdP>pY
z2E>)@xs{B4_y1}o0pJQM4f3YJ`hN_Iz)@RN&mJ%>`IggUo7G8>nH=?hnYMvE#xY42
zw``kSHvoEMofM}C4ecZN3Z&^PQ9EH~#|<h4koT2}L$bVL$r{T8ai(rN24CBa*b=|V
z#M<zqO{UEQV+Byj9JAZqeIxNv!kN0pjuuLW?klwNg9_}xT&*wqGZ|vr1N(;^xiL0W
zGdn@vZz@HLu@+dYH84%Gb9wW!srB~hGLp^Tp%PNY+SgjW;^4~!dH3cfF^y@jI==_>
zgev7EfmczuISy$O#f>r35Yb$m*0RX~!|gc+HDuWt79xTCY4}7@=0=8?_{_LJih%ii
zzy*JuW6Y7iebZXP`Y&#jMxU;Le*6Vc6w`bLXaqp_R?P?((;SPFG^w%bB(?n7TX!DE
ziroAhC%7FaDQ+B^XW`)X;>FXsf{di5m6SF60KiFa;uMYdX>}U_hstdPfSBjUo?wa{
z0Z=!8|D6b+!+KMylbr2vnjO4w)%Qj3RKJi@u8M2_QM-YJ1C?I_%>4l1>w&m0@oVCp
z#RRA3)7lZt=@qMC4o2qtJ<xDwop+qtTt7lBAeXeIIVPn0jmFWRb6<k1(7o7M&xoJv
zP0w?IjliPy2F29!pH~X9i}=X5O#kWS@VbGB(lDtJML$D8-v{S)@D_y+uNA?J^nX8w
zzA=fu&TPowIY9ZbyUuG8oh*Zrv#pB=uWfyqy-^ph7bmai5V%7hssBl=;k|vc{A&o>
z{~~~}Ibz3gte;Ew7-1lW)4>707BHx<5T>-td?G;Ym?BzX2dcw(?FEv4#-nz0cyj6z
z&HgA6xrAx#n6j>LL@;Rm(W`ql?z}>Rz;`u}cR<Bh{qsT9ZYFlVqkz9w+QCg!j^Tmu
zg=0mV4%#w&hO@{ieKA5X9(u3*f`gv!2?0dN0qihp)Byl;k3n{K0iXvG|9jDN=hLI?
z%akGsH@nvEKNJ%b$=H)Gq5I9d3tWH~Dw~7xB7HXi40CnF_zlJet3V9HK029bDV70S
zPD!*xvyMF$oqj}QK78DW^~$RBxZSIyA9DGC6Kb0ZGq3!XDh7;NP1y1*K*RM^fzdJq
z+!gXW^>YxuFtRO%bl0K0s(ioKHIn&%W5OZx%G~vh^LWp6tfpU?(+vo^NW^WBoTA<}
zh$~yFn(i=k8?nD|;$1;RMP~-$RCKj0hsho*G(!`Oz8-X^W*3FL|Ml?MZs{dz34PB@
z=EfY|YdZ>|d))nN<fj{W*a<XqdFk?zW7tVp;Pf%p%d(4}Fx{H`tHaw|j6_lGHd2P1
zx=4^kp4B!h;WY{lZBHLpb=7fiY_*)e_lK9Qx>9c_0o_xo<aaK1zE}+9*F|3=Fclg<
z^1!D>I&@P&x9L&N6PnWFpXWAE&l4vwD#D?pd;w|>()35`|DLJ}91uaqQ>*2q>IF?j
zt!8ms_&^X{(r0d!<qVqOJS-baZEozfAaAwHd@MliYLyMddYSQOBxoHYMP%pcm=|a&
zoTxIALEpQ6nvI8ir<&fQb+=a^Yg>&fF&JNz*@zZ_v&s4x#BS%%1SeEyhk~o*-9x)}
zX#TSWaRRAl%v-1veTL7R#C*o+@zohYRWOGyB_vHK`b$0OlF7iECcf|s<7#)*+2aug
zou7xuK8PGc7NM=`!5MV=dj!=rQngH^>Sy@98GUaT0w%iq<i0ndq<{ZqNR~V&TN!2w
zmT!5>jji7E$~o>!gyI!Ga9Flyo)Vb`9oVOpK5F<Gnn+Hdx35sDxDSP}Q~6(h@b`z~
zDK$d!a%;w5h}uEq6cwd-i*X3Yo@xL9pm#>Ft1Eb})2^Gbv%`=TU;unoD|~&`5Lq|4
z$Ut9m{(dvpF7``arl5^cP<G$O>TR##TNvE9Jtx)WrFJK_Qf#0GeHN5Za@M+ycicw_
zXaMk-pNqyoCih}o%FJB5o@)G2lb#ZS1T)d7!7Bf*6i4l7zT@-27_Rfod7V{?t>F&K
z>vN*Cp>+XA;_zUT-CS_QD+bzmd-dgFmoIOTCoF~<UZU0}e+9viJ~Ud-gwTLZ1kw2f
z@vQ4<zs`LJYJ8>_|H?ecUb%Ek6~r?MhGO~HT?Nc8!94W`7vruxdN66+<)IE|=qFCX
z8J5%62p9>&wZ!YWDnB}FBI%6;hP2S$GDSBzH(Rl45dU6v#(qJq63Y?7l?d1TdA0nN
z?~ms;qMPs^Pm{fVrjV^O%s~}iK#Y5vX<t$-6+$sBd4Gu?^+>-Ij;etcPzE?RGvT#*
z)Tl)^xC;69N^j&kBZ5@UGyEm;1_j)P7YpgO&IsIXXnZyI`mABM2&h7TQpWFP_(k~p
zw&z05{2yB~uA+OFf;cPw+p=IgfcApPsGF#&xpc4TqEaWL7e;c3z&tkbc3HWZElu*f
z&L@eE{?$&OafhGqQKtB7SVN)+lBJ}*3<$qh_2JYaneBbcnO?=^!*FlE@~p_cG<;LS
z@~2G^U}gmVL7)(jQj%cB!xmW|xhHptuOT*)y?XD*QM!8SzF6`6U0WPP`z$)Xa`wVv
zX&L|i*e;(juknL#P^KI(Rp$!lE?h2^u`<CM5T!zS3rgt;V@Z1-PQ=efeQpICB19St
zVatl|<C}Z~<evEBa|}RPj`)v3p+fdVS<We}mUrpfw&towXm-uygR@lpG}|k=dJgCN
z@tzpzJZMw&Y&B&22Rv~Ri+~Ta0HGpqu~ke{X*vjxkx0u44x$0ZbFVr04+H>ywJ90q
zu&0mL@4F!H&=<+pFch7Gvk5XmrVk~^X#%CkW6^!umtUZt1S<wXa*v!#rxr`9Ph*b`
zEcD<U3o<`EGCBMpPC@40z*?CkqWJFd-kehGApsFj_C+m^%(PspY`$mKGF0<liyJDV
zUatE7ru_3@%^F%8vO~yUO~LN>vF?%itK4DE*d3O{VqZhTSi^;KFe4}*zBs|0SrFv9
zuziNp{GYD4I7hXdH3J|gE4F)1ufvmNb%wq(aS?KtU|EZtXmeQqO2S<&cePr&`9UM=
z$!~>c`_DC+{@oKq{|S>6@sDazNaWwR0Y24n)SGzeFmCr-qU)YH#_+e{H90EnG??h0
zYr<}QVk5f<RPX{m`pB$|QROb@qLpBU)%)UQFz)DXTr3H;Si7jctP@<|1B{0d@(k+G
zMF<F&S|1>2qzU98kw_)>F0lT2g*T+fF~=ZJtjNdU1iQ`l?U!?BET1p_*&i4A=La!S
zL|b4j)kbjShhVHvGL(iKEaej2-PAZ}3#{*OW*BbJLhZ2tv;F7v2-uj`2R#$Davl)7
z<juDj*sY}{Ws3u0z>>TEBL&SUWxwv0I@q4@^;5O|S~p1=^HB9mz5O1cH=%>7O#C?g
z&-GI2NiVs-DRyt>W-0d1uZOIRP7R=XejpaE|BHozq2HuKFnYc`VoC=^2upsP)ws9%
zOLxcIg+ngNhEE_nyNQy3F)nsvHKCl+lykIB`;E$cZKOXlX*npq#5^uqaU+GM+<Eb*
zdYYV}@+o<X^p(J~p2_1`G9k~MG-<upTGy1zc;4=d_l@)o(xu-GZzBy#(!KZ%c>Wd}
z?0M$f(6$ZzQ0+Zx*kAzcfBF?sw?8Jk_2d$Zd^(M$jt@S??f=xTwbc0(;*^A0&V5$F
zl~4@}b~0yR5f5K4fvscLP)ns!0<AIZa*?t6wt(Mw(QLH#)3BZ}3@RPszskbt?1IhT
z-hjTz?h#c{dtXkO<sgnx?K-R5RNupu%)LkKC-cRbcl`5N&0hHhymr+bg3DI9@o?g=
zqgpe8Y}%Xrs$XYaU6hX?z4}p34tutbbtL~J_eCOI(ZEfyB%d7(j|-{&Cd#cak<$Ra
zqUunIoRQ}3c}RL(sg{XtzG;D3j^f7J4MmKN^Eq1DmD~ml9$D|p)M5?ehLFrnX`y6z
zYy&q)T9etf%mL!`=_UD34(wRBfyVCZMwp@y%KOYxm1LoxL}_&I+={z-yXR0tXWNjy
zPm)88&exU3nk||2eSQ76y{}}5gde$+TaP^({)&3A2)nzLwJtx7&$0z@$|_&@99&!(
z9|mqM*Uv1j$+;SO9~c~5*MjYy(HCIs1_#z=Ny+AQj|e+iF}hA!1*m}tf{MW38Nt%E
zg>BiYM<Lm+Bk9WB6#j0rMOQ)lf1fvez%6BoEDb`wXHfcMw6#WAM39qF@d2}@)pxa=
zCM)R))ejFD_h^7a<e&iceeR3xH_HUcuU$gTh{YQCfkAXU7P2$e!k#Ex`h&GX0ScZU
zHXjRyPL8*~b&WwM>W{b2yVTn6WA4Ab@DgJ+jIZ*I`O6koZt=3-w~*!b?RIJoyxC9M
z+0`4Z%(bvY1g*mCxf#s*@KyO+QU`LBH=E^O46+^*DBZTka<YsMS{hdlaYO?;?jowL
zQ`Vg))#Y?<`Q^07?q>V?cc;>_Rnok3z_+hq&1a~7lNr)af@rog;GcBIpZo(p@=!wI
zR*LhYAg6PtYAeZkHw6E4r(!2v-eMO+m4Wf=Qq+&HJHTBJY6qB%uh|(6KKuPhvm@jH
z9W4$J8^Za_l)9m~?)$JSEiwkyi5(R19@1fZk_J1<&uH`%Vdu{e(4oy05)1p%7L!_7
zv4LnvF_utBvijbQhgGpzQ9<7I9uW3ybgEzA7K*(z8{mXI3z?%nlzG8D(?2>~wnRT}
zS7{Rwqus#%yc1OOiw_7b=S0779u*3K%^uyp(k&7TE%nCBqK%RONp$vxnR>l`K>cEE
zK>SsK*=BdZ!&0r;ROyF+$GJBHW)W|@f>m7DhRvd*E#L*!f|l_FGlVfLHMZ~&-ydX*
zc0~V4Fo2F%nRr+`6n{RMRePOFsww=@|M5BKPd;Z#yqB<#-M{0yb}jjj*?|&AN-Tf<
z?1lwRyomQ0A!d_7Gg8#v7b`F^20FnH*DRTry|EPv_V8h~`q5lmWJBz<7bi1h+R@>P
zQSM<+J6{PPy~ktQ!p2=Acgu5Sd(!oB=lMT~MWz95!^hQj+pR>vQLw?i50IY`8%2ZI
z*b^O}FE}gFF%vEW3RO5(<!BgOVJ8bBw82ST;TwCie1=<2Yln~lf+%1|i5>EkvIZ^5
zGPg2Jyupgcb<Yo;G$4}pkb8$SlV(6BJu4*McRE%%X!v)~iN?`a3wDkTx#!$Q^&CON
z5y-w8om?B$9{9#`X7^$92p~NIraYm2%>L~GQrF~6dTq*;xRB)!te`>I*MF1c;J;m*
z)R$o$GNm-JR`UQ1DO>S?edW%E2~+sy+q@m#c6>hKzb_WjTe@MYl}~E@Fk!2d?7Z-Y
z)cdI20anav)|Hfmh`3j0w>NNGb700d3O;2TE0Q*;KmB;rZSA6@l3$A7aL7HU)QUkY
z-LJodyv}Zp8~{?YpN2SmjWQSysA2<S2?8qE?!-j4)rwz}HJ_?a+<Y^yc8l&QEJR-;
z?`CSp9iR?RziVtgQS@*f7iD|-P2q0b36;)JKFt+n4@0U@*%OY>{_jwO?W6$6Vw4=?
zc@xm(C##uuVi6(VP``{P7oz@enY2@wfVzw-)HLt^&oq*Sx}!oy132<*Y7I|-re%Q-
zt)Zha&iYUP3*4192g<$%emmcE`r-j=@@@3JMbiPSbWF-37&Hjv>eGz%t~Q9XnKx+S
zc$p9pUQx($^ABNn)$-+)ctAmf5rmghz@NZiqjyPSViP^v7Z@YM_*3CvdvZodY3Jn~
zbLqbTVt~~S5Y}AxC;a}3zPDsC@O{P1OLC2s(|CS1ZOgJS<Mf!uW9uNA?4p}fEY(1R
za&Ey3y=-Q&%}6?p;4{>Pw2{r&_DxylJaWO+sJTWj+ehvI{JQ%c?d221QNh@y3l0%f
z#lt-`eAZo~N-my<)Fl2MV7ZMIuyyvVwyR)EqRQ{!`ut5QmmWyx;)m8@@vK*~YP8uv
zqIh1y9gM}YnJ^thFJ1lD445h5wXJjBDtC<6>j==Y&q`C7STsjHoY<30bR77j*_7Os
zCB8O1ICl860JFwS`14`F0AYoxOIm6dwvK>qnO@^whRaIYuJz(<`<jNwZb?0c5pw&B
zMuoVDY=hRq5lV*ByK~>rjQQhT<pv8KAJp&HackJ6!JS$Ftb?_yQv6urI_SIqA57q1
zZ%cgv^oq7P%#bc)F9q7Ta+8R5aGT$TOaouFM=bCIh2WTg9sF<l0Q5Ir*{nv5hf3U<
zHD!Rn{w8QvvSIkm>D~t}MZJ5X(=-b%O}*Dlm&vYz{3vttLofxQ<AmmP&2XjX#RF_!
zAhZ7h9m9CQzr{yKRV-|o+1lw*P7`476#M^ye*&t$Ny3V=?g_XoUFFt6b&r2IXZ;8y
zP40PaS<-6BUZFSPvrt@h+)B1qgp>i=A<QSibKHn7DM86u5;IEO7l3Mo{CR{M?Z=ju
zpQi9*_RY23eK9D<Hm<WSjI*@jc>z);5{?tC!1Bv2jK-@KpCD8${@)tWoJq3Ps$YTe
z=^w(zUQcji+CU+t_PQyK%YHbs!R2^a{A28rbx88$BelSvb3lWEQg8EN3(3dv;y@!?
zlTCj;x9_F&^@w^!>M-7$XK*J13OX*RuK!)0W9O;ULS``<*?%@AeEqjiw!6EJ?EYod
z0X1L!yGirCQVg*pf}V4+jsHEs!EB=1i$TA?CQWY=O`c7leQuoh@t84Opej2+(4@op
z7wy==w8=jE)2n3H0ng(Dg^3TWMqi!GHo3C=j03%8tT{_K@wtk)J@3t`&Hvc=Ng(->
z6(l%Bp>|*qiM1q6J3Xl<R~y}oYi>`QK<W*PY%WsyMKn0qHfO6J*_p_>gX&WPPWBL7
zPTLg|mO8hc9$?ZX;BLQ_aU|p3WjW)XvRO{_ePc$q(>(ZD{j6(=*M9pxi9T1k!05#D
zy19BUGLxU66x>VHET*s6(v)A(Nf%cvGc?Ox*_1QA92)cwR@|n}7SAH4h1T$rc{U6e
z{bmC{-~b<m{V_rT`{s=-XG?CdVg-Y)O^iTqFZMdOx!V9~x`t?E2D3&cq(I5YV*Zs*
zQC`&KA?bn1bT%OGa9r|TSe82oh{m~$6<8O$l$v>#q5sw_9$Ni=3sY?74a4g;F0N}<
zNWFsi{sY4S@i}>f0+XE?jai-pQ?nX}>Y{`yOz)|!7Ffwbi_%tD9`As-<Ma_sNA58s
z@=kuB4?v00Bec55{%hon=4)#8fVn8c9F+3)PM4qBnX13z9~6``0N=HqGdTJC3U+?$
zU49Rbbbd1#lgj&qtWsHwq*B$Q7EvZ5Jl*(&&~Ug=i++dyg?I|`klV-EnJ9krrNzxw
zsx>X`I8Uh6-S_3QDyxA}WqWiQAOkJ@5=HB$Ps_xWg&JFLSeh>9lyZJd8$p+2gnPv1
z=Cm6+*L2ZZKkNBRhWZvNafp$o^0n#AvkuIeH%%AG24n7}syL%^gvfUCa|0p=cQQ3@
z>(d3bvQ6e&kgzf*o5a8e3sDc9vZGVW>8{p!F^eFPZ<36#|3leZKt=VoZQm-=B_PtE
zwA7FTNGU2E5+c&wsdSfgNJztgAR*nI(%lT*-8Iwz?;iZ$&vSqG^FHhQzO`6OmTQ=q
zz4x`ReO>2y9KWM8Pa%V83=n3O*BnXBQ}IkqyJhSFUh1;F?opIdo=Q4c9AxAwW4+}i
zl*rmqn#*Kr`^)|$#?{niH)W3N63Ni*g!zR6<du1LaekC5eaWq5bRJ08*E8S7)$*Hf
z_OpIZN^qfP5;hj6Z_o;0x*gpm+eG>qpJ3H1#O~lxs_vSl)>WAugKcD9kNZie&Ko;;
z`sOq2Q}0Fc=xwEW+8TCP!Py!PZ__OsOY%ASjPS*uh=lX>1jt?@xM(nygJ?iAcWc4d
zg(wAiPoQ$qc&-KfxzP^WVl;JXDElbc|ESopJjL}@`4W;P{IxpyYky+j5opDD>c=?Y
zLU*}c`@JD@waUuzaTH<Ha563dE+?!5*L>@wO4;(-l>Tr+QlR|v*QW5&7U0G;<Ag*$
zC~(daM^)E{Hs|J-jWrC)9`>wX0_z$82S7oQ$nmEK>LXmT!e!K|S-hEGD`O4CBAyMh
zr{(Al(?-)lm3wzLk82Qh2324tnJ3h8{z{qeW=hK9ZbePj%~4-YH1;YFWykGoA1t*^
zSNjJ=c64WaN6cyMtfqLY;2Zn-(QvrS4_Q}y6x{f(SEkSxl-#=xs@K!xswFNM>MBi&
zC@bXwF{Iflh7_B^bK>S}2TK!rHe<r{J9#M@$YLT;BCwTKbL=;1O0RjxO8t~KyO|Nh
z$7rJwp0DL6^qn)PhEA;qjC)%dIz_8e3ldPo=h6luUPeDFGZ11v_ZBWb3UjvjTwVRt
z<=nEj_l8nzmN#W9%s@yOEsS!83~Xjuc~nu8=z(RoV~aFuakr?Hg^5AGlP#l#MaTpX
zZpg9t(=o;FI$#r@UvZGgGn(!Qu=_MFMt4d}4QG2&9y-eR(e7|;iI{v4@9)f*Mw)kW
zQZ&!QE*kLBo%UsFd_qKJbq#a>E>um3)q~#7zFJ<~`dm`>8DnfQVPNq?pXwgyCYcml
zW;-KyEP2`o8I_(YA3tK#iq=<{I#i}`KN6Xk6(8=}Q#YZ{RQv;&)qwqXDC6KQ`<tBw
zi^~%S=~Bw6k`>Ud2A~AwWc2}6v?Gg~1gwRC8b{pR`z9QQL~es|i5`oOI@Y+;%ET9U
zalEJM5Hg&`XQU8_pgKZzudH!?GCC7|h6YPG)QCvcpVIkuNB$}~2)WAFC&)$PyI9=h
ziQI&E-@Xu_h@Mo)ZwkCoJpsKdS*X42KTqEs&Q4Pb&~5dgfbE5CJ&)X*aeR|Kv+sL^
z%Vq<QdV0Ma9|NK=BpuRS-u-%;d3pI{eL$^H^X(4Ew!!Hn>|B+a?0Sz#!--yG+hF`F
zS(@gKt8Vu)5jh~m5qFgz-v8xbx&N&{Q68k~ei@V61;cd^?X2|GosF-FeGK*)G>f05
z;;Ij&Dk;?#u$GZG^VsJMUn)J2zM0+~p2k17-9w$om?=0fc_(oDdqyw^V`U0HUB~Wz
zOfYnm^`_NW2#((ph}1HCY7)>y>{yVevL3H{#T7vYbuh8M{lzPMq9=26;N&&6Li7mO
zkJ}<j4~#a&T;3@d8c&ktIu7VwLx;D<_FTZ!t>$A7>F;dc3{9>8t1A#O=biGC=siBE
zl_TShlGRr}DQrzRL>++?FEZ!bjo(he@%oyCaFP>^dqP_*?2sICLWgNQbc`#MCRGwl
z?Pg#1x7G}nTFQKvhj4|4X>sfvVWK5h8ePpU^%2F2=yh>>Oi;OXsW$Esf@|Hb@^yz6
zN4{Q?ke_KHPtV#J(xXRYICO~cLBa-%*ZONCWIgr0T*!w<>mawxm@?aBxX&F_YvfP9
z7~_0H!Or)vQO0k4jMWlA0cgxdS7jjlZNH0t|CqTo+nn1ls*Ipnx8-hNx;}dN(e*+v
z<BllINNE9TI=7CoxzHpuqg8H0n6&S&>uS{*c?Kv!n=5T8$Hl2wb!mEcCu;!jx$a2Y
zLmA(b75RiKS`mHf<xDv@{+}0u0$mCUU)mWT6FATETby)Be%+O@J7RW@!p~ro-^=@8
zQI5Bs+ZY+SenAti*_~476#Y`y$z6J*Y0b76kQxsCx+ac|>Ga$|h`<Dur!iAKjULt4
zn{mpJIx?JY53B8zDI00ipYGngrZ<I{+mzqToy+M3U5Uj(aHA>raf=F*9+H(Cj+aHJ
ztysD=+3hiV8fCz4RIkQJH%G0>rIlBdxuK4U`Df-TLc9q#iigAEsG?O?1yDO4z6$B4
z#+Ai9_v;#av~uaDph5MB*`ZpFh2XdfbMV`U8F54WBi5oiEdRL$iglAgR6eiAs4>Nn
zXW|sy`1|ot?c>!D8RkdsIGeoQQ#_47DamY`j{2P~KJ;zJQGJusEY$aytS={d#x^7#
zo75uAkK@xqyFbE^-?V?KH{WLr=-eh;WiL5DjmCt>cc+356ObT0^ztBGB>&j_<Odm4
z+nwPgNIy|dodd(-iM%uRuFpLJ>59m-?H{^^w{D5rI_utG?U8J`G&F)5A7XCsqpE7A
z3B{#jv$;DhkPk;->q?J4!@f}9P{pC&3EL1Q-wPitTJ{2wKk!Ftfp)pnSE}{RS=9|5
z?U;jx80g18YbKcthEA7-a-{8k!!Cn<Wy{;aRMQkfiC*e*fj@c(Njmzp#TSUZ()!S{
z>&sqQ;AkHp3TZ=FPJTXUe#aZGQw8GVTs(HEGJM4ct@(rV{ZlKct_Z>I>dnb}a$8RK
z;MC2w%vTP2Lq$Z;7wzHra@xL5VMfgZKkyi?SSdWSt&^Q_cuGYixjk+-ksU$<a3=^3
z0J@f-d$DyxG2F_r+&gxXw+H^^4a)BGaE~Y1!?x#h>sfhFqz69T?iNiPxTi-Jm}C0&
zUEl~mq1K*p_@ape@1Zw?(mvM?bTG8jgu=M!i@LcBxqAe#m!^7?^{A4h<220yTXREX
z4RDd8hagWKuRBzNeKQ(DBhTUkskp9imfX3F#wesf5+R|1CoIShD<q$@^v7m`mBZ+W
zN{o9xu*fiqzhPr~{+j=>)%TF0kG&GFzZ$W(M`adAb%9gM5k$frZY~ZU@&P3g-QyLD
z*BG}+!fHDKIQ7>mIj&l)6Od+8Lo<jwQ@NErzaZxpWo%z3XzPH2`<rVQtNQ_IcKa29
zVCGrNKG_!mgcwR7yx;f8VeBgF=D?ye{71jjD=wx597<jk(&Eb%<GIe=A*vm5Qykkd
z*SbpCrIOBl<8C){U2RS$rZV(i6-uJmuqK00GV;&buZ+Og6_0S(5&+!pDU}T~x0Oc3
znaB|_4&&C`xlufq^*eg!@FvFg$P?<TQ}W)d0FrYXPhtm8+?=likPDI<=Un}`*dd6#
zw=%3t5u7+L7qY{qzw>d>ayg@*!P<o!J;K-a`?E>SfeD90qUx_k!u^#Txs4`UNV}hr
zvcI*&+b|8FU=JF0uCtK-*t*mAp&~f;INb*)w)1&x3+>)$ZjL^1cif)9K*8GXf*tYW
zF5nuWK*Vic5o_3di1oI`B9gJ{^`hB1a()77IJ_*jXh9?XFcO5_CAz@(01MUUjKl_0
zdNBi~h*kJtxyu<3s1D>#+D|vI*O&8#^m~n0T%Y9&K1Piu$vEz>??(N8HlfFk5}~WX
zmxjH<I-Z){?*a6VmJuy;g$FvNGZ!Rj&Kwa&op>nFEe^-h*=C$5?N$Quw3R@d-9s#J
zzwZI+71t{<VuEm~V&4-?YUoR5ii5jfJ`Vw~9%%GU8%`la9zOMQ0yDZRx)&aOW>h>H
zs*Tb$QqpUmfKy4dE@xM#o`gR0Xrox%hWiH*g+NswEtCtI%Dg?+Eu3F&4q~fXC%GC-
z=b~94G@ahX2kU7+9_S3VxEuOBpaj=yb>4d4?SeF>GeSxA3Z44N{NU=h_aoaRoRp?t
z)7dAnX&B`>lZO;GYJm`sXY%2ga%y!m(DNU{K6Pf}1khJjQk_B__=gwP{Pa5<3J1va
zr5p*KmM191yup#{51Un8P2JmTH(YU_noO@!Blvkf)~i9HT=8x!OHJ!-jlFF_H}2Q$
zQ1H!x8ES#Lc1ur$Ptk=kH$#d>Lct13c5VwTE6#MYpP-!SCNlj&Sf-FqCG{b|m4>Yp
zNj8``*x$%N+(-R)L0DKe9<DM5D?0RWx;!doh*sys#g1ChhL^M8XYHN7ZT8K^!oDh^
z`1(%mYtqK`{vpeT=2a(1-E^7up389l<*$T|wy>Rs1!ux%$yxrI>U)>y5ZN28$03qm
z0P|{E0h-F>!>S9M?N(a1%EF2B;B+Cj3z@(smSW$dXWQh6W(_9LmVEcnD+LGc44LuC
zz&w0k%GdguzC?2}n<ik9=povz-Ea|&DXRF3#V%(9A}U=yFI;MXu#;k`J@Y$Kd1U-S
zg=`Vt-D4$XP_jI}X_nAfH~kLYSY_RsStCP7QHI5bBfeWv_OMs>rQNQeP@+KHjY-TV
zRD9pQkS^{%ny`BI<x}R#2>NNH-&jAGKk4mQpQ~?Vw_h9WF(*wqV8-P}!(SIzT+Y)9
zhKOcHx6P^OBK2grcbpnKgG6<`oQB#O8EM#~PIG&!`96@Nq=)hxTT*eLV=}}o20CVL
zAFRl)TDje5g0+RHr6!WL#>*;xMMO*g{v?84Xt^&$_^Sm_DGyFe;K8oKDz3gwo`PP6
zzfz3P=N&CDX9lOez<{0Xuy75DqQ?#5$bHk7sZ!;vE>ph4;o*Bk)v{$3D|i`ouz`<S
zQhrLjWwn0XaR!~{Xw^CiGhA*aH5m5!n6JH!5}w~Vd?FDrQ>1S$5P!%kws=kjRj3`H
zu(AfKcln-4tXgAy6{Oc6ivz_zX*Ey4fT%TZr6uCQD857K%G7ml?Vq`}?06BQrNJh4
z<KHYS+w{E#9jX^k`V)yxH<W-%q8p-{$oSx}0Df^IT!k$DAcuA$0*^ND$T(C&R`|C%
zvOcY^`+!I?&5*<u%B8bzL0KE95r5qL`r4)S7LM75ZZEK2bNxP0Y%-~3;AikWZs(>8
zmCZCcq5B7e-W=5<	|L+zAD}3nlB7##~KxII?ewTE%TwRp9b-2uX!otR*!drX%V`
zd6(d9`_^urkd8w)AQAM;0N+aCXx|l5%p=!8lLR<?5|ay4iq>3fe(m+kB3QigBLq9w
zxNzTs{`$k^Bn!>EsA;JtoHp~(%<gtKdBPo3gg*7K;9{y=lB!?p+=naOO6bUo0Eyhf
z=VX}X0As#r45m`k0{)n{<qp5OPMJhDbu$!#y~h9`uE6^IQh_-_Bx}$+!ri6F<0cFH
zD$oJ`qd<y^-1Cja1^gsw9+YOe!Nr=B@Ee|B)MWWxM%hB`?wR_^bG9D}0K6rz>6v)t
zQQW4>aAC&zSfLSoJJ@&WNwR-u>JT7L=psa@6k0{s2pYtfR&r2i-k8C;6_9DVt!Zzt
zZJ0=rrY>>&l?$mWb2ZwpAEF9OU5tun_5^K@W)drn1UBO|V01e5@fHPtsqq2v{(RuM
zOb&aoAh=Aat(Dh<&P|8&W)*jQ)l|n_86XGOr@gB!PP=&e{BtGSir%=>$^-!i!3}b*
zP5Hbr^k~@Ff`Va21fE;~vUZ|W&-;+!0!PA|&FLUGq7Gj=IN`TxP7s`JP(R?+{!$|R
zJ`KZuN9<%ERiBBxe?p-0?pwi%!Qh7LRIMC^d!r|zTv0A=$3?SoqQv9u;mD<U{R{iy
zRKi(c7)&NaUY!_ifqw9OU2Jx21w2EIpr-X5>ooZYj-l*2xSouwA5vO9dln|Xt{_r8
zr2pq*?Z+YU*>^GRPT}xH_dCa@T{xM?dFI#Q{Zm{og{hWZmpZVRPwG*Da`^JSA3#E1
z4(=~mIc?eR;d!wnzISJa8*>eFar|J&sNE>I&}yoJb5yXI8<P#u-^kMt2@lqJHy^ov
zz5aZj*UYU#9umP5*_N<+#V@uYI%%uI^0=guS9>Fj9Qlbf2fu~j!q3OHi(BOW+v=){
z%V<&q%Y&wG*J8;(VQYg!0Ie2gC5+xpM~tMy#E#HL*@LAC<%x3Mm#&tF@u58Ur62Qt
zPYV1#KTB>#1w`qFDHEJ#HfkivBPh9~I#(%b@Ea*qnCIYn?m2bWYh2Lu+(f8QZ+vzm
zhbs4-GZ>04-m(Hev(ia%JORCgnGJHQFWi1G<-kAdh<OFz@y;)Sx%<|GN8hdvqher$
z02Z&qi=tM8XK)xt0k^+eO?b;WT2rWY$yDB8<z@4V61eFZ?}Zm5i-E{AHI1EMDJ_&g
zS<_S=of9bIas&HG+A+)_9BO}vAgD~qE*%%`UNImt*UOOVpJMg~Zpy298Xtdu?00}U
z?meZ$NZy?<5Fy8V`Jz)YCo`(dC@?e4O}m`2hv=e9o$%|2DrmRk<?%a8Q<vXX?R5Ln
zY-8Ch{C1lR(38m`sKfJR3xeDCJaU(D=7;T7m$bjV8Tlx1eY!X1T$gu5mfQ`znmUx*
z-NG8yW5j>f6WPSvAj35kK*CKy1$Zo;mt0r9CP^Bakb*gOkA_eLfm6n#{4u@!C)J))
z!;4LUY9fY<d;+R3R9at!m?@|hrztb;dXSFliZXsP)8BT=HxPBmREIP?Cu*Vvklk13
zF3sojfbcvJ1EIFjU1EGwTG?UBP*!mhcExGop$SnRj_;lK>K`A7uq<mDlk~B)z8+Kx
z3fwk?bHge=o$=Ft1juxCZp><j51&)5?zCI{UVLj4M#@mKXE}4~Z21uiE3CzSLmSLT
zpdl$*kxJhXV>;v@V|FdwW*C_>K_ht<&J{~L&ebv|)buO{R5WOPJ9cbuK|wSwKYZ7}
zP-QKmp0^C1lNo7g43DgyFZtt{NW|-~r${xlC^m;V(1o?>WVS_G8IF`+#Bt>=@8wO%
zS4!F9=f-wjMp3&tI=6`Cs-G#)DDf<Mj8Cm@?Q9-YZHN?b5yK;H2BAx4qu->ajqsc-
zPVLw{e~y$Hnrl83(Q`f3iimPp%-vwPj~2P<FKWQ_-A`<;L@fCH5RwhmC-z?3nmJ7|
zgr1~1oN-j)$-G+AN^#fhqvg&>m0Caq$u9+1Jj+6}w$5rmj#k_uB-ct6vS$LUM_IJk
z!?Ip{k&dP(Dme@Ov7?*a%6QGHF6<RXb{3b2=NGTnKijT+FvguNpB9w!R*ICfl3nR5
z4Iwu|R#slEcJQab&5Fh%=m<HiLT(Z@%mRb`19iXl$fu%0PlXb|NEFb7F+-F1wNY)H
z$!4ztdkO}0Ds=&7A=H6fdBc~ZxF{=FcjfHMWc5&XJ%51ZiU#b$t=OdH#G+}g;1g!H
z{_hTLtLv8#9L66Ka7i$ViKX+OYvdX&i*<0vVu^;&WRcSOjxN01bxN3ikGF|((2Eri
z1R)`YPx`xSIacT?2cMn+f-pKpwOfuu#SOqIHN5x~TjY8t!niy-vEp1ya2x9#P7g`k
zK;-?2yk2U?6FM{QwPCSx({Zwz%T7U7=>*Qfw0>{hobpu)IP0uC!c$hrg*KL5&r{vm
zjdnE}iF`g_<2f_qzw^d}apA8RvrJMgsP;<R2rwkNO$az!&Zlc~-9LvNw){F5o2%}0
zX5K)r$35aXjQN3TR|;wS;;G1ucLB#`BT!(oP>;<{oyew_B!nySHb_w=167A-&|G#s
z%IkO15SN65Z`t~t=3E&!oiVkiHBXzjXnT87zoe*T8nL=lUWJj(D=;Cs7cBdlgx2hi
zqu*Yd4R3Q+b|}F3YAGRxho8{<-Ln30@ImBRzg03J{VW~;4%NGcEA=R4Qt0KFpb{j&
z13=`)?9`a8p;TJ-8A5QkHSHYMr30RiRs-AToO|h0Ofwz|9ZhYCCAp?okW&*5wL{0@
z-WBkOgrl$MSec{Vgp+s@?LhTVb1d`frWidy&jvKnDQ04eA>v?@P57hdXuke)13jHr
zr(64w^3HJmt-yq0gCop*t#=;H*G6NTT37C6CzvJ+9%%VmH>4sv2E!|P>0jgUrrj@v
z#)`B9sOSQz(^QXtXSqmyoKlL3ML7vnqK21`u`33bjvV=p#@p>deJrW?4snUy?y#T~
zI>N3p6AS!0)GRK`beqE)Z^TC$BPSN}++n#E*Yj_b614oo)I9A0XYPeoA-d=268vbY
z1^ugL^sLq5;^UQX6*!OlzD?GsSE*Q#T`TxXXVnZEgAT*GuRNyDG*6*So5Pkd8NvlK
z8ZBAB+pfYrO}y=L3?qcwms>%E(-lpLRQ(iZY<$dIrJg;1LZG#4_Hoie4BT~D!exd?
zabg3j;_ZDYmsqYxyRDalr9>0-Dd4?CBNUS`GNAw}LN1_axSN2x5oNvb=1x5t6ndTt
z;0@aDEtelq#DC%f)p9z?$@)W-a!Ves20zs7`l;)+NsBiATNxW5Y6vEFvv;s(kTQ;^
z?|LmK>kFz#;RGUU7M?_}>-!ISJ^M8O5YV~!Zz)0ix>txwEd{~uJ<)>Qo($zyu7qX0
z>+i8JVi&Fp9bz+YE=o>R#i35|&3mP$(rITEjxE18z^{KWHF<O3!t^%+;U5aj4CA>?
zA1LR#3A(bgvZmw-JwFQo$QM4PB)vMp4hM@4y(%~9`4%KGBFAXNLfbfi!I{P=!U4E>
zp5wicO>P{O;;4{p+$MFbk*xJUC6T6+Ev){|+7*r~;>#w~+5D41Tko1P-(w4VGxtW_
zl6J|udY=<^P?6$|WMz0G5<7cvSIll2(S232vd)!M-ipEyTJKBOXX|&vS~b5Rpow)0
zh>ZtJET&0iPQ@ihv*L=8ry6gRHfK#Av#6B^@AVb~@Z8Pxvmy>tptyF0(OJ=Q;|)a3
zSdtrxc+5UpihKIixX6nR?!inl@~vMNf(zVccy6gCj!CmbsHz)4kHC25VxZ_H!87!o
zP|2%rC#i-rmeXKw(rmQ_yx4cZ+};cV-pZ<vTvrI7PoqdbD=?bjJ7nJXRwTw1o~*rj
z^Z<gp;4JOAp!RXEW*8^6b4&AxBwlQh;4$7a#I%MN2p}W<Yvf}}6;Sw=9||b)MKzv{
zyQk0DBc<I@;1=T@Lv3>b>5daT!FJc!kNU9Jt0ttg7>^kG;_br4`{q`D=FcS6VVy%8
zvz?uQb^L<vjdFQ?BVONUZmE!r+W9TD%4b)0R7hcL`;X5Co4#4<Z>UmDd9uksT&a|W
zBk>zAvsC((bYRK4Lu#&ve^AxPbt?eb#`N1&Gu7HtLulomzC*sl-lz_2SJ%R>;iy<*
zm8u4=foE`p@jDprBmJ4B58<JjqKv^(c0EO3+vguK7nIN@cY0IL%^`Vq{Q->4VW12;
zYwdbK&}m{VQwefkC#*h$;jufw^)2j#8=D;#z0}jTe$-M4e3Z*E09d%YCr<!9jqjiH
z*s97u-<L+Gd`dzohf^Su419dLeK`Wq&7+3vM(>B`GCq`!q2Edq<jc-4@;#P#;swq>
zJJf%TyZ+4CV87GBZl(;{f4In~t|!RoEA`4K3dW~c3#g}n(GbC5+hn;TJ^-lcs$$uk
zNvuqHiKROGbZ<ChjBuH@9?K?edF+k>QmE7YuUs=)M}4nM7Ahadg0!<Q2X2285#;e9
z%Vp2_w<J&XPL+Upwb0MDw?=zceNp04lblVhR+!nmf(h5O!-GL#mJ8|pGacH84raW5
z4eFQ>aV++r`t@JD>FztOQ{~BMaywId;b02^?8P!Isy(i77VTPpAzr<x86ZU3vSOnv
zoEO^%k@m#l299x##lj3<?=&93;r&qzA`Go&f4I;ziL^`d1T{$0riANRD2XS(*ABi5
z2kV<BFJ4X-!RiOgD-Mzb{#PGW8DldZssvD{6%AwS^gA~b20d^5AdbBILD$%PaK9m+
z#y4XNPbO!D-1DP+VQbX2qCsJ5qjw|6o2Hu^`6*!^+%IP!frv1F-qYT=0&k!_FA7AQ
zy`Gv~n#Q|%oHtoUHsz3AcqQ+2GgMHmWLW{<syN(Y&~6@(FL!BZ3Zdu#;gM>%P?lHU
za^~3k3kNK>c)|%{g^#Fb_MKd%fl8>-j)lvaB_CkiP3!OfWEgDo69T;&2q@m$i&*kw
z`jNTbU~uIqJaX1wdX*wjGV8c#!fNE^WvdgkV%$F{&eH8JYREFV_K<0Y8_SJx=4znx
zLygq?UL>OM)afcmk+Iybq+<X(-9s5QyI`vPhyr|!?s?qz#LdD!selD<(89uYrsW!i
z;B@!BIh<RVSnG=<1+3Y;Sa0nWv4{trE2G<Kr1#aEyASyi@R}o>k2=hDmIP>L^vZWK
zYO9ab)6J8-4lw$k{HR8F%0wsWy+w(!vf{YEMDkb+#}!N#C3j5#z(#epD6AJq_tvd{
zd5=gquORLB*pJ^O62zf!7oN~7F)IGe_U|i6twRgkqmF{E(v%F38<=mniT;OkCK_<g
zB+|-Ngfui8{#l2gz;_LDmswR$AL%cQvN3$rd&%&u>F1eqR;*Av{-J|S;kACb=b+<(
ztz3fAIXrPF&(%D~VFVo>NMc?iAbf=xN^wq1CTrz&<0nNJQ9+)Uvxm#P)z(<tR9=!=
z4&lBn>>GQ9@{cklrJqRTFjf6hVei6seWmIs!^zD9cF{1kf(CnYb!;$~SG0JYDpOKt
z&6y<Z7;Zd6HjywtXkc!g4B$hMPTaJwdZLz)1Vl4>2Ltv5ZXXG`#FVa(X{EGJkmMI?
z9-_sTIqU)A%r*2_uPR?{7V?a<GY9iQREKRX8+zbYC2);o{lkNE^Bl$#fSa_h=1;Qw
z2?xj`Vc3~l>F}_E2y_6}TI)W2b4(J;3AAgdyUUj(-i<PQl;{JNHJu=sjF-}KpR<KB
zY^|zLO`sNSqXg98=}+e!*0ee2xhMcb4tgjYZMQS-j}0DqY5Wq!lW?|f#-y?VXiQu5
z9|n?5UBPf*=0*miGa(uk7Z$O@^%?Rv(RgOe<eK5=JAfvyKQzk`=?068`v+a~4ZFv$
zy|{d1jyd6AOY7+?KJ}2p(iZJt{ZfkTul7hH+AG=fy{Yzli^?SzJ70Z?z<t3tL=P1d
z<$SHzANRb+Zx5A8jlqY#+XWPU-|4@e(Sm5=ew=y|gY!KxrvV>}`_;;zBqIOlzoFRx
z2<Wf(?;Zps6k=2I!zS*9Jo;+O+pLz3F!d?4^7#g<F<de7D#x7r@%uW57qBld{QKBd
zXw}k|{Bv3xIW(rUXe+uo6e_uE)afeJ4QNUb_|@kbaqRc5p78ew{3^ncbdO)f)OaS{
zN=e9f2M^xhZO)opFwWR%Xr^_&2{HS#7MI$K54(-;nE>Mgxjhje5A;g2+oSj8MmDq_
zC+6FKDkiyg40UGRuXiL*=Y?iopDB3^R$@OBI-*82u37!nhHy|n**s(ISP@XQKwvN;
zmXC-^J7<g{X#P!QQZ0Sm#AAgtN}sfC)WhxGcDfpHUB>eV$fu0CVJ?D`z;-W#Eot};
zTmk-QDgQy3sZpiH6b{mUYFAXKxt2=|MepMNw4Zn}z0LPF#J*=G%aWyLAGvM-d*B<8
z*bq?5l55pE?g)Dx{@c@Si3xh!l3hHmg3_o_DHis7sN|$#x0i$h;DA5xjR|tmb2#P}
z{_saeEQzP}4KE3qzD<C4w`1&=eA)E@7nEAl^Uo-s3R3b-4jPMe=R?<uUC-kZ-n>QO
zGs*2R{g{59vUi#*CTp7%((+K8FNKlwmj$~zm*BFP&EdxmBY&fnpY|ri8aV}rLi&6K
z@(m#aJ}~&)Gp)rqd?1p+4_H6yq1$osJ;6lvwBMfSag_S#*Ur7qQ7L4&{-pBRlCiJt
z5`SnnT-quWy~$4psZ#@bdn*TH{#tIGE2NFZxYah8iP)3@JV!sFNHoB()8!@N+!js{
zu0kwwmfGj*51JnDfcVc3)kAKhvW^^&iH2%tQScr8*_K4n-TgyVM!J;NzlgrVk!37c
zL(EL0rU-N7;(%!2<1ZTnf2Ppj)An+{wzy|s=Xog$ZT-}el2A&ojZSk?r-+7t@%etK
zVddRlx!;G@`JkZ+$+RaZz6b}R!4f`<-rE2TVB3y#eO%XPUW*N(ecRRiZzI+VlkSwJ
zDc!GO=fAv$n4z-SDN;o0-gSCXP_7FUxyaHTkajT=P{|`l;53wTHG)#|++$5M>wwQ%
z{5f~FrTzI!)zwReElC$HCTGR~WyI`{Ukx}nnXGh1SNl?`qHbCdxu};4;X@T_#k;Q#
z#y0lm8&|ks<{0`e#MdA;*6OQD%D!DwGCnh^=;inv$4-Nt?HQ!6FF31v1TB7PB)$U+
zdDwaxi0vkIn-rkC%4@u*j8(+J1NfnJz8Nzm*5N@{nO=j5eADxm?;6OQtoJuY*E6y2
zFU<Gu=#l%D!XvjJRv#J?8yB+O-N5EyOh8i8Yqxu9;NBW3ckzUCw%tR)TO{F@xFbTF
z(Q&~9H|CFc-_SwQHiKdG**mM3iGz>aH&5Q&stIXMChf~n6X_h26P#3+1G-zVNJ9w~
z9iJIxcC9V#juOs*(n9tEA0x#MXIO}rY*J;ZeadupiG6bMj>d+(g(;xCO=25{M&QHg
zyd6s$*<0Q<n3py-4|xiG&yU+!-t?F<5c2U27>UmZFmKKX-E!bNFVOStF<FQR)rL@8
z4c?we7<Y5#ZX(fwSGc7#R3qeqYavRvgFGn4gOUEwu#Dnz(DGA}2lf?=qBH8;YSFz6
z_Jeh(<^&D=iDjM_3DthpSV<<iO3*zSO;r4>?8&5YfN(VOBTWg>RDd21Ia5DuFI}Mm
zuVWD@fmcky<2VBho_iH>#~BDW&6~`JiTD8k{}w;syI7seg~}~ISFE8{xiUgnJ^T!y
zU;cF16^GvSpBJC3403JQKK>JL>zF8}?=<`9-v)l|6I=~Bsm}C&Jn0s%_LOU(7_)}M
zej{q(?~4vtG_N@OQtG0@QpFFK%FV`mcE1^2oLm5e`2fv2n8Cm&U?&zzIu{*Q3|PXY
z6|>s#%q;y9)x`ca`l6>_+CIh7$VBF)3%~byPj;EmY4L?r2Me<NT3Hm8Y&eeL3p9@C
zUh{i*AQflx*E+8mh|0$BYh~#EOMuwHDGp4U7RdXhO3N}M<&R1zLNs)C5bk*?#)nyL
z-PFZ|e|?xQA^51`Q=pNnE{Qo8<JWfrGbN3o$oo&SZ<09ABvtERgBDGmcQ6~v5Pypq
zANB|1I<kId)@?5AWef-`?#S0MjHB;>KfQ)Ki1w|HLHQO0KcEYAUvI|&IK%fo>&&H3
z2hF$Ex>qrD7MTr1@m<zazwyfJm%hw|1PWR`3=(+z|8@|64cLSGi%4dlr$!NPViEKC
zM0!esD_MQqTZ$#7*pd40tqb_yD=Q`Z3S}RCV=q;cr1_K(P!J;>Ey4L%OiUX68VkRH
zS+n&U{uLHUL1rWZvu8*CA3j^nVY*$s0}9}YiSV&fD?j@DuM49e?7ClXtUJ6sZXqS|
zc;i=<ynUJdOs55R<HJVfCVL<-D>cl^RvPbj1K!S?-c(?gc|JCckSqT`zl19_`Mn|!
zcq~)n{qyKAjl)~$Ym!&5Y-+;90iY&kDb5&4WZ(PgHeD)&{(qbzN8b6m1F{RV0}p5;
zpaW~ER4EWCy+)z3DyxS6Mi6H27XpBRIi}dyJOr42X+NOR<o&}K_}4XzDIR=2TsK0e
z!I(byeOQWU*&A?l^Zj*n|JN)lAO*z)Oxv=L(kH}n9+x7N+q5<drybAE_l2%?dde9P
zHN^`&N1LY}*MJhyBVxN(t3L<DoLbqHX2LgC2v9q}&f$r?H|+nNDWhJnbE*L|-x9*=
zw2A}#hG0ecaVO+4IFo1^QdZ)#*`pIlZE_iuk9>1l{G#|jo;`reJZ9nxi_2e{0aM98
z&y~f-Eub_rHBF5NjCnv}%ifR~nb+5;(!)w5>Hpn(>ksWNUL3)QtxbBSaIIg1IIq-C
z$6wOrchV-T2B-gg)U*G1%!T6v0gLy?P=Uwf&2oP`&GI%J1tww3K(ldW_-q!bisfet
z2eL2g>W428RhY^>_2B=$frd&MfDwyULC^U=M(Mu>2!bqq4_kiX>~{sAP%|Hli+r@b
zG0hN3c$b)V%>%Z%b%BrT05XlKa^4K_Grw>k%9t)GAM|=N6{y(*rV-#A-cNi_kl){a
zd9~F;udvUo{o$Sp1pNQ}{S5ucm(nMQ+SlsR?=4Zz#o{FSC#|@J1M*0k%|{618SvA6
zU(*JDt0piO4*6oppQcL2kA4Pj_ez4%DbTV#2MWB1A4apkzF**#PpML!%_0bToGl~0
zfBu24&;`=I<>kJAw{|21nGEq4fq_AcHL+LZ6e`W;k{prWAjb}juQpwPQX23Ms|dsZ
z@DIMr1f3%GTyqKdjapJwyj6(X_>XD6%Qh0nU^k-C0)%H4U~&=NTbu(!1x2#5cs{>=
zD)gl)u2ojzJTF?$lt4-7*YDkzQqsQ8Z7=Z}-$%u6`~OnimVAO$yM*{6)CGw1px8%e
zKV51#-t*}a@BUyOx6&u-$tbVALVafLjBd5@PAUJ4ek|RUj%5tNF?+{>Pyhl}SBZ4-
z*P_p)q{O7m5=R+^-i^JW!zNLW&#j51&Qjgc^lNj@9o<es9Phub$Gd^_?9V>m#m-jr
zMx+x^ZuscW^jYxY-v0zp!xO+Osku*4!+}4LK)FAv=P`kB1^NNwjPfgbG+dS{&E<mw
zH&2P5jM|Gd)*t)c%;_=s9Ixd2bUQYxGjV))`-I=yiW_kVi2Dd}db$}31`pb!90Px)
z{?D$R+{z$LyP`a%k5uOr{hFSgR?K9S1Z#=Mz{Ncc_kIZJBKS8tJAm1-6k)Y9;oiMD
z-U+qVz_~q?_Yg}#SUV@GctqT$ZZ6~CGRHcWSFiw!soNGd4{pJH?D=7M=^72R!4)Jz
z5Z>V0q<b4`cny=kB%wBbYOze6F8>wz=KJU8mXxa6a`R)K7?+UgM!q4pXFupJwI?CU
z>maKVHl_s^A3n7i2hIugsdxZzig0?)@`Mh!EZs{RcLv%lDK?AP@!~_w7g*{$wPq^$
zqFi^4u^T@>)lkf4RgVkZL;>>tdMbUr(AT<GD19C0ub$wzoluDURstUV=8a^puW?w@
zS~;5ZB+n;4of~d|0TWO`e?HJ43?)%P`6vg+W^wnx7pOd0nJYG#mr^}xC9_yagDZ%O
z_L7mlJs0ME0}wsTh`vL*toWKic(H6A!%LVgoJm?4?bAo3>$n7~2;brcxS`^E0k(@H
zS^t3J_H;07pcWHohR$yUj|-qdlBFf6bfoE4on(+cD`@n!e6_aJxs>84{H4;Au0Xb^
z=JE90m$3OP*CuZ;>w(?|*eZ2!f<$8(+UQOsc(y8Cy#A{nE)mQ0F+3ovpL?N7tDwqT
z#RIWn{z|qG-#I~IVQW1P7iMukHiW|0ui>h(!9T(Q&F6r3cuL{7V8RG{^j?QHIxB1L
zlHmEDKY$G&RPJ}1x1RU>m+AD69F_VAAk6DE`Co6D{x#X56Y?(WgMe9#@vkc<DPI@!
z=3wvY2f-9+iA3gX;=3qa_IQ<+=6}ExifG>=hwoUdnV$YeNnbK*brqs%(n8K?)G9b|
zVqfKz)!npCG*$O0j53gyinA@rltn?{@(~=Emz4hySN?^*y)TN)8dS!1EUSO`ez2uu
zYyAD9b@J!oye+U0xUb+e<v9E=7NT@>LM(E{ork@DnH__=2n<SYKm&r{@d8UTj|{C}
z@E&`APd$;$N=8}eLyrle4JJttd>ICa>8Xu@j!fE~J(B99SqT>XC`zKo5v<8&_^;-<
z1L@{Jel9iPv8&r_l<{fkFda`DaOU}3dOPHaQ+DzPRZe%~9Yq6B)i-POSC2X)W}x53
zgLCQ*gc=T&?CBWi^TV?}{^VJ|tpg(gbWrQssx6NK*$olY6f2TP0wPm?!n-*Qmsf@w
zP9y-WcN3tR6!Pi5;|Cb83<dp;`!OeBG-uM{gZ>X4<%)sicm^$oRRp@~l4~bzuAi>h
zY@FU9^m!n$#H_w#BNB#!M+f(B1(?MDMfOAUd|_=yLY40^9%XIr{VWMV%#y+JTS}>3
zQFZjixu(BwmfnFC?|$^3=&tnw=r2I>sRQ)=K1WeRBdm0}G3A{Gg5k48HhshFIyMYo
zYR`RwU+lp4)M>5%a>%%umE}dKHmMHF=VV;;8&{Do_zPnB?!&^v9<mc4{bE#StBgPw
zsKe+unSyfcRj{EqQ*joL-M@o~TNPnl_{47Kv<<TIg^zsq>nJ9y60w<8WQ}^aD@Ju^
zGar6f8T-vrGoQUd4807}NJiQx$CXZXinPqQNlVw%DR2l=5fnj>732a)!2G!WVY?3z
zx$t!&3wf86%revZrHuxQ27p5@)vpLJmV^SnfQdrLL+Mzd$L5p$UR%>=+Y>7&02c6*
z4Z?Hv#gm6_5B=w15Un<W?g&FBGeR-f`bzS}U&)-f+8$c+dm#3JpN-G@jP_U_PXo@E
zC(#I;)WD~}a@B`X(FVtvoIgV6QN6|OjfENom<Jm*+dS8Cc<MvtqKC;sF0!;JE>)ZO
zA57?A_EgL8L4=4<xU9Q=wrpJ%ds-&JOowXydLN*<PSSRbvz)9`4Kcn&6VI0bjD>zL
z4Sg%(2lYyRro3u#?dEGAXmuVev6RV?hX`Q-4#S11ETZK0fr_R*(q|ii=@Exz^NU?q
z{nquJl{=a_Q-ig*iJF#YVSuy~mtMZ^(`a&c)Q-spYIq{A*tp~NEdIKQWr)&3{LKVV
zX)kYDGgB>h!Wsh%Qw~_sutsf^)m>d0h&<2fyN~6omm})I@*|cxq|$79gE8r<@gKr{
zG?ZU^eL<UjPL!S(8|k&)rxQ#ksrw7Vj1TendLGz(Kjfas;Nt@IHcS0d4<N9PC2|*;
z-n#C*5nZj)sxPY~@}WJSi^puatGm1Y@?1QV(TTZ>=rmS%6tCC>a3SsBDdDsgB!U5?
zGSeIQFhdmidf<JIGw52udlO~Y<@L8;?!e=ePPWtJ`hZVrr&Dl99Ezgf0n?Rq6~S@4
z7K)M8?g3I5o#+sr>KWLhjY5bPm^Qi`!fTAIPB5=}Oc+TI!k{Jisez5`+m*!FZ<z|a
zrmMa9At)K75R(o2$OxWBJ5gTflK2Y8>nvPld$Sii`hl9IZ<XnoiFWh&@A`!jSnRyR
z^Fg;sR;M7?2KcEYGSg(;_9W~i+Iy=6m#MtiYyuGlRi4Em1^tVdMtI4k;xYNW>`}0e
zSy)Mi5G||wa=T1YaMYPm?U~7+^|W&&ZT67Grvkt`c}JG>u2(wh5BqFP#Ub;JW6L?y
zNZ$dB!m|$S1Bt<C2<gaGGnw%$&ce~t*omw-YRMm%Po<$4@1PIfXc$_YV$%_g=shKs
z#`(ZV&%CDoMl`fsO*aL(ID*@wHRK|ZKY-}_J|RP{S+pC9G%rALevC(f*xpdLNrdW)
zYJy8b(<%Wkted>d==vEzr2YZq5jnlddpJ<kxZXZF1*<4vi<43&ORL=C`LZV70^6&X
zt>FAPQx1p?PJ|fpu_x-gE7i;EWg4&(HCxY<E#~v(zm4v0OLJQ~SF$1veAO;cBdl!r
zcHQ;zRPf!Cnzv^s&ys{!QAPAAn<GlmNufks9vif1I#C=L2IgWT-e&x^JAYT7Xw>iT
zhu8PLgRQUalMNr-r-{k8*rdZo<<`}25k5fmm2ys_!}nDkb{zHtdEFCCfU)V-LNH|S
zU<dvB1at`T@4zPKKR|Ab2alp`Bo6>DSRI{&wtyZTwwHHBF=+c-5!nyLhTrzKPZ^$D
z%{fdKf}n}dzlbJF)<OxqFpxiRjzG|+#ix)-{g@_gp0r9nIC=8RFsMy3eQ|_X(hS0<
zpMLFyxD(K*jqkr}DAILPFKvgf42jZCAE{d}Rq`uwL+)y_b-eR8dOTwLb+%Yk$exSM
z=N1$w(n%`lLs=E|O{%XKP&`IwoR-E>#9D<|u3;X3s?dCBWVTnicB0oqqj?;9Q<W=@
zH60<xzusbg4hXGj#OqS;dh&=MZ8jf$0fVA06G{8?=5tYMg+|G!1CtCoNCjSurYg4V
z5OP07R#uh@f-b~@So^>CcixT$mV<z(jGxH>P}*EBVtW9>vqriIZO)oj4)L;z159nY
zpfcHhZ5O!B&=uxmlKSc9UB{(mMd-zqpk=~NQ+=!^_;78w^u??+aqt<QFJ*zEY$6C^
zwkuPl6BAO2t|rzY2iVO~!hB*OG^I;N1!+g?KU1X)P4vd&zbFJ$OMKmxOxF-5D^cyG
z$9Pox4LE(r_gsjsX8zGIQ%eU=7@c~a*XY=4uo=T$xb;IsVAcZXi%p5E@uKCUD{es}
zmVGz38OzT}$8kHPqkTDEqw`J*1!@i$KT{w%O^f5^v_b=KtleHH+JM48Qm>9ms8Jam
zCpJrPy_gP5)cF4bS-`2#UJg8o${cr09uh}fVMglv(-#=2F<*XbS*fCXvU6};aMAM4
zDQxsNpp`msYXcCkuGeoH*D_m&uVVO^&|xVMw}A-&!$_x|#G%!CHJx`*T!E3a;1*%c
zuWM;z_ObG2U*!z!G?_hTHa0W^`?Cirt#~r(i$CBEA>C}@o?@I-YI)vo8Q%rP+|24z
ze!qaG$z`X*;!LY9xX|-ea5$>DC$q-=hu)`rlQ(mYe)y&hFxw8+H30~=IF%|eJJa4<
zphVeBo!Hc^RnKFuyX<k<CF`!Hu-!ns+zna6>L5?EFva|&1t(nah1&~-9U1rei#f=h
zEYoG5)<}<d-sG!4{2TE0xYR!rWLoa(g81`eAh9NpU{mBl&1mKnokasY;Zuy8wa1A&
zw7BIO(@&aoOm}Ckck*TbT0|3?$;2v7Jf|}UITpKY1oLVBYm=w9mQO1ERa}9U-OiJ>
zyhN0jQqvu@SPH-L3mWk{ejwY?*1eOMpjg0IS<qwNVw7<H4c6VJ)whV^8&AiWL^B6H
zH#(<2)(})k8*+aS%*;;_tmi=L%+Ddx=@Xx?di7lwgcV=m2?P2M@2p|;w}516>Zvf_
zBs_k}FgC8(#XDnN92e4ZKHK6aB!}(`3emhBQ}Y;0TI-!CB_8UKfF+qa$olpj2@7>d
z$ZIQsErV=gJbP8*0MJBU@v*dcKS0}{oz^+mBIYuCw(vE(KkQQ}(Nt+Ap$YaOrvJ!r
zSpxR$_KV$+Z=>>&f>N)3Cv>`WezxN<L^)QO%)tO(qOWl3=UFe_{i*Inoru3wiI-Bf
z_`}CZ@Pq38v9iT6+SC(gOL6rj{xCWS;h=fO2A<hh#FO^*{os=VQsxu<Di}>8B&Puj
zs809Kaf{$s4IlI+j+Fz061`AM^*QW3mByaISD*Q;>Fg}j9|1TT9MVH={AzEHBp&PL
z%4~VpsIac{M}xY06sB(N-%=&Fft34HQzeCwRJum-0U+|Wp3y>y#rhP<{R}+hR#|8J
zB>xM=k0KEh7PnpJO}iO)8-Subl;$mbynuZ=bC)^;1&4QZzN=pdbcW;WGy|!|%#g%v
z^$0Q<%Qv|ETB>UFBSxkC8Pd-s{jfjASQdWRdcGGO0q&0pBD<0kZbcIX4_NFvqYFzU
zNP#bNlv7`}GHs;r&CIR}T*k#(2_ufai}ydWIZlhf|Lb0I1?@EWhev;9gUNKgFxh8o
zl<b5A!F>*}8fkP#Sdzx$u)=z;u*S}JyAxFo#Rg};e~_s@s_zME(ha(uuZqxvB~=O!
zOc=2zgm=4-u>pHZk?n}~b^bpcE-$N^nh5-H6JwpXEZbK+``4E{750cl=uaFEZll_R
z>|c`1V}}n<0Q<;^8{zBXIXdaYs=B4r&_RT4m{pBuWcQSR+w-DRWT}0F*C=ZuS8<J(
z-FTSs{IS4!1JJc+Ut4-V1s<P10YvqucsI~2Yfm(Fa1(A`>Rg)#s~62I$M8b7_{*||
zYG+|}^~J_2>jh*Qf5Aj&Nmi||$pTFi3E7*LhuxG7*f39AzzU0jg1z^s(_}~3G1PF4
z#o2|ttSs$P>v&a~-7w07m09K0v*`+J6dp6>Z?87sL3CT^d)FvnODIM!<Z~x?S6WL-
z`b)@fhL8ep%eq?=J;H{!%<M3sJz%T%tOWY++qgT8SN9=l(ar9oWIv$)uw5(sfxNbM
z2Wx0ONWDng^wu@;O!tU>FA;NGbXXUR;h(BVMI~jM&?e=;rn*}upgVWPaeR=cGd7ub
zkt*=kV-(B(wvuFFw8SA%##Pr9z@#G^H!4N|08}$QIw1yq8dHm?&#W4|becwtpKwUE
zMiHqeR-EeBdPg9I_##>ZI6jD}im~1?8BKHMv1kMKkGOn+%=>~M@HLY@*wXxU_lq*t
zs*E%Po^_L{XauBUi(e*BXOZa)K!hS#7LntzN^JBY0l)sixk~+CaIV)E?D#NFyKM&{
zyKN7lh2BzN<?*@H?b=n~63>xT!+{d8W~1MpK>iF}nF)Ev^!f?;&`fUhIVEn{&V;?%
znDC2Q(&W+Z{PWwGw;4#u6(oS6(eFs3$oQo|F61%&KpqrAg;dhhb$Z8Pk|!hA#;ve(
z$0M43?&?#GO_4)zfvl$wqTbL6$A_U!xbE++S@<;=lO>-p5OZf@N{?g|+KpK(03WK2
z5`&f88g=7=SpFm2qB$RTp2n@)nd-#7-Tzy^CnCO1b>Pp%$E4ex!>i>*2kNCkTn;r4
zF4W#~Jk<nQ(@H^U0Fvt@eWV-XE14pbRVJc~4@|AJDp>u5A-L`g7*$q}C85RwGFgR6
zPa7CRaFGK<b3<P9YxTb#?*Q<u;D1Ffa+kog_3&Z%`v8e?o&v$u04_=<?JnWJYQHW(
z^%ZC;W|q^(YqfNJ+y>HeCTJBAm2e5DE0CoragW=j&P6P`zM#_W2EBWYeti+o<xaAI
zue4D=TJej1;j2%^o}+tdK$tRb)H$DMeT(77p9)?LoQi?5jrvYn4!JPjYWIg;FtW*p
zc8i&E^PY$+{Ss4isFSu7IY~suT~*VashA3o2$JUU=LDMLngW@sKUwMMGi)MeOcLnh
z+kJ3T;CzG)J>8x;9GIap!e1iB4G(RC2fGYi^kB-4HdX5!<sGm9!O0hP*%Mi6b)?;{
z!16s6leMsp<>Xk6#d?bXRJs>lRb^m~T3P*>tp4Q1?!_Vqj@BHxRspU)CC^c*z!3S_
ze%Qy%e%peapTETwaF_wEKJZ2L^!p{>*e5o&(si{*WwMv&t^Gjxw;fBe<_5Qk^Pb@@
z@v?=~242;FURY^0AY!lwBJ;PCN^q!|PSOg*bW{A(S-me4IcKy1{S~7Ooz0hLN9p8o
z-LI!6kOOdUUOz4D&!OEdqo7lddl^g_(;o2it#(bP<S%OO5<Lm5kT5@_f50pNYSjF$
zJ+_#rLk}qQ7;%^Q3Dek`Aa{r^AR0EJJm;(5F?ZeNXXFwckhSa?bq9-gC2Q8GUlyPP
z%AI3?wfg;@7t^AnHuu_O{okOYQ3wmbe>WaD6Y>~s`Gx32-b&sDOxb}EqCw%C8L|NP
zZUEIv8(j4rQ0cfxAx6tjblK^ab2)#2A)*V4-<um;AtHIU;Nv6nPe=N{SgwuPnm|<z
zp9z5RLNV{Bj<Cn{yLkLc=}aLAsxg}v3j)jQ?CXRN3zTG}3db2`CUx#{hcg1H=gj^a
zL4V|7ub;l}qTLNJ5`)gvN-3NTPym?3YbhZ)rp^MAoN3k5s1EuBfKwWfsZJrAf5VVc
z?0E6)={lnK*PKEqo3B{^HcxNYA3*OW0Z?F|d`A%P@GVQii~x2|vi*<7RYbwR)J;2L
z|4P5~18$uECbYh6u!{xyyrW0!_|>b)4i;=K<R95E2XOD`JtHmlzRDZ#*e&ix+tnH}
zjx{ZUH%GtbtC5jkP|nZ7g=d7XlSNAGzjPaGvlvX-`ln{H{icZ(M2S0hK*xJcTcVz#
z3i~nv=4X`n4_YJzS(f5Yma!%A7F%VGi>PQ(mUcdAyP!8b`5(s749O|GmmS?sNkS=_
zo~Q^m;o(r+22t6IPYTngv!d1s-!O+#RJ|Ux@_H-MeeSI&yD6Qz;p&grW(5u*Zur1=
zTxpp!>Hr^#uOUpPl{7n^1Z+a;if0Zi#oze)Zya-NW!^PZmGH=UbKQDhez62Hv)SD?
zGhE2Vaxkz-lm84peR;Z@_&WIVXJ-BWo7VGkJU%6+UV0q-Y@*HH<w3qQD(-^>A`t`b
zXEE51|DB>4&8VpT@I|~G6(EsTy*Vah*F7cx#O9Do663i9hj%3T9_SSzy6t6T7bFab
z<b^s^`SFVA4|S1{7muPRW=al4#1P#O-TN>bTGC_zD((1?#a<)ZOd<M%x#o8;(!KOT
z{TX~HP@f0#rfJd*R6luu8y4XwDYY7%l}B`?d?S?6=T^mF)iRP-b1smq%fN~jpk#J2
zg%)_?{jvgA{}O9f#UX0?yIwby;u%lSqbhlPvWaI(V=sP+=y&S~n+R<=bf{)3fWQ9-
zhtwsx?ZZNY_TGIi_W1+ho8CCL-tKvKmAwF}6;SKkaK%*IlIKNa6_`8RA*eV#M)TKc
zmW(Mh_uDkk5&4;R@6HLzRs!_iI4*9w%m7Iv8pc{!?_1Cbz_ga=lrBqB+1-Yep@7c%
z1A@T?C>SmL+;#BB@b;m<=#qR>D_T(NR7Y;7^R7$JE>JiA1YT2ilaq0rDnRSb$#N9e
zoZ^MQGLq?1p`Io`kN8!Pvoxz<6-P{_IBq}d@+n_V&jQ=g=jo;?jZKzDZ2xnKe9km&
zeOn#jt*X_B;w8bh07};9`8c*7!6Rc^wud<#G29LKUk37SZXcVpdG@Q8EZ>PnGjEqz
z^8QXS7_YSaN;u<?F)jIHY1*8$cn>JQf2-tmF&`DpkIQTB`m9da{6452!!835_^(p9
zY_Z@8oQ`}JP3Zert|gZo=kZS`oiO5#r`%#3O>N$WL^=P|++)nIjpC<t^#@VN3jiE{
z98;6L7=S~5fzy*E)Gjvds!%dgGE}-|Xk*D-fL)HN@gjDaRu2(WJcEbs_!&|7Pbd8Q
zjD8xTYN6nCz=u~`32?8;Ox=%I@R=OifC=hB<HQHFZ!M>KH0N;xq^f(Yt|LeB09ijd
zRBL!hiS00h6f*aEN?5edFv&%b=I&O|^(dd}qID|0sUh*;TifuA;;siefZc7^<3#!G
z@T?)TL~^~{=IF(iia0^isUj$);v$Z*?=SovCFmSCk+AOWicq@C_^5K>=37>Q*+gyn
z%nI5q6N7TFpabS#@;{>UO#c#;{Ql|xq3f-qs*Jj}UqCtqlx`428tLxt5ReAxPHCh|
zx}_WGPU-IM?rzvLoO|=U&-ae+jC1~U=y30S$69mE>zdb3XrdMo7WMDVF93dA`oA<b
zG)?M60GAC!!q=tRGkw$g?+!mOE^PuN!UH(+|4Fk#Fk)7lc(mXA{NT9=h+WzMZ;D^A
zv|7&RawpOsD5tg&(KJK2McxbxexfwmQAqSuTBsYqKX<QXyIWW0@Yts+bnPX+VQa@8
zmimy<0Z1t<|G!uGzbQ}(HUQ-Ev>&E@4i7PlF-gaAJEKCpijQD5*MK=B_nHliK`Pf;
zm*x~p@u~9sx6Pt|wr@u5+~RK!q;oUKe^scPsF;`FsE{maCIVpqIr;{%t>c&=xwZe`
zg$voE=lokva?RA%RjH^E+`({PaQ=2RQ4O#)vI3t#kgv*U8xNp=-$JPR%>hZA_swi#
z#vA<ud^&6trj(ab-UN^~QnrIBTzfPAAVSRwYKKE@mpW>V$PX`VCmcJnX%2Ehpn9FI
z{NcYpJ2Wi$6}w$=L-QUJa^1KL5b(87X$ropAl?+wJIQ?OATa%m2TLz5ynuxwBUZPP
z-LAL?5&k}ZB+lv`mNAp^EvZq~A=}A=f<N7`amLg<3<Wh-CxSLHaAM1bcvCh-DuvP5
zHU=#4Des_qiM}t|tR`)ZiC#g|bxqNiWm<VTOt4|3+WNaJR+Yb;!L@#b5a)H6ZiN*L
z8|t|UY5A?C(g}zbeFWaEDSf}eyAF2KJ_V~Kq|k^oaPr@Vf0LhceNuTzfQac_^`VF)
z7PX!a?$$-<b#VmjbryEBejQ}Y8ZdV4mcH@F5$$U0*_MF~_(t=PciuY@>zb=ifZ^vp
z0jkUiNnImLBQCt~9&vNjAm_9lh?yss(8}s}^%+zrlksY}CzU+GaHqX5v8-JE6>~#B
zyGsfPolIcyH@hL~v;ua2L2MQ`U&o2Id$Gh|4@)$L-p;z9_;<pGpiv3;C)5u#+u-o_
zWoJ{<uc*f~SsaI3N+ufq#(YpaWbb;DR3>%OSo$)?z2~;Gzf_C~D!!j!l|ITV&`bdO
zidf*AD{|TI_*4j&B(eFAR<dc^@v(8sH1Pv#oOz+$5w2oQ#!Qfr{9=nF4n-~rTX1<=
zyV*E1E}P*p$_@i$!z`&to_{`{Z)Oi{W%axS7vn%GmY{D1W_wVP<nsTyd&(j)i=Rf-
z_aCS}G%*5u)LVQG@bQcTheZYV9fI%OA=T(Sb|$UMB_hI<eS5G-+8LdMmfAsJ_`wds
zT#~Q~g+DcURH^ZY{NaWHS4(}Q;rf<e6uK7Zo%2Xd_DlwJ+$Pbv_x51(0L<GU4hdrP
zKi*&44eJ!VrW3-t0CZTfNHTWzwCIA-^E$r5CC=FfBM_1LS7kv+Zu$7dA^}d2(0w@i
z*qsn8oz)=#{%};eka}Pm5Q;nY+f|*h<MDwOCnEwltQQ6G6t(2qQ|H|afs|{Ge}TdQ
z@jnFR`3gr5hv|ig1b?R$<tvKlKLXknJwPOKcewcyMJ1iKp_D{sr9bHbi{rw6drycR
zN6l;0T`#)eugP7~Av<!|#12~o3Z}|rARs)%NdhsVPSU|Jj{?*kkUaJa!)0Xh2_4)j
zBze1K4~oa?)}D{TNnf|j)NhVqy(osVgoCoYGvKmbTu-tr7csn8;Hi&Ju%!2RX5B&o
z?u?}Q)xWk(hLfG3n#jKoSoM=eN&bSH04ktee$oE!bviNQDdBaH?iIV|@~f`a=rjA`
zPL*yN;@DGb%Gu;&LtxV?9Z{RQDt0}{QG2-zO?SDh*i`>kx_u|gmGiibgw)1{T8JC<
zh~xxBDmjT{BAeAO`qA*Xl3iM8k@tV$Fkr>&EwY)WrKYusq5ccyGIzFEr`L@G{(Us9
zi#5>rtUL*D$Or(3nbw7938JC>ZMRE?o}&W|{i%4QRNx2|E4%cs6+Y|$u_RDLPd7|P
zNV|VwxBF$GWSOX_uA7oMzZe8B5Mhncgbmi%2>#>Kz*H?(-Oz*pZ<#8b7wjGkklV+C
zq+{|Fl5Qk}X0*PYG1K&RhDF6HwW>>$SS>|U%X%SFTq3cIrnNB(1f=bVxYB(6Kku2M
zhYphlrL1YBMDp=Gkln&WX1xEP&$0%Eqq{1S5@1d7g891dIl<%CiV!tt(@^=t1e?vs
z#7kkt2>gtKx@<N$j)#kg`D2Y}gNghx8POC~K?(l+&lGI`47|*1W?GU`$lkkJGk*JL
zfLvzpgLXhBBh*vD3|%YDqt0|C1Wl$WyNB@`%$!<1iYpY8<%`(O21yfyv{9hL;Pi**
zqOZ&XpZ^zZ*-?a8FFXg9>T7NDG!4IE1av^<gd}XP^Zi2gl0Z!QWXlJ)eK$ZNiAHAT
z^;znf2ZUhp1RW4Nr-0$Cu?b0o>qxD1seU6;>$Yj_QvtctC?xr(O(&irApJ_c1ZKD7
z?!y}WeGofQEh*rJv3#qXf5aR>D9JN2N}4xKYxUl}tBs4=U0OnK&SWlfVm-){5xz?#
zOCYgibPy%}-`@sL#@n#GJb2q|{Qlu9*@i&vZ=7o^$?X>_DAiZVPVkBw9n&lI#t{GS
z-83*cOVkBCC(b_-{P>?@GZ27)CE)45Pfm*RXyB-L<pxcD*UB}FId1_{ZYypGM;LUs
z7a5LR_C|UCtcVZGa|Pd<(SN&T>XYl1>7HW{tQV%pf_4%jM(UIyItvn635|vmCa!tH
zsCzd2=T*6eXV!Xa5Xw7B$#gI0@aCR+i91qrUo}~L_~AL17P7=O-UcJ0M_=YyeNN<V
zwX`X5TA9Im_di*)+CXYCi-jHaQbOMCNsf9|vvU1W7BD1xujmbo>-zzzm_|nW$;Rg#
z>3y8*iG2JagiH4q*6J)apGsUeQ0i>nDs>=3Fv~q7<!LW?Olu@VgV|$d5T(LkfreJ!
z22S_=63`LI@(i-bG|1Dv$_YOGA-6EaXwo}nxfB+oubriF1v?ImIV1o%oX6bGEpq$^
zb|xpQ&!1$HE8v0rLb(6&!T?HRR_|Zctk)Jouo-4kTzlN3+|8OiE+Q2{wFr>V6Xf-R
zetK2m6L}wVs$WeS*V(i$o;3|+X^zddp459j%8l$<&OhJlctIz<qNbGx7^}_giCZQu
zY6ZW|4@fr*XFe#_^?$MigL70nqk~tnZTn$BLbsGY3Krxl&E#+8N_!0ri|ButLm%gd
z;Iebx%fU`LnZC99yuZCRABI?P&ibK-I9gI>_3xg`dvw7ZT!scQpn4;nJs*_=6yvo-
ze$>$y@3uX6YN`zR4~uPSqnp}Lh!{EBDzyJ4H(An%cj|n$L+=mhndmOkcR6bykDG_a
zpr<R83EBF;c{G5@1q^%n?4J!7N3`<(k9)DU;~G({ObN{$3?CB0(Xis&F(_$Bjf(I?
z%DWN4`&`h$?f`7TAqHYV?L-ck-hu-@g?$sQUpgJ4II>9n7Cw>}Ij{?4fDezN5px8j
zVOHt`ab|ThD~!OnXELBhw@(m1h%#!lZePsU3SW6KdLG6et1p+hS-*2MXNP8Mdr1dk
z+KykIq=B1}GJe|YbBqM>F@fiT%<;_Ln-U1neC66pO}FAdc><*U3J2?TiKo*vaif?U
zpe<`;L13iIWzL4&`jeS}XP;e(f6OySp{IZo6s#C7TAOhSG*SM;mKpyLjghP{14pWW
zs|aI!gSlKhz${~`s2V0g<=Xl(x|xIc`B#QkO@KKrhFcf~AgB9Od8_(CN8gTPE;>sh
z()83Pa>J;R0!y*)8_Hb-m2!(J(jEB=fddP;c?p=U2zL1t5tWTmebiP{(L7<7J2A$R
zf;0Gb5Uh7bQ>%;=CSUd0P>>_CiFFo{bX=S$HS>pYkR7liePqvQ4f-t7RQ*0x11XvV
zV$(<agp+bB-#hZ_v8!OjN6DQ=c?sEC@(8z(BFYl;9J^)e^jvY~_JbZeItz$?g?oD;
zvst?$@6r;WJi{H5A2B6^o8YvmuLSiVxY!G2A|2?DUx2?mYe4}FS7;7I_z1DYgzy&Z
z!@jG}i-#w-hujLFeI;w7JjZR?I%^M6UiHgsj-={!FZJTuWc{$I?Wgzl=41}K{AVjK
zEc`#mzbbc1w8J1^_~wR-=Bq^fA0nsZkP{k)3E80-afwLu?ojmAd@0Oinhba^;R0(=
zP<p^HUjpaKO2Wz6+4iB!LK^@$Eb04eN965Bkn+$gm*Om|FCa-(*J3*=H8#(f6^Y;u
zU9jQmu^>DSB*tqOBz9~WIRpvcuCiA>1Ge{$z#E&&lR|*;+dwi>W(E<KzypDTmyR@3
zl-3_X%0bp@R4lgGSg8PHCLOCU)O@l$B;W@T-=%*#vOD}EDjzZ9p*1@8!^~kyb^J}F
z@j|HZe59|lvIU|{EGc;FrZP#jlNg{2#>^bH;u%d|o*B&++lMb)Y+bjdMwHd(1$sc&
z0*B&HC&Y-<w~F2GRe7Pke6cVo{}mRA<@$-}oE7?pc1HR~&Mz@qLRP{hec%3`FR{ok
zsV{~hE%Fg@TWs>Pq?H_Sc|L>TET17555gTLAL#&t)y|<y2l^g|uRGug(Z)Q2P5RXr
zebB!<w`yqdl_T40`24`K8JxQaSdCg=kQaZNLV7;bn}j?9Sz=q_U3w=6n_3S>Q`d?o
zH}qUcuqe$dxQPSMds7)t`y_?V_8;qX-CmLFK%J&-L6j>eYhG$|W5Y9TJK<>ai|sCr
z$4no+dOQ~Xek+w49(ec=wGW5$2DkMUnNx*Qeo|6ZEG;kQ+Z03Ej9B=LCpG9dH)@Q#
ztjM8B#+!S<MblS=OOfqGFId?7DXU%rL-FiA$0oLlg!&pZyToESz!v&1&4<jM5jT=S
z>*K^aGjCSw^SdK>J7`^lFiCfr?`TBUo*kvZY*r~%-JS%re_nY~Het8EG@dyFi3<b^
z5e(^~w*JqT1-kl6xRz+jBV~WbVh+iGAx>0Ho3$5eO?;K&qIzD+Q@|1P3FAsHv4U#Y
zz-@ixkq1OAP=q?6QUh=Ue;g4Kdymd`1YiCUae*Xv6s3sj)>)%~p+uJ^^^o&-sz6XV
zO5MFbQkB$F#l%p^WHbnES+^0_>F_P50*VDMa5v`z1URt3b@MqPZ%lsk@Xibn)YdKw
zn2C2zK%KKQW0%w>wiJBClIdzL1&&8c+`y3jO*qBYB#K?D2SIeCN7Iz|_f*Oe`VGyO
zRE0(i&8<T3uXjei(ZdXctVnE@3Nni({SNXxDi>xEY4B^mLP3oUI)a1amfz3Y;CSJ9
zkd;k%0nY6s+|q4#HNg==kPJ0ntop!cKm|yIS@PKF23`gb7D)a&AuA;+0N@=Zq16-y
z5eTyg-ZSXmD?bu;7s1wQgST3Idz$0qN9&F}fP$=IjmWL927IEH+E>V)FSohC)R#^j
zU&}1DWhvCCMWm(o+I<+*S5~>w-g=&OUdJ{NKFhZAH==ZWkFy_jILbRBQZeaRWZ|Dm
zzE-w~M&~-qkmZA-&5I-!mDef6eH>eAIcuy++mN>Zemu&Wp%d2M-`A+Hzp+>ya33%r
zWX`jd-K>~FwveX?tI;0&*F&RNp@1QHUZgLg@I%nhoVYO`aVZmBCyGK_Gc*8vn6L-`
z0MS7v53g%5_4lZ--Sxl@F^Lgy_*%5WBxBb}p*sZO6;yX{?naw?zn9lA?31{}m-J@0
zoo=mt_Ekj`Uyx5{3m$QbT>tzCVn_3BtWN045_TiZ?!`6h%&^5EOef*cKW6hW*G4k<
zo|^h0TP!MyC8eys?iZCYnqSji`Y5WE$CYQ&c|S(NzqZ;v9Gv@c50|IEtQ9BEX}0_4
z@-kZyUw`#~r!L9qha$a{$}Trl=`i#1ZIJr0v<)1E(1HJ2!u9o5igleJW_{eT1d@_Z
z3+W(qHdOrBtPBHq+92P23QFMZMiU=lzmNm$OyL~7&Ym`n8Jf5j7$SCpUYp#@yX`5E
z;9traG#P?NqC9~^IYEvk#_iW}3!8HrJ>D5_8Is^|8Rz`MPxWE_3I!dT6kls*S(U+-
zOjWGk&GyS{9T^YDKW?7qc?q2Payk}s+XztoHvIfmk6MLu7dNBfOZ9m9k1tvaMf_S0
zQv&<!E?<5VuoLgnc|q!^Hz$9(y77N`wTmSB9AJSNRBFy5E4m#fqRSt-eu-5r(jgQQ
zCQ!?lbws=fnI*LON0T&@qDQV0ygN|8B@E`O@aSa7eS+7wG-_Ae<xnDCFu}9R;J;pV
ztTm?+a(yxCOdo3zA8^;7@BbIYE_Dd#cj`a}a?_3F$mUw30~%1`^^8#_SI9HDdLb&&
zr4ubaG1bZ!rF@Y%9~NUTxjT9dpbS6k7;m=mOH2q4(uh6PM|i?7vx2W3E9qc?^M1mX
z9yKKMUfrBwbe`Oz^@$vcpSukTq<g6B;czafLni!+zsukysGB}MM?3M_K|dy%5ks|E
z>~l)yEZHsdy<8(Ri8Y7B+02qS;?wnqIdZDPI{DS5>Ldo!KS%Vq?o7HmK=}amKR>lF
zQ$C~`kU<x|2Lec#(b+(&)YOiGPK`=SI!!7VL$7<;@}jnWr=gUy>RP76mVkM#8I8UZ
zn?}6e-zQylcA*4ge_dmyJH{+%^^vfb9%-OKMH|#dA5jP{gSXH`=DA1|q49js(6DD-
z@3<4%&t`VGk~s3QOT3*iG;JR{&{wj$Mr0REJ=p77+GwgRCtvtBOrM4?yOGY@w|i)l
z&uW)vtK-~5v%@XB*JFZxRJ01<{bXc3wV@Ov=-}efvGg$u13x^hQ9~A8q(sq3v667>
zp9n|wWjo10Zu4ScO1{0A60C2|lb$4euUixE*gwv&tYY@MW;*x3rgt4!+wLEl|1CdV
z|Dww?=LT|2!mP@GqL5(zm2e45Iby`S?dAk+Vf7w6bzt^BGyR)${Fgo;FT;#MEPEyP
z+BPOv{ZfvUC9o7VU;FRkPBwyX96ATRjj*B0Gmwk;3l}Lpz%@+w$J92EjznNa44oQS
z0K%Q&O~jIA*p;FP>>cg7A@_|9J=#M5kvaeF8%MAP{o=Q5<go6nG0`YHWq7A^fm(A(
z^H8W(S}sT$Q|2LE%!?SL7d)UhvrP;(Iz}rUmEc*r#Q|qGOn_z773t=Om->#-YMjk_
z5fDlTJD8;(<n`<ZnT_v|>k8-lC(F86Ev4{(TMR#c1x`)pLlNl0`IRSN=K~5oZ=zKY
znTYtpU4Y1Z;=ASz<e^XF6i%0_hLTFh0RnQ1%9ZcIaBCHGwrB%&WaAwC)B7W}W6^41
z@iAY30yw@`%8)#WaUE&lUC!j%cEsRq_&vdezZzC^adkW?ys6J<b6+RfXtujq|9LQ^
zVzh=f-HDi|B<<?Eb=wZ!F28h?6oNn^Q^O~rs6Q5);41=tkSpgK;Nyl6Fp|hPKW7(8
zIJh_UTJt8&A5M6JOS)}@x2xbgun0_)8!b&0@Ia>MdM8>n`|K(Bv9um$t7EiSBQu=z
zw32{b&*P<Co}CBow%zUnzk>8<s>`c((&~Snf0bb2wJv!CMh2=+M#|gzFfP1wk@aLe
z-F{CPtPSZvX)@O!@#pw{KX$+M>e8V-V%r%3oisC(9~G+>cg_w_?#N4FY-mzQ7F#b*
zZAzd|icc=GKquv9a2=T?WVWe3Hv5wAv$x*!9X~!of}p-9MA@w1G_B62%7#?0Phha|
zj$&2^8DU&$+gPCz0`uBfydk=2bR4y)7(<_a`$9EE{LNP{7tWgDcJ$0PeV>A+nppJ8
z*Aq@n@V6fWtCfqoRDeRi;C|D!JN@A?8ffaCgPXb^n1JdZ3OsOuPq+FS0hj4r4yOm0
z?cat^e3w4`l+^<+(f68xkLs)EldH57;gXLL?1}B?F@I^xTAqw*l4$!FK7jcub)~1q
zdr^R1D-1|>l_#eCpdS`#ii8hnhwZI0>Q|d7<7Fc)j7FVgz-0jdD_y56x}Cf1tA2n5
zqRL@uZLQ~!(L+TZB(%cOG4bSgmIl4VKM%9gyo`o~&C?fi-F$l-@JYTE`rU)9UQ@5y
zw6tWW4*9gONNa{bX9B|;FZ6PSKvA#A&%@~h%_-nyi2Bk=8G~%GrUs|CEqt{;dz!NW
z?BN=|t>AjfVQve_<f5ng4(hwiq@fUz_=F*I>*1OFIuQ%ckbSx>LG<sh&WsVc-Ab0Z
zWhcD8C@Mj}KK@LSyhE4Lf*#x$z*ScnG@^JfP6!>3G0SP0v)!yH9G7u6juaqp^4Uye
zE*R*w-hf*)p*RF|e6G9i3+?{Akj((eWkVR(fLXr>YncsWy-<*jGTlKEj3DaEAcDK0
z=S%T4tJn=-&&+)V{3U<9YzH_`G6(YNg<o~hs;J!=U3CElRp;H`TLFMJ&<F<!$z4OI
z$?D4)CRTZ12kt7+YCJ(*AMn90)-)h3uGumm?_a%7*&{M6ayUJxFda}Y{F%k~>Y2LE
zO86|G3o_88Z9b091Ew23n|g1NBr#vuakwa0<ir*WoOxL_2EZjaOKOz!Pud=YQ1SS(
z4U1w@9Woce_)RS?FB*O**<on1o@^kJp*=wv<Gp~8o;93Y#UW(qt~T96=!>SP)sHct
zhE<wdC4z)23Nrp6`HVRo+z2nKF6O&PUMn&m__vuL|JS=7BIN+(xd>ou6^Z$u0G6A9
zhca%=dZ$cpui$nqtw-YxUeID)PxP`Ozs31Tw%+Kq5fHXnb;T@sOR`>g%*yZ&HD$I9
z<?LTA`PUvK$~`-xdmxGwB0S|zJ^)-BR@kj*4#VzcBnmDD>dvq-w4!qJ-Nh}O0C}S_
zsY$uzPR1w=+mjCJzH8&$6Y}d4ccnI-@bU6IM1NRq*2l};oZlTUJ)%h0Y{l@7)XTV8
zHUs{`PVDB}j!}8%>WORk#Tg&~HC3gQS^D&79+BCHA#Vk6k=Q!Il+sx@g%Y!F)vVlC
z8dq-+j_f9)0?X<ECPQakDRK1#X#U&X&@Z5A01M;?U$-RS(u%tErFQ@@5i4;DV8Xdu
z3ZRSp?@SKuIJcZ8Y9ZcIB_I-FtGw2Lty<K#Y|&<Ft6EaYJ*xEQq{K2<#3||Thynqa
zFYFBi57O!WB;uD01x5Z2%Unc<O|xB|4Gyv|N{>si${+sd?F`un?5wHyH`U5-iEkRM
z&B3jUo~)_zx4yP{cc}3{`<)qa9wzQ=)T>4sF12xP;+T5%x+=W%lh1pp@Z3C!-Cbn7
z?z(f;*S7lFnB`X8o}t$`H!Ld(=3fa5xBm6*65}5~KU}qJdAX;yv=UU00PI6-BLbbh
zWOn<igbn(b5)C!)e{c~5g|r7%z{j@d$AA4;RMjBmd7P(}0uYk{ZLXuAw3Y~LDFN$d
z|8M`$47>kZ=!;MgJhr4+2+RLx)<>}aaD042cREHS<|KV1Al3pRA$>m#@TbcE7^VK#
z%7iQQ>GyxyWgokuL)O5?IPuSw2ozf1gSqu1;Xi=^GM&wG`)(vq#FriCwSkecT^0`&
zH_~G=wv?91X^uYMct%pG2*jU4hn+f!qD_6ZGrSS%*RB{<cusNDd1@&xx9jnj>bwzu
zMkI~ib}Db|zrxj8d<z-^*evvIg>3wIsVQfljdZ^xY1D6%dXmTUdPip7>)-gd7~|9s
z(mF4s{i8bm<iLa#-zBmxTm+vYG1x1ze5Mv{Z}Y~$=JM+93(|V;QJRGlq4Wfhk^z57
zoW*}((>z81{QW<3V$6&7gFWgO99f$xQqsBT;0^K=oPZF=0i1b>uE(0V&8!m@DuYU0
zTJh-vut(ADKRBI(&^3`7xv=7}C~y$)=b*n<BPnGP%XvqTD+2zT-{SA5Ha{NCKeX5L
zw+|++>MfaV0XW@2S??DJF*z;hxu4C%#`HQcz#-5lBOE@YPKHtuuJj)1kU2PQyjS!+
z3m@c*b<XGMUGjXlZs}G(R824VM-?3gIf}}HMo6B|?O%X`<^$B5Oc&fQ9LZ8e*2RUW
zm$8fb&m1^<SkU*M{KDsi3W&>fn>Rqf`yO#jaGAN&i_p7`flXP<G+|ZYvctvUct_Qm
zD0dzf)R&<FSHJei_;>@cAvpfX!rYu0u}LC8D~Y&967ova6JqYPHA@vDP)U{B>54!;
zf(`aX+`TEG{*JG;N!(2sP7L|GilxQoSE8qDlnOV*2);rN=*m&dT2|Rb(kk(&`+ZX~
zle#J!;wZgFp(x^W#9VQ*<AzmBsqOM#)f2e`WqjzC_;$rZ(IfGh&Yy8-!s!mgKRcj9
zBQO=2-=pW!+iiVTdhcf*45W1aEqe9sT7*d8H5Y)4abnn|1O|@(qf9|ZQy}KXo|FVi
z_rx9k0R<x9dm|nZOZT%C<ZB;n#_(``H8mZk&#Wv+zVa_-(W~SX3_EGR<*!H_S=9Ka
zl|e=|H(lz0$HP<NZcL3kiE91tQEF_b3V$q!a!$Mb*cR2D{d_3cy>B>1esPOc`qD|c
zGl$IQr~ntXR@Od6>Sd+6kbsg&g>k@YS8_h{O8bDIH)HS!Kgytp256r|W4;~Ox%wUb
zucpZB;_x{9aSX=f9kzoMoKTh&dZx#Tw#g>arj4D$rQer0YFl<@TD2cT;kp0ely8uF
zhn0fF$YevLZS!lYf7>w5v&Zli+kB4eQ}xrqwYlx2Sv+r|LpcZGA1yUBBAjWM^fYb^
zqR$6Vs@}Jnrz3+k_cYH14;@Ew>lS$t;zm$&ieHh<k1ApvDdHO8eS2}(8-t7I^SkUm
zMOrDyj1Ws}zvE&}-Z{fKkGaBz0JftEaI1?Xc>28{Lt_%xVv&>VuT`>;<u=*OLw@)O
z=IqbY%|w%{4)NE~Xu;*eoy*YIJHYDdBTgXx4hx?u>7Uomjg`mPz*8Q>Rq3@tJ;;pl
zb)$LXGR)$NTm$rUayE*6h=)z%Ge<>0^bt`@euS}#EynSKhAR}t*QV87a>PV3m7^@Y
zSV;yWyw0`{V0NO;;h$YtazFr`gar?Ef0p5SoOa!}q{{auCV3wg;_qp2_(!CZO%4h#
zQEXu;n0nr$*2kpr6j+b0KGwq}{p;miZE05oH9LDJH#8VE)bE8`{vqJL6VgtF4W6M6
zxQXe?F}N?XvD|6mkUL}@5!DL8(I#dK%++a!CYfP@nS60Y=*k)QXF&jN?Z$tm)TOpF
zO*Bc;OiWm{zT~USRiMP$JVM7Lei2nH90@X0TXc`wjw}vokMt1m7(sYX<hKCBI(yFI
z#E+hxtKI>G?^S7_Co6)~(G4Xb!C>*z6Fc%a25)HoY(HIv<GUa1YcwL84FZ40GOY=n
zUn0H`JWJQDf?ni+tmAAqN+(B8`hMZ)3(Ptt&YGX`-+|deN~Y~xGMK6FKt1740bVP7
z1?*?hBC;|K4!eqHSp(dFV-q&V`-en$67x%iADdCue!+NQjUhQ3E1tcIO6npxM<{Go
z_rJnx>VJJ?W{?yKD+V~50xOH&n{ZXf_du6M)$~s5cl(^=*Wbr@VT83-o-mrX_#C+T
zqb_(L&)SR=X>^TL1QG@(^bcQt4^T3mNU4fT{UAGeV8N8uc9{Ov<Cq3kunFG$4iW|j
z@AgWC2?RT)lgG$m#_ytKq)E<6RsCguEXGGLs^uK}Vf4~fqwKfO4L|=#QrOa4xJtSE
zmVjSYhBhP57ky@AH}$=Ja3N7^OYub&$b_n|^0>LF_b84!H*?HTelZR7v0{aqUmd?>
zHg!DUw0!)#bF`ix(Um1#Udd=PVLlH^XF6Q%v5VqxU~SU*939q`DDko~z#gCw6o(->
zwRiB496ItyC{oV|ybp43rmzFjDaBkOLtChtv>uVpRkaw9MIz`9U6asWF;9eDGr~8P
zp0d*=src86$H=n07>?^_+v%s)a<p?^!c?x^-GsyrD-ZwbWHp^NM7UPW$lm&2@$nkV
z@6*&WFKuWpOKI;e7dkr&vY_g-jB)ax3G3`f&-n=4Y{ZjTok*tJ&S~qwXGTBQuy2)$
zw<sw$f$7hEUH)eBRsMXG7x7{nCh*FQNduy$kmd4K13w8n6V417b`~iRKS<5rkpnsJ
zh0%kVfK07~_Vxm4@jNEF?z<FWX^?mt{x$zRqly`1o`Qq-Z6?ou6{t~QP*t`Ngy_Y%
zr2s!@vU46eveHL2d<5}I*d0iRkK@FDT>m_z57Q*O9>~@-Yzh_L9T~32G?{RSEf8~E
zW76u-r=RhKsBk>IUX;=f>he)wWw_H$p8$(z-Ha^&0scBr^m_4=(SHBreDX4Aqa&I(
zvu7qzU}WmGee~q+{Hyx?lUNd$A?`Ik2ZHq>2QnVp35@nB4$|{!8np0P2ILi1*&uDw
zG&@7{%jFyiDNJ$V56&!!?lyLCo$`fi=1bvW9{dJN;GN-L13n2+Q668~q%c4*S4{eU
z>!}$IWQ^Eztr{+oEp=9Q6u0wZcpkBzRhZS(FO}Q@)E)e?vAl&QRH@%@oyl?KT{|Hz
zqW>gxVDwUkD4!TGY%qsWew(yVzy1%Pdhj9`g%&YLOOL_vjO_j5JpAR}tUlR)pKNCF
zB!%y#TVP5&5+a_?N>J$>KfeZNH}WujAyeNRkNf4`E5WYjJ*4;?nx`q>+Ej<)a9BZn
zg+DJ@PKdA75`O@9ko=xImn>d7+U@E@;2ETc-M*&FnXdPa`{`5blbtPy!F%ks<aoXj
z|MYY6;H~4~EGG)EHFVzoYiu<6yOjukl)%?re;F=c&!`t!xvVITr|IL_uq4TaIq29C
z5y@vLdF$!@nqUQHQ1c=XGlc_M%`M7t_bm&-h2*q!l^iaG7MFtTeDfd-pTFolbiQ^L
zHO5cEZQpeZlpb5h7T@UN={MK=A`se>0uH5-Pwqni3zt7|fSfIMj6ZIG?;V~H&o-+j
zU4rKS0zI06b0kC*pnei}g1E`{>lNmxps7uH^n<AzA#*!xLS&jJYmp}j1TCY~{Vsns
zA|8y*@pP&SSGn}KwZ_Bf4x7t$Z+(6&YI~#uUpNp$iFepp9$fu_O;NgOspi9ZV}{V0
z`5!+C8#PH2M5EcKDdJ4g>WA@KVWgxx0o^24E0Pjj-jC@??7k;kleLCO`{QM}ngBl*
z(>z0!yjq3#0Qz(wqx1~dVb_9VGZvm>()l;0VCLUKadSLwSCL6aS95RWN=G=bctQ+b
zfyv4(JuTpdlUtF$m3qVz5AXuv@$Yxzcu{~^nD$5)WgNm?h}w!IiFp8p4MZ^d579~)
zV0SJyXC5jE*+g?(GffM|lla-js7E3zPQyi?lWWJx;*bCj3)}B^D7sfst>k)hd%B$w
z^3GQ+wANR%>}{J+t?@<+Md9Fnwg&?t&*P<WYy6J?8bM$EWbl~hhX<@F0)RYU5${O(
z$pL)<Rh2JGiIRG)ZX|v@%kyg;4@Z>=Wi<bVk5`4>u4lzdfLZ5Ga-VeLUPFbdKvg5{
zwsS=R-0DqKs9Y^#r3z<>GK{<0-<wRp0$B@KAnP&vf#d)GOhJ<M0FfA_wq~AAx;T8M
zh+xjRv)8fAkL{FF8~rK|z+&u7&QC@%wHHx#HDZu-QR9Iun6($MuV6LQa5G4;VO-G@
z<sY>i6*|;RKRFXx^AgQfjnT_mje?s}?)v*Cq221#t)BBJ8cR(F{$pgdoVUaJ60zQn
zw9R?T_SOwdfTrh^&)73&PwB_rpf0vScU~dgm@?Wb5~nc6Z3zUW#Q6|TSk+r#M%PAZ
zA%VxV2@Uw-_5fuRA-f$Mp$u+D=v_ASWp2s(fqO$3$qkVxW2naS=_aUdWz(E;jIIqF
zMWVaF^syL!FL9wGLiCflKn&vysug~(PhdG`I`RQO1W<T19-K6Umx&J&yFxE}oEUj9
z3;{<4C8j~5y>#M-OeZ2Q=-8WDi?q($N#FJKf#;JSJ&LVD0)mEU22}UAG^P^fA_`LU
zVsph<(J8&v(TPF`xc2sDomiXt^7VjBC$jk;g$vxw0%Yw0+O2{Wq_igO42{E7wOATR
zwl)yG^zv5Wqj1pWqkX$bOf{+{GR)s5<h(@2h^){FbYa`9#VJB>NvOm3{v<{c3-cmA
zM;PtnGa{Yx>o}}JOM!Noq%3F$y6E*2H%Sd3e5+H|L9e0N$U7ZorkEQLb}yNZt^x&i
zhiA;vQom%<WuB$Bh|eH0PGOf!r+id4RA7M!t&t0N{1-Xae1bVr_Pe<9#o!{<5KRVA
zfr60WG~jvZrvi7Np)!F83w5Pcbs%({4VXakH142y8Dx#D*!B!C^y;y+oqleZaF}Py
z3e1yFtw*(=_jSN;sAfsLj2D{hE>aEFESEWkk@$cvh%8BDWMfaGEtH!j!7(JSx2!FZ
zVS0K*wwc7salV39mUC2Y<BYIo52TUlu9fhu4Q(bo$wRrfK5-9wczj%*JtGB$L2+|D
zqtPF9?7h5S)w~yC{!Fdm{}CDEn}3gHNV^^Vi*`uqi?IQ{$c!fmu;BTtu==0-(=pwm
zFAaD@gZnvrZ!JcIk2KGJKAmN_c`;8n-&r~%>t1tt4q{qztNw5+qdx9bDsTQ3ylx=X
zP$5C^fMZ>Vqs0ZU6c{Hf(jm^pDks9kWZBx;WDG*@-aCSl7gGpC3rj5D($uj1(IRu%
zB-wS9>2XhfmmV>9&81jzbk)}J^ohYF!6d=Te@?CSs9A<i8+L>nIbi954?2>+8Wok$
zUT#78p8exS1nS7W&LsLW=EW-gCC6qkGb5U=U}&)AAr1n+&*s(<M@SxWit<#Mid}T~
z2kD+*kElWE0l)mED4@5;@YdTy1~^?@8}w{nh&4D&j50Y-(2ieG>so95=zxaiK;$mg
z0S*uMAAR@CiEBMHAiD=V2aw#utQu{OPWimrverKPlO@Ar0Z5eL&*>dab7y&sOls>S
z`*UO>xL}0L^chnSIRwwh=E`tUm4;M+N3b~ahY2Pe-PEszAg63nM|444`t|*Xcm{6W
z4>a7YGCjJ)@92Q6@G~Qge+Im3p;%AMdaIL(%tK%0!!SY3d`~n4PZw*;ewa`82D`#P
zt8zUr(U;*amMIT8HWlD$O$nhE2JkiFP^&WqmqOa*W4vqg{0)#R{Zsv&B@l{a<74wt
zG=qb9SFnJ4`#1CdrhogqlwkTaHh@Mt-H&Un!pIx@yk^%^L_5GCPpopThUo=5urkhT
zwobYn*TC!?z;n<W_HVw!wHb%i)p;D%B<}RFY26Zc`a<+-VLF=)S#vh->q-5Kq1meR
zUi>W{2xS)t3wIbyxZwRMCFVHwq&;|BCDCK6d1Z+_cWnq{{j>y^qJVc52jjcG@s_!8
zMw8vClJMT&&6s#BJc$_-r>w2Unsv?AdsOb3%FP{??xWcg;NX&q!$uSZE(1!HznA#x
zRR&Pp^*_(%7;SdHR_yMuYv39e0T@)kc8%rOoPz>nwo7<TM?9Xi-iP1n9mFzJ*AhPE
ziTX*8`9x^H#rn*kxp|l~c7Uxu$1ffX{7~PY`h<hz=7g$?!_6poevc&mca!_0E?KS*
zj5Iw;X#ulD<ZQN|Y2m<iswEUcBC7Axy;G6|(8TVgyj;7X+~=n*GsbAuqChD>9hp7K
z$mpEau-yEWWvmC%PctQYp}Vq<JKw>PNvX$3ck?1&oBI3Eyw>>?^7D$9De1cHK7uJC
z`X~wdHszgecil-wXIfXmRf|izE;{nbT!idd&fo~IG~v~Hx#q+BwNSheYn@?v{GAtF
z*Va$%=s`kyN=n#@<4w|(^@Lw8AgPo@7etiU*vRe!V!=8OWwb1V34Pw+=9k+GQfz3z
zxD}Xl1V)2{ja%U9!njotF$S~!FIR6!@i%g~6M5F(g7z){@hnJesiTk@<^L<AR}8FK
z`Gzb3Iq$|m7EDVI#)emBK9K)2zkDl}O@O)GK;nWdezb*T<@b9&QIIPzMVc%m%OKD2
ztK`E+`EnzRchX}Wv*9iqoV>3707iAWox-yKB&-*o&l+=+l*`~9m!8^3aUavez>MMr
zxgJrgN06+`Wsilt&%PQ{(nry}fn9zDI5vEo*-?E}$7~GxmfO*9R+@hg0zR_o@gtv{
zV{qtC$2+R74!_qQeW5Av4^bHOPoL1c^={F-jp$=&gH@Q=6n~97Lg6>PS6xBQE+P+F
z*m@P>im^wpLLSZP;%|&7v+?SI@sLI>R|H9>4FR&mYh(-K&zIb0{q2={7^&Gktd5(6
zCKG#qa@2tK++4+PEScpbfIXs~WMk)W4V`qAFZZ7~x)6(8Xlh(t7PVy)60moNVuL$*
z=u@qRS2AQj0n?)82`muWVLu1W<|JLqpV}|h)>h@pSK@JxUiYs;AM1C9Gee3ooOtbR
zt7ooFC&AuipW<!9dYCQxQ@}5P+4s2bHixr1ZQvpqwvlIc-dX%r|7qI2ipi01hgcXG
z4w`-joL@o(Wu^0#T!6O#v~W2qWIFo~vZh%!1JeaC7Xv-=Ze6H-1UufwwGHQ$@-&q4
zRupnrc|#{Nj~y7s`w77cyCEp`hHXgoW;?bLb>8Igl0IDoz0QB`iRi<<Xfm^j%U_d$
z!OnGTm>lALB#T;$>h0u<3eb!OqbEpVf3}+SD$K~x-aFpvN0Hz`p@!wkU8q~d_JJLv
zNyqb_$mH>6T}|h)brw#3Q@%-g<Vc}GdWUlkF?kQH+!Q~AcP!}zA^r?`o0gjpS>FZR
zcgai0QL@5h^nEn$C-EZ`FFr<c2Y#wnZEV~z@}NN)H9P~^&uKXfmq!LJ*2?*S&*8V`
zuWYm_oXDrsFP?whHf<Wo{z_<V+13lxa2P1U%Sj!QB2L5piO$s~xebt!!xO2DQa8w^
z$-q%^rFI#2#0RUiZvnLEemTd*7a5OdS4kNJdqI7QNAQ$gL%VBZF$X(CCIx04(6&G-
zabOk9tuVlKQD=6!BYC_==}-*Se+jlk1}3sJfXb}ETfV0E?~FPbu}0IKXdTFKMEc>H
zjpOjuH*9oMy!x2`obWdeb9fK;e(RF{d|^87E3Aw5;+iuGfGzSr7+<@C+wFoJnMERC
zwfS9iHVsJe)p`_w-)&h8VB3RdP2Pp2(*}GuQ8DS05+5}l`1uuZvyO>PsugDTaO5K4
zXGLa~gX8@jYCst*PI|j899oOI2O9#IGzYAkvwi-bki}&bq0O#<w}dpJY778zpwCPp
zU!nVuJCZt*f3p9TbM)%%{_`D)xZ*C#KesEvCmHBd`n%QSyL815PV6L!{v?o*v1sU+
zAN!%C5@N|y$siTsT@t&|_)hP{#z+QXVK4+JvW2cWB!BsDzppALQ9PHp)(I)F7vw~X
zsd%1MR{lCQr9Gpqs=YGBef4_4xWYSjrJ{Of{IjmE-fgEohCSuPdre<XEp<v2RFpEk
z8hx24LH}a@kTM@Hjm<`c?eRe2+)1^za>;Svn={|2dvCjSv0%YRX29RHuI{{@1s4%R
zbNH1n-<lY-N%5ehw5OGFkzk_GUs$RhCH-3D=H&TfPTYyXl6&)mO>{1@>;U^0LWHXQ
zP=#6-Dp&l8?ZnBS5D@2(m>>x4h_Y`~j;d}1vTh6-S}E80A(s~AuW(F0CBm3ZNMosE
zyF@=-3}l_v<UH@@5$+$~y9&E&y+&`|ub%Pes&f_MN~kriJ=LCtp2YG!r6&wravfsw
z^4YyWa6cpo-=RNqn>0za@|nq>=v*f+s-BfXF-3c?v@O$vA~L+8@j0x@=4Tfi#w0E8
z)n2lg%-S}Ki+dahOH1W-dX`kf^#|iT$zpF{&z{IhVJC8;3{>jg(HRc(D>_|zI<bAD
zs<N2zJ?axrHdYvFYE<_%{HYm?$A@SqW^}xmU3`@Be8%73f;U%ubfm&w^xP3RVi4}G
z3rf8+x2yGdt~^UAsAzMU+D_=>d%czaac%nJ-0u3?Y)3YmKY@@1Q4P8=9WJa~oGv<q
zWe&CbzDU`ZxygJfklE`)nzHat<Y$|(lTr3**NiU^5G4AYM43afURG8H>)7&Cug9--
zDQD<Mol_W}H86J+S<G~G)%Y0p_&koGZ9Frbjt{W$Sth)ezKp1Xo=~oOQTWZT;}e=D
zlI_Nq7B&JcXs2>{wL|3h5g$e(`AF+>tXnUvC)l~z3%sv6Tv?W=_?$vrTD@0t=+qYE
zlgYe8u7+(Eo^ScT)N!YD$aAYc`Fx%zfZ{q(njBJ%f#pG-EAtP6#YZ;Zu^JWsDT*S$
zmCu}<_uIEdZEgzs1)YlCIEe68vY^i_<Db$u^XyMQzm>_=!PvnPb62C;G+u)GKCkU%
zj##*w&Ey3uM+nCxLiYBAa@8tI^!@ogh`kK=S#Dc6D3T09;AfXVuRtO`KfNoA)5=yP
z@27OLd)%oId!b4yhQiY4bhR?RRQun{LuZ~>tDn-(UO?|QrQ9&Q$y+ubY(n=M&t5Gm
zOTtGFh<Q5>weLtBP~WrXev0G=Ib0Cbni*PAr{=6(jqZNDfW0|LguH1=VS7>M!TgT6
z7bOhBUaA@Oyod#bj(&fR>+~z#<XpOwAwtB9>tB&FhCHL9gwyp5nX#ig;vLJ(R80xh
zo8uD?!Mi3V?%~K`J~(-P+@%XCJ_-|Iue))L_cUR8T4sD`NR(Ni3X8xCF!$9O3##Xt
z6S8YPyo;JhOx0$Uy0z!x$xvRhCtD?x#65IH7PaT%#}b?P3jy)VyaRj{ip#ajoA915
zh<T~gU(LVS(}8Gj@|-WFPK38)D;`!V%K4%iiJvU&B7mDbvsSpGQb(;P|8k7IDHy@C
zo3^P@z$a;K7)bDnS&88Oumic0lrTQ?dX{P@b;b1cQE}|~Ub*yWp5Y}Zvchair!U?6
zCGxoQstg^iX|pwJCl3FK1p|R8_!N2eofF5`kT|H(GU<ULT_fUw>Mt8kW(1pD5QL(<
z#Igz~ihau7_%~IB?`-}(>+XEx-2d`OTtXb<u3&g4$bw9hRwa)`$ci`vybEYH(i=GT
z0}g|B68413Me2$LY_ze6xZ*c8$Wx}n(IiU_Dj^Pk>M>7<{4Lus_Anqa-QV7dnYpsH
zz1mXt_j#;SX3s8Fky>nhy<Rpcl=aK5XMUjYg57|A{U-aC*5MJ}HCF*1@fG{6`kt?J
z8NzC1N8=U)P8-#0=&uXgi+v_bb+dV4hnT0XyjR6*1EqLPgD-!)Id!#a<$<pq`-olL
zQC?#G6d-8AboaNLBp12oXAarBj~<>{lYYly$1Yw3Q^(YJmos&RTXrNWgN0gO&pW)m
zfAiL|Hu`H$m?98;4~0YKHx1Oc$~sOIHM#md#5u>q#66jrpy)8_M=en4`FB`Ce5Ef>
z`!d~6(H^ySW%}PQc@&6WiZI}X^f#qW#?!&KN^RYtR_mv`3;VE5=fP5Y)hsl{nr_rm
z`Pc2Sy2{5nKC*}jcio2`A<tKM&W;mJ&ln!Q1^MT{%?EYT^Y2ouE&15~@F#fhOjSu?
z$}TdzT(4DGw7Rt36+?WddJQZuGe+XS)UexW(GATYEAgsW@2KMsp6@jo<c2N#O@pG{
zdlIMIsb10@+FbYP(!+VWB0)9w1A+AUjFDoW8k6LNcCCP`)AP~GSVrA)|4iRje0dT{
z@!U8Dk7Lf#smdz^u1C~w{sgO-{S>31aAD7MXq};2EwV9_HUllG#2e2}wt|vMin~Je
z4HoM#3L{ggRx58Y?xhQ{&E^voNC*K8%45i;`NO3%E3FQ%X?HaAF{c_>4^PQtt5Ez&
z<G;F%8R8!w-isbhPJO)I)?Bls4|8}dItv-FJhXUrB7Cl5KJaQRb^i41W#jJWQkKGO
zl+l~(=_#2WNWlMDM0U+v*Li9%Y~jG$Tk8ZHOa4Wu#A*%CW?Y@3yUt>#0$-Rr?`M}d
z)e^_J$P}!n-HA2DSdo7a=sPY}jWv71`r;W!5jkan$HzEksh8~mE8v~)izFT@(`p)j
z+=|;;rPDN3TG_9w*6{dEC$vO4!9RqGA7lZgJr*gdBW;>>(ul?28`-*V@qtZgsZ;DC
z;AS@2Gm;E9AT=jAdkYc*!oD5+>Su>Mq<ouBuv<7gnVOILJ8-jl)JGJj7P&NEqpxng
z6T1-9o3gjvY^v)S%ykgAIO51|br;<*nO^2mfIksqxJO8tJ~i%Iz9vy0|55;9ZkY3r
z`dIfLedEh>4Z9rr*u~9>EK6ejzzg;EHd;%Nw&Orw4;x){wd@OqzRJ~0S7MOw>SONu
zWT0|Hc?Zrklld3k{-XLa!>N4S*=lRHg!Or=Pl`h(f+1302FtY4QW;;1TRj)jV|7OA
zpFmsv!e}4bkb|>}kIf|0N5Apk2kuRg)qJPrwjwA)aO4%o(lx0fM={4`Er4e}Gmaga
zQz&rWc8LC0v(cC%w=>A0Av(kzIFX)_2!I7wARr)Q=HEGz|H*ys?xX+6uJEZK{l)8o
z;Ah3Z?@up}Uj6A$2W*vAzK3g9CyzTie2ov8>K3aHD5kXOlzj!n=f;(ukAr8G&G#G!
zJd?Wb#>35I57(T6R#g8KpWXSG`juIiuX(T?t^*suSn=7m-PkN&KIOjoYtG9qvh#^*
zJ<H1^+0gG~8^gzAWli~vvYPq?H?I#2-#!1iRH%ALmOTDQ7{`d0xUeJhn6_0hovPeC
zXvQVrE|UFPVzOdM>*HK%VH(WEha@I4I2nIaG;ldfv+;c6m(a}#vN0ZLp*&;%N4H)R
z-?{eJ`rS|j_)$q4i*+?1V}AU_50fyun{%Na-e2bGG~{)+UuRS2snlCfWVukhnN-|8
zbWimM+x7(#LW~f6iEzK$cwTU-RrGmZ=g#xL<mSx(_H*lTUY_V1MORyk{VM-ha^Vli
zg5Nml^m8_sgDh-rW`i2G9wI-C-Fm%}RuZ6cKC}c;%qGq?K62lN1w~!JKAfKMw3oG9
z0vGWzC=%UbppTIM6`m<XZs=0$I=Or>CnlfoS;#M;HY*jVGA6W;*49yYOReO)T{JgO
zyznZ*=)(ld2lri6muv?wSaJNg%4By29b{(T9;0v3>+?6zMdar9UvRpHR$U%%cjTrz
zyl=jNV0(0fSh?fM`JWxoq_vF7AA1Dn`n|s|4Od<DZu}A0bLhd;c0d|&AX1>w(zfX<
z3}@z2s_DnE4EnmbMQ)!91t9?Uwt;5eESEPPA07LhafVra*VX)D+<UB3cfNJiQ#T4T
zB!?Z_=m_IH>(40*rCYPj<-9a<8%#6mS$Wzu$>4q_{Gn7?{%B_E$Is_PD5^C3-!7v<
zTbX@3i6wgVrCQ{XMH4@sdolpLrM&yrQH|Aj_dV69E=e6b?=?9SdWLfk51o<2Y4%W$
z2rs(DRG2Q?w3kRG%^7w&h;t`~J(B-LWoyvwJAa!Sf!FZGOKfy@rV~D2?}nispSi7(
z?CAR(9#MtuDAlR5XNpCwuIyPW8jQ<p{!x?CVBM#-SLTK$ogl7{o=Hm%lf`jr3u|__
ze)bkMcKq1%)H7R*#^it6dG?0L>OZL@n<xy|meac$y<o5_mx%qC``2W|AGeXfTE!86
zwH#E3b7Hvf9|<fxcfJVjzfc7*FHC^1c(SGaRbIcZWsjCAL^gKVzpSL~32Dzr=cAlO
zzxUHJTCGfyuJZ(n<rSJY8D$@W6+@eI{GCzz%Fj~6$A2LC`5v`Ya-A#EYP6^4()})M
zYa?PFW5sH;#0)x`Zqqfilr}}LwSPYcN*AH|R-$Gpyvup1D#Kn}qvu;ep(zUkeQAB)
zysy@_Q4HT*V86dDclW1zyl!KmJUS?P_>}Q#de~>)J6vD=b&OBaa!TY?l8V;2zVt!I
zP$`{qjos=Az4<A_`Uh2pt=HDwruSWNe`(H@>xS!GuoDQ|YrFb5zua3*W>WdZs@^Lj
zt$802{pa1<ux&;~WNQ94u5zmI6`FLom4~mO&$1QH=Nx=77z070-!1eL31hue78<p2
zCQwo-=IT>TW}-q7r->U&uT6^9tiVRFBRg<1u6Gn^SyX%+MD*&qxm#Zr%75d$Wfe{C
zH2syRnv!vhv--|0hA_E++^+IFpR|c&jnTLZ?sKsWJ}0+9O4)B4-ubG;YWJOirHL5W
zA}g^gU%UDrdXim0QRvu_-TG1fYVU8Yc5QJW*7^_|N6N}v#yw}9muX(&SY=dfiuT6(
z0o5SgSC$RWe*H5&@s`{7C(ImNOLZ?&TD-ctQsO~}8SWc@k9)9xhUj8M!LhHArK?)2
zd%-ntG;z;g|NIxA?fzV_qsUs!r^Tpn)WuXgZ~;eLN=GyG{$8eWqsgWUzjgT07UHZc
zks(Ul&>%y@v1`Exli|bCT&a2|$o+@~m+YP4MGf?5&rq4k<M9>Mab0uSd(7)ZUFLqG
z*UCfK8d(Sab9yH9HD(RsV#1p{L-{HsKBv;8JNfE0&9(L;Q1vs&-H%5JJCx>_;c1$6
zk@fHZ8-C8?!lQ049dif0EKi=Auq4c%BIn~h?57l94gNPBW2iv#l>%#hy5KlhL<HpX
zP3$=o26yLkgGb1>d|>yo{(_a%nnFKTPa=DmCGIW5Q{M9U{i4KZrG}EU@4dubr(EMs
zUp2Sa*u(1!S)Ie4wQKHfb;frR$`gEhP&2j0ei+TE!P=em5wBw5$(YyWhsFXi%tsRj
zZTegDedD@4&uf>)zIpG{2OGAhzSlT+b8o&E){<KzZ~EKG`Ih+XtEOl6vB8!<kJfD^
zhEx{5i6^A>%s7#{m`Qt2JIN(ue3?LY0X#G9TIWq-N(hS|S2AhkuWN1A#0v@~X7eN`
z5k3c0?V=rZiZ8;NXm!W9@e$CtJ>o2zbQtkj)c0a}<bqh*X1%bpBQ=x%^p>%4O$pG_
zDA%`*sTr{mR)&A1f4~GOaLCMK2T?Z59yi{K*u`_nhVIoq;NMmic!7{AX=Ob(QxnoT
zw=3Q)Jo4Sq*R3s?AB{}KpF4{3b2@Fnbyp*5<F?%g$yW@NuV|L2*^t!&Q>i7jX!X+x
zX_Cw8?3c+^_vnXRSgS!T-QTaJD#uDJ${Hl6^HauAS1(=dO0d8W#RbA+lY3iRbm{%+
zJIkp+;t<b=JVe?2v)j71AD&aUs;fV~1<%+Z_B7b&-gwq|KK?&+y=7Qbf7JJje-))f
z=|++6?ocU-p%G~*X^?JE5RmTfZieoV8oIkBhlZhZ=IqhuzOU;%=Q?lZ4KLXH7c0K&
zv(~oliGy9}qsy-%hONM@`or#S%Z*X(FF&I9=o^mVUx=sFpWJ6uLo*OLJG18xtKFz@
zXPZ+CL_FL~XciAmMvCbaOSWzutxkXihT5O>CKzQ=RN2>B0|!?3)Y0(0lmJJ$em1Uo
zfKzBE-uo!=0G8=rUrIth1p^d?$NE$a6~pLDVa2(S{?l6#F>0-P`0bzyJeWxup`N~Q
zi<m>^xko&@h*49^nK3YfbEha`k#_6n10QRQc3Q7l=gM`B#eWW7pIxgYz2yoyM>7-c
zJSyL9v?OxUHUueiRb~P%y-C?PhEq5vz|GWxxmhiwUS1^t>Dsv%DbYBCBVNJ?WEPZ$
zF(7#+nUWemrG#k@vRc^<u$3s}=fr8gbW+HUU-yxEqnmYcvq`*|{CQm|?X4zn!PdHC
zba3J_ir1U+#*^uj6raKB7!IY}CmAf|7Bi))pl8}W4lJ;M{-_G$fb0N+c@hdM{85^9
zEOg_5?*FXrBrw}bX{f5OsZ9dt|HB-hZSYL`cZFnM5eXRU1pHm2O0|-IYt(P|T$Ju0
z3|y`lvmyTK0vpjnRT&PiJWlT!5e@TxQ)bhB{PRJ-PsHa|%>C6OTtBhGmff4K7;Wf6
zA)Zcqv;HUJ8D<EvKZ7skVEqZFrXuGa5f3hg?5cXn`t(J2o%Y|%c)`8GWVFGNdW>Hh
z2UwGVu~^TXiKy2h<<F_w;*A&8n1(T=B!e)y!vhuk01tMWI22=u<$<z;8D_f`usT_5
zkW5Gw2nYoahE?99t#m;b`*E1f%9?c#wn5;|KjgPcQuP=XIjLMLkcg5hb3Ztdi-5?O
z#U|ApV^^dBs30F-MiR4Q0-J(NR(@s@{}2*$@e}gOi(h5=kMr<CU=JI9beIGsqb8V;
zdcmq6q>&E0p<nNYB@>ubi(~zLIp*6veOMUb%m-rVa8X#D_q)80lvd5Ej-KN!7y5vG
z_aZi1q(niX#{|+I6z1jg+T&l)4JhS!6}No(RYP6BKSbkv*?k;J-;w!_xS&`??s+%=
zRs*$NxcRnQ<?r)l$8s;4qQ(IKv;__7@izrpH-}5LDP{LFT8*<)8BM7PKW>56hx}W~
zM-Q9^TYJWoOkH-Ry1my+xlgtQ+6fwi#tt4^ls`C)IDB+8AEc)v*qjQNi>67aoT~z{
z8O+(BAAAmi7FHDyqM$7N2jr+I1V{RJ&pj^dg}SxQq}#93P77__R;wbS>+hp%Xg!wl
zIb3Z&DOY4>#W_HPYuBx-a{eeo88S>mi;4oA(vl0PC9_q_JdbWyP%jUBCa3p79&Ovx
z&Kk2Rh1i;0=K$Aq>GHsIT+lGk_8&9kzi617EZUGmImVsfnqL8`)ZnZ~c?ChNzexS3
z``vLf>jo1r@THHq`kb7vnJ6gi!NA2p5tQC|s9Ga6wN4a8T*lR=%u&FPG_6KkX@d@P
zgWK%t4?mS(Gp0UX3FK^2mL+`=My-Nr(GZ#!3Y%qEJQZM-{IVHom(;{4zaZrFK5A2E
zF#U4x_KJJ}5<$vzg|0$rdfzdwe(A&GwYm^4p)rs6@({Vi6jbbUj^gkM%1pyP>Q;Y$
zJBCvx&r@pp*RCJ?%W1Q2*jp6*#8v))Tgy9OMlY9(UAN32*v9Q9l;6+C%Ji7Y!y5&q
z2^V<;Fhy>vt*y@9D*ieIg7cT7b}YwE)~nY9<hFU}#|!vg^sU;WB+RyCTd5R^Wep1M
z^pltt#C!=9X3P7|iVL^}tV+>T<3;gll)?>~K$>1DO-6mS@Gd?43JyA?-B3Os^iONn
zo?W;qhp*AE-Ma{vI259d+*S5*X@ZSEkJ8J(GM-(gwT)S9+uOmP@vW+em<bl6V?igC
z)_Z!{QRf)$nltrZAUX=lG;rNeP{b!!xcx=0yHdK17Dx;Fa=wO(Zoj<z-KB3@zwG2M
z98GiFe9G8T!nY$%otsD@q*m5hq-vLR{V|tlirs>iU~`P8W6?PL&Bpdbuj%rBC&XNN
zwh<#(LQrigvO%cr?)#Dj<JO-0;~HvgU2{swcAZQbPZB82?XQqlcvU*gk$V~ueh{l2
znX+e;orhLeoE7O(C3^Q=YsGuf{-FuVDVmdS_?~r{>CPjbp}^*bQcg}X=QQdyVwoeh
zeyz@l3<X6{r=rldXY$_7!dm07IP2-P!jqA&R~M$<;boLsyIT&-o13(bB@ol^YX!c%
z@@Ky9=f6H*<@{wkj$=EW=({bsA=>|L!?lQJ<+T7#bm+_`X{_~n$0wK$)k@~OufM2%
zH|c*xV^I#m^COWOruPj!HPmUhVgj~IlZOyPVRd+$eMUUuHqlkjW0X*Zq^8hv)&IGq
z{ejZ;hqitbEo?YYpQpb{-uYUso>)OA=4Mao;d-XHr8jIz+Zs%Mccf_#{(9#lGT2ss
z8E6mQW;5agKP(?X=OIvp_)*az&n3RqHWNbS=;kc!cxe<4jbQ6*O;!z8ET3Fu@J%>=
zQ!j6}62G0kSj`mqYeY2D6tDPsm07m`;AS0XPiDBXD$-La3s#c<^;F8Zm9GbFrq@}b
zYqHANYJf;2;$I|!g<EUzFA@pRHGc`KrC!X}GS_2glr_$@6Lu{A9+Q&yTZh__E5=O)
z@Lp?x8_TowBp~t@M0zjdSnD?Cdd_uR5?Dac0{bw=r-wD<0?M@**i$nnWfdBHt_uMC
zmK=)!>Qhc}yhjQToN^XHvj67<8OP=*&C7U5emRJ=D-s;6tdpO^wQe#|EEUo#kckaE
zQ<!r}aRz>{ZeDP?Chh)g-DWXgaVcJ*+Yd4T<*-Xe8r(dY708fCeou}(D^a>Rjw&;3
zuS1iAo(MRW#2ZBwVp;=1xv7pIj#=v4AlDFT>%#4)8xx#k*-Rhain7a1qXMsb6!F&s
z1^kZ(k^xTE=zVKRpLP-G*j6jKtkZ*eC2(n(oj#H{`}wzBjibt!lpD(54pZv;$;BaV
z!N2F;+_5$|Yfa=&Vi44(HD{UkR=Isw`Rt{T?}-D2FVz7X^eLZo#__P(A>AljE?GHz
z|DC-GZ_0QAC*<vf!^{cQL|~m0oo$yoe*_B3SP`(uC`yUJZBc-|TMW$Fc}MJ3yc0ND
zW5lBDA10~Xa0nf)cno<s7nX`jDL_d`wow++;Ij{Zlxg-UBb%Y4QEb}nhcHiUW5Cm}
z#o@{K*r%c6pK?oYcFoUj#u;+72Y44eMXgf6JAvR%^2aFO(E;5G#W5Xnr{BfbiQGk0
z6YUR}&ErarE%=l@SQZi=f{9;%lbb7+`KRz6u|AvvQG0hGdBO<2Yb)bqMZ8^1@bA7i
z8=Ov59fBq57EO}Y=uStn<;LF)Kl^CqKY_x}x1Taw*8F60z``J@FL};A5(WcwAEn;H
zBA{G#)v{%b$7#`<$MuRz#1zXvKG-Bw{6s7qb#X#qJxrzceKKL_-P^}ufWnq3pIGq>
zrLd;uI9sF@qn<?-l7dh$?+ViFU6}Ej4Orwbw`Fckce12<V@jzy$zloJL#Kg~VSLA)
z$p~OTYZS|Kdo)Ngzt$*(pHm23kVukSbN#Ynl_Hp!7iyzkiDm0Gpj{Xm^B4tX858L(
z{fl=<-AioZxC1_N^%HoVXXTP0gVBU?w|)+KL%`LV3OgM7s_QZHd-0LdY_?MI@aS?Y
zEpi(Wr^hl8|4zGr+DLs^Q8c-AvDSS1D&6bdXw@O=tS}f;&|qrxbL8OM|Gv2$+_L82
zhTgV2J8R<BxTFfXE5XQJdB-?RYT<V^&+?}3crYZKO}rdeDUvar&h)DvgKb}BybkGQ
zUG9Gzr6$wHxBO}D$3_#PwLeWBQkNgm$6Y`zW&dl9y*gW6TXmp%d-pxvuFy-YR;N9{
z?Z6l7?CauG2Q%w(@sJV~dC~fA?V&2*9@MNOt$~D<eF5o=(d%yl#SHkE+x?ckW(D1H
zpM+gi2H2W0J+$%JLWQx)51}8&AMJ|@V0M};B?*W^I+5$LKJmhC#6eyS(2491nS!<i
z<4$2A!BaT#7Ys|{2`K0r=K{&me>ToqBm3PkAdSKAW}OG&?$%Q}%zb$wEB2h|jWrwH
zRc{h1@rJd}+%RfygmXELk}*q9lmI8I`+wPcl^6fFz5lAge<WgBY3hAJ+E@T&Lj{!#
zs``T~R~;^u4=cPN3Kl}p%<8vga;saRmCo{3_~fY>8c7CLKnB6TgAat>az{%KX?OEL
z8Vo)p61+`xsp>(vCNE8XJdD`xnM!TgCJIGH-4Y+7$AdlnqRU);trh;qW7O$)pA70z
zG2Xq41`P1)95R1l{gJq$fQ!%+^(T-Q#0kN{VRvIWqO|F=D`!t(R@Bu7gXx+-0}*%G
zOaI}vFFpt_Uf=uliPW}2+I)vSxAX@@UY4%%pfbD8Q`l>F^fr%m((YsJlkkCoTcm!%
zYhVl9A_k<~q>Q%C$!a1iMV|B3@O_;wE-YSV;AF;+#RIbS_6I~D`d9tOiGchmb|6Km
z+6jNXZYfg-ss)lxwb(wu2kxLgqT`eM(fD!aEA3fs15nQF5`pK?^~4<(__(5JH(hH1
z$u;W@dg5e@BD?F|7>>&MV%XHJBwzVjtEOZJXTbJ^)YtHWN}}zC*vrn^HDNI>dR4Cs
z`hTXKo>GiSIcSPA-r9KPF@4Y^)KJzJJZc^2Qy+wBg4Wdp8D{z~%mH7gU2n^L@>4F$
z##i0jf}wu;%`);)9f%>dN=~R9eIB>fVgVUUs%YW$V!$JmFWjB@KyIvKpr3k+e<@8V
z6i9xt;xdo}^11G!oL8jq2nG)A!@>OO(aYW!V)g{u?yw}oW@hW=&U9|kx0qr)H9&TO
zIWY6V__dGwrHjy9UD@c=W!~z;kwD!+CtL6#*Qh`c$YTSqQde=&;W)?&s6|@{nXBC8
zxN-2%h5z5l7;XBOl$c+9e_ceyAz{(2`eir0V#mc`6X!6p1{3^e?oElkll_u{#+-B>
z@6lLwD4q^5=1XHv`<r{goV=`#!SPla9oxDRT~v0df$|)`+AFhhc?>X6{;my84tbh)
zyw?~CN{UwQElm0xX+0~^ViL{Sa`9y_-R|aNrd>GoU)RpS6FMO6%iQ#qd;RhLUq}AW
zyV?9;I(RhT2mCa3Z+cx4#H_)3-vqY$u?RbeYFF-2RVp6op1o{Fv{43I)2p7fLef1z
zhLhN?4ueMR^*nS&W>uo_+gJv09QbSDx^{BB=*>=MwiNM(v-w-D-pS9HtNP8c3ha#Q
z<~q3tGH(p0bE){9`-*kai0W0pmQ3Niuy)-e_|8!<+89dBmeJMnOFT^?xGF%>U_(hV
zQ<95i0N!-4F#3E0ht<e|IAJhR$KI+jwC7_k@E<DFwK9tNT~Sa9sgO$fqvY3Z_aV?q
zm`5n?on@O>yDKVbGT$8PakKUZtJMMzFa6`9#92{ui3R$60X-g<HBN<}PZN%)?HPIV
zTVlI7$a1EOl%Djy(2Of_kX6YPEUG0V87M`Ij5&jTmW{3-)z<6IQlOI5?8>gNE2J9B
zP8xs1`#J%pew4a-5c%;+tPpTdsa&?!t_P*oMIT#~i+TGth)_^6Cjg&4MgxhUoNX8o
z-IuCW#%l(Qr@1f%0v2-`r~IJ}yxizezaz#X9U}_~u@md;Wu_cOtVPrkE)-|-?M78`
zk051uSL$Q8KQn=*={GFkdP!050Scdc=qbLCqUa3<Z;e|>VS>`wu6dMMSLXQ8OUdt(
zpNFDxl;@vChbF5$A>LwW&>NPC1*}bJvije5Wcpz7Y)3MXLNNc4nShUi3AOv3@RmQB
z1&vB94>LMxAme{5$ufnT(-)PQ=gWB8QCI`9p@LGw-M(Z^U&|_ZJ@M)am3Qs%x*IGx
zD>}s=p?oJPHNf`oBDw4-0vq`3?NWXzV7>n?T<kJzf=n7bu@|AWKAkp*qv_Mag?y~8
zgF5Y*3ebMwR#d*oRXTP2qQJ%ZRY=K|`riIbLF1SCUBib5ptzBmD9=a!tkJCBY5i+c
ztfJFi;fv|F*fwpF@?-j!H>6tekF?xjy>n~QLpV2tED!dFo}N%f*g}lhGQ>s?X6<}@
zMS*kYbUup)2$Zu{Ye{s!^u24?uS+chPFN)Gxkm2;$bsQ@hIh$ZguhI_k+=$nZ8S7O
zE7I7G3^NeW>Nx;5h-9<q<4wPxXPYacEMDgO69D!J2X6oV-12H;hVK15xWemlK4rP?
z)*iON)qcyO4U3yTSn=Qs=FQHRUk)5jym5BEU8SV!EvN7s$PW|yTEBYfRkxc(IrtXj
zal{ivZr=yD`eVD=d%5J0Ell#t+>iE}4SugNUuQjBizq^1D2T+9JDqo&J~y2M)1Fmw
zQ~woq=mBAe=7hSdCB8x;T<#S)GUWX!tiO(QDgQQkK33?wF9Og6E!+BV%LKC%G$E~N
zClQ~!R#0auBKj^A*o6dK-K?-Y2*YEhgP2W-Llz)e!S6!8;xIPjDKN8FS!#6hH_7~e
z4GUFwVbI4&WzW#Q%8zOR`J1E(;ZL+2rI9HDGECThnv*SoO42LN*FkcLvW5H^TYn^A
z4#GVWbd=-)1^BGzB-8=AORlq*40+R!L!c;ZLN_9$A@8n%53<jIuhFo?TmTuU<bBn9
zfA1u_TVI;6eqg4l_iyd)26LPg?;n+pmFtiQRNJ>&z}0p>alP&N=oJkw7dplrrJeD-
zdIXcly*O8Dywc~ZR293&KCq$C&L}OAxxMX8v5E9p1nSB}w@<(CC_#vJ&UV&Ez2ee5
zF1LVx%H6+mEquG|V8uf@D09a4$6Aa&yf)I>YNf@W<-U28hjzaJR1)S_Ue$hm##ezs
zs6Pk*7Ct>3`wsh5;{oq;kU?NUB|i<r1lc~lTj}(n#=_mLK=43r`~c?IU=+G8<1AD@
zDC64;1U$A?HNLXYnymPFjTvpnzOcfN+Eh6qYDU$6dYS!BGEV}p?0$lzxVo@L6f*u&
zBK8A<zv928<V>dReYv%v#0zH_nRHBP$Y^6W2bv=8456;wi@T0|(jNR9u{qbb%}17l
zbDa^RreM$M!Jbow3#R3b9a*=g&NK?gE#WL0x4qt-_Zq!~=KVMNFOBlyDyx*9=LEbB
zr`enMr(~cReTN&6rw;fFRFC~Ph3Dp1Ue`g%g9SGIKJN;Hr+3$M3Iw>rS75K7EQroB
z&JrCu@9WleH$VpN4?4rU7kQ{2J^E5|eE(xk_aw?~UEm1r6j5c80Vjl23g0ht&G8`Y
zMLoV}l%#yp?Sxw|Ys}RjwQ1>SDpJ}eH|V=Am`W&++zL1Sxwq_QLwOFW@PSW6oVm|d
zFCI07J}i38R&UMvB&T^pp3O7q_lrfOBXSzt^#sgE$1OL(h_9&47o>=xwkAVj;NyX8
z)m&mLY@)P^A~$lEfnxU&zbbnb)l9tM%8uQ6wgCe32B`izx6(RiAD@91wBt%~H(Unf
z2bg1WSQK@CzKGwLWHPOySqa}T_VT@X&-2eUp{=wZk+2WQVRPCn<VBQkF#7GJM7*e*
z`Bp;Ie>zUH8hWSJGa^@KREUgEGtrPB$e;X<=5GA6pR{^VimO>eq44r#u)ext%z<AC
zD}hFiG+o1$$0E1$>F>&ecp)~UyH>AceyNztnppYisf76~croqnzV}r#xnWg0Jorz%
z5Ikx4;m_J>T>gflaBy0)>pQDe?(0J`%NwCKxovYte^0NXRNK<qbXNt3<14-(rtRIm
z`y`h3+@gy$mr>E)_=m3cU!y@ih?7@!n_NDl`Nx!+!GVhaba$|cP1#j`vu^lG`hpcv
zKssM7tigxZWYOlW{!rv2l(2kKJNhfnR!cPE(&Mqvjrg$vAGqn);nmE73N|-~qUDsd
z$ldB_5Yw)?A;K2}{z)fGv2ezYt|i6GsYk_mx;Ddtj16RXfJ&+1_fl+#)FR07Fhe<=
zs;{>71CcYNc=W50vmh2#KLc0LfN{LNRf*mr2f3iy(P$8-SeC2yi(wj-o$poQl6D(P
zA<2>s<kkBxsob1tbVx%73yr4anfo+mJ(+U!R~Am@G@&Z$5AT5Du{x4Ddb|Yi1U*fM
zu-@GQW`k#nVo@yZU*mq|pzZFrT@r6Koz-{_omyJj^=3G#ZEKyXZ8VW4@BqWXxMoUh
zpK0AfH&r9zJlwg_KpGbMm;Q=DGUjPHt3m~psbl$Al|1uUD}nVt`P*y}f9~IQpS1!N
zM9(3|9mcDa_DkJLuZh6boO}PLVJQ?D{I-t(3eqM2GRRv~DWj&qneEo<u=t8Uxv%|5
zu0G^&L(!Fw^tw-{a`~kYT8;C=DVtBzUj}j&XVmE;Ww0%{zJ#PJcD9vQK-1RC#?5!5
zw@d4*5RiekHR@p{eRL?z*)+GxCD-tn(cH3xkO>`3(J1#asIpYYnpM>8Qke=meqYLX
zOyh$y-p&@djl-}&LeN8gJ5Ht8R)0)Kef%!luvSN|ITU<Sx%IlqU_PM}f?KqvGeR{k
zoe*b_S10mx;q@?vKocq;7$Vh?7w-n!#-=x6y&GZ{dQTN__Z@rgV~fCX(pux{8{BSk
zxK%YjLyH{P_|=6c1!3yV18rO2$0W99)5likZt3w3Ic|?V`?6{;jwyW(7+?pGhh?5J
zBgKc@i0Q+6Z+!~T!u420cemgJl`jB8W^F~iK#)G{1N3D3^<+WNA*cnuEnQC?@S&ah
zWrj4^v+=WR?-ZTrjRDLP@eom`V<KiUAiADJd$h}Q#ij9G!jy4ZXyeE@kQmPb=+sbL
z(4MNc(847XvQp=iO??JE_#~BQUzfl|E|s-Jc~&K6?y4QGuaBd-z)5gX)|CCcZXlxn
z%xNZ0IcE1gq?Ogdm34!$+J{)HNkyu?r8Y{Xu;xr#OPs-&vUm(1NSOas;4Ql*)nQKD
zy9#n0V;f5tiun7ZlhY^W8WEEuS3haX)ipTx#{{ob_4HGA%4Z9M+b2q=!S+)e&j!mJ
zW3J}qgeS379jd)1nKvTRToPT|?u4*|nReAb>G%}5rVAbG8fub~r^q<j_oY7cV43%C
zL_FvSOIt6f-3|KXtU6S{&k9YJYRR>rZF(+~kZpaov2HPz0&%l`vg8^?mDG=m4svKS
z0y=L5fD`1r6rDObbm|!fB2fpx$)7-W60J-%E+7;nYZA|eJ~5c|BIn&vgq@J6jH^dD
zh;|~@X(v<i@h30kC)!)RH5>aSmN6r`P9GhKQ{)h-4$Qc5Yph|tk{0|4XXDrrK#ipG
z=rPLgoqx-+pNs?#cx;Yj`Cj73Tw;j5o-r!O`!+F_dt6uGLsPfAT~ZSoKAe+&H#J)9
z<S)XKV`#CEl0Q)PyodBq<SLmyeV2@msD<MSw&Hd446emIe0E^#QZel^Wy44+$(LKU
z8hiOsYmZjaE4rm4(~s-xjQ6Y4?u|0s%k?v}eJ;bhI_>t+6#>g1a66@Qj@J|yOBrGp
zaxk6x>q#Fb+VkD*8ba2#c&}u!J-ID4#U5`#uGDWb5+;;NT<EG#h~qU#cw#OkqFxFa
z>sE#1c<Ey?S<LWWg!-QE?Fz`g^LM>*t=q-*g+K7~rhHVLsbsdnFMnQVRW6xFw{P@g
zx7od|U2pqSEu6ddytH_^<}ksoCF2pP`Z%CHo=oxpi48q6v3Z%CnZePlVKwP~p3f&$
zrq}?z7y>WRD(~a&uS?}qfWi^k*)T}m1vx9od+}cV2W9wil^e;$#YuY-gYzsh8X|KV
zoNapxK(}OC5UD^t2jZ&?Q29e#N1Ziy$A}PhpvvMXn<%lO#=QPn&)M9#^ei0Rx4a55
zCPP@!0rqA(B}5HAGxrKkF{)Jg7*6#&kK8aZfqZ-65q`wRTXKGLTirT^R1ByUPmVps
z0!!w;5)K<>v$NNKC%okC0rVcF%qE(vj5@EZ=5<1xy+wv&^#&VgW?36UNAp;qWP=hW
z4RRZHQ6#Ke65&wq%`Ke)Bg)i%QMD{46qGMWwT=Aie2-}`)1EmwED!4Q<^q!IR5RkX
zJnW(ep;+<^ne>Zawu235w0;m)%~&*uh^E`uUSShzb0gr3-lF7diiQJWM_*2D%Cp(4
z2zH%qx;HtB7c!^U#y*aSe09R=XB1jd@VS>>n7kB@xC{KrH3dwE;G0f2LtT$s9pV-P
z9-)n6QM&RLFoC|<D~3J(IOl_iefH2Z9a8%5RZIjA)fB;09S4!bvgckGg@u}n@}^{k
z8f;P3n0P~ry+vV^R<|zeb2Xg9_nGPH4ODYgjfTX|klN7^Bd3pQ`TQ=gAsXZeE%#5;
zHUu+ZGXJ+l>LS)CC`#6Vp1H$bgEulECrV4}u=}+}ZUo`$DH&m7sFhJ;+Z$$dwbWcU
z6dEGjC}p*3l5k$`iL8Y&?ey!qtk9*SE&qzAtu`-8x!rFPoh~=7KR#irw<a9DYrVi)
z*ZgB*M%4;)!)vrFC9d|C&uC+7-i;5L+cov9xWvF%c~NGqpHpZcBAosutt&2WW8m8<
z@2fbkcf(Fr<!;o+Pii@rv-6$Kf}aQ)>(cr2(}`l(Luf>5b=fwu)Of<Gez-ljx-xBk
zuTR;ql%i4<!W-l*?ZoehAEw#75!_3*NF~IOU!5MjYwZk>QymeA4;uKE0NJpgIT?TY
zCNHv{S9s!5=IjH0F8~SZ{dL-hS9RtGOBf3Hizz<WxXxXtLfy{Ew<JemH!#)p`G6xx
zLH^F3d6jj^&4jQb+_A*999n$vhAu*zd*#okwY2gBKWh!+`yfhhu<J_An)JG?$1VR@
zm|1@%k~W_l6TFWQ%fE58yp^!Fiskcv2$=;DjE?^?pUnRSrK6BriR67V6EF^q8WOnK
zyaMPqQB^CkZ-^YLMYen9-i?UuClHeWhW{|-=o*|E?%ur!)^Zzg>C6>TCaYuBrNc?9
zz9`I9*4+4Qb)k|z@Z-%fv~`(MVx`XLY#`mzQj^#4uSsC-xHI-zc_EoM%W2bJE62LQ
z)S}vBxtk=%mlpUn>N8}w+B)Zzz7({o@Bds3Bv?$ow_^nT^$b9GgytTu9+a&1Th7K7
zrsqo9?ZD%lf=-V3h^Ro%pSlk7v(tQrJQV-|qG4g{af=`ZEvU$Q*|(>ce9*`ulsh|`
z{R66uxEiiO6<Pk4=xH>41thm_@0NbW!v_qHo*Cj^CZpzPKvO<H3{(CZ#cpcyf|M={
zo07VcKBMH+wJO*1%9s{GL-Lj3*YwSodvOrCrJ+O^Z{4V<N62BpCdfJNajodQaED-^
z+-3tF$EMnXFar{wwsQ!6cUDdtHUN}*qUYam0UuTqawAT-Sn_U7g;xE>ceZIItm$6N
z_jwkstCVk!t@S_4m8BbVA!QJp+4rU^p1Rqr<4uoFX@i_LRWGM9@zF$ssflAp1D04C
zM{zb5<Hk>v{}cZCr2(9jReQZ6?jj+JCM)f%Q@Ge}A!OkyL*G74I60hlgl)YtR$SwB
zHbJ~n(koXd`9$CAuJyE#qDLtuR+Ume3*D>z;ImG?*EjeppwY!=nV$Brf4sM4^7?BP
zcfWo+qu867+UE8<l;+D~?e5EdN<k^+ACqbrqVS(CAu^7)*E;8o;9o{Xt|VR*y7xEW
zDbWau<>=+BkL`Ot25a_bO9Q{spt(aR4&{01qpDA~i7yLNYUqNgtHrnJ1&$lXU)5Ts
z?C$+Yzf)G4FMlcGvNEb~;D5Na^MOiTh<Z?}#>o+{BOXMn=3b?jbs(G<YN)qp5fT4}
zD1r`W$MXd<PHomm7+oC43+<=zs3_m}kg4Re9GRg!LLYe@J5bD;9HGy9kEcRC6HmnE
zYrH?pcNge#C~7ny6ZYUB_bLFgOrrxafZ?+!gJ9hrUTY$PMaFZIp2)5U{=Twd2<h@=
z+s4JXAi!8&z0T}Evevd7>>R@~2aT}_@7Udlm3^u2@AoE8E^WWAu0?Zgs?wiDoxbZX
zP<XYNLUGH{eC?nQYBls;tTUZ!gzB);?)C5bAUfhB7<Ap*PXHslzK#^V!fH?T`FL&y
z{Rn30Qs^2yzvBSqeKkC%RM|5?4w3kL`G}{BJx{06En2{&NfLx!d*rL>tGY{XXJfB^
zlAGy0H7G?nrZ&*XP~&PIn=|%e(7t7+KsxG+?pw&6aqF9nrO#h(%hR88huf&+^G_)U
zPunpt+5Zb;JU;zk<=)znr>=bvj}|GMYJ7@<qKdTC-yEm!tK=*!y~F@*eBLCZ{k>Jm
z5PxBU7ZBY99X20p^(uHG8V`WxA9MMENoo<$U()g**A2=Wu3vHB7?tv@R=d43!~4}q
zP=qdHJS>ZM=lvpKs)N64m?3@L%IxJt3kEA}7*=`qB1XFw&GS&@U8U$@EmXpJTr7>I
z4%~W5;CzY(4WY3$6Y)F+zJWUTuuow!2#4vD#9TOXEz)E>cnq`2T3`(&-tanaXXEzp
zg3dFzeK?vAXO8)3x6DtyCLxs7gNPf;17o<>!}U4_B7Qh=T<2?`ccbIl&1_XK<(+`G
z>jHERq>m_1%q_TIw7C-%O7Ibl>;kgwl+&i2pREBGz1=Xso#^ekSnYH-73W+7mK^qi
zj$F&DT#?EnU)uL}b4`wTG*A(-m0u%$((kiDcuQ(|v4Te`Bn)BpcnevY;q$k63^=8h
z-FT<RtC{(B)w@5OTiFatNmxMwE~A{|>@74y4M86>P3n_Vao?&~T+<(?)G4YTFr%Qv
z{>`z!(@rXiJUli`p<Un25|`W1xvDh*FcAO|8Z)=P1tdi>A=ld=!{~I{q5Z*`F?()2
z?RG&?>|~U-?OVlab6X$|i!DDbKy$Z9p{UW-56m#6kB8g;PCC5%^I?@wh#<in5WnTD
z{e)n6)g}A;Uxz$}wBd0y>v|}T{_w4Yl|F@sF|i7l5q|&H<HG7Wf2i*8(F2_JmC)70
zu_2%5VRdERH$2yNGxO9f{TuE@2XGKji4dWSpSZb}W-~ZWt$imykpZYt>j{sI+w0qY
zus|tCf&M0iN`2`J+>G{3dEmp8``zldvtGNThU@T6I0lL3va0^by=(i>uha8KXDX12
zk6#lDI9|;LkwoY!sd7iw4=W3qgFmVO4vCy@MDd7`V){<?Lq;|zZpt`ofiqo&lnG?W
zz3@gbbeU5p3AhLO==y}#jqz8v4<oZgB2%zwd>{X%M#rpBo5i~EUu+c)0KfL-hc2M{
z8ipkaxE)c!Q{7D=VT&4UF)1iBdVDw5Nf}1%OYlShV9iTiIv}}gwI)e{hx3InjZDh>
z^lEq8)7Tz+#5CC798RuM>&x6uiY<n;x~v!&b~<Jt&I*KWh3|&J8Ter3wEDIOtb+%n
zba2|)j$cVz=rc8HZkx;)LEXBi4t^PfKtS2xp$Jd>iq!`jX(Kes<kSrBKiYE)lu3gl
zx0`|UxiVm!UiHEk7cmsO|JBW|#LS&PzO4O>kpTq;8D2MnDOEXLV|1{ypd#09>1~J<
z%~hdGd?R*b$lL{d`GlcTZP*+`j6TBRBmij1dUos+TMx5$Ub=sK?Mho^bnRAPVxo7w
zN;r?W8`Td{g56Bq(K=r@n^s|Cw<M6ebiT40O5|AQY2D65oGV@#A8$EbWsbgu!w55^
z5j0{Em#Yb_7iB*E+g>N&`Nj-E$!=aAOC`z(_tB066ShUaFmj;)k@^icm_Q}vsK|Ps
z-`s==Mvvg-a406^htmuiPq_I!p1<xQ=aAe_WLk1mKsCm%kyFvn5YnWe`+P1qrH!c$
zRTS^WM5i>~L@`?8$CA2*b1OBUXW>=9c)Pr0ikgV<-D;HXftU-teo58CZPfJ=>Tv>&
zDuzo?=9$aoG(O$v9bggJDy->pBx+X=uHsV|t}}xgx7tktpqj48D>Jg?TzQT{quEN5
zz$3IeN@;7?eGH(w2$BFwcT^JLCfS_Ac>%~;lu1G|?1yGCFFnm-gCge`KgIV-lhh^V
zr`uFnDM>>ys3Et&_{xk6F0&<dag38Ziqzqw{n2|eQqpjr;iffGmp)z7H*|cDs(3`T
znMJK@vVbz3ET8yw{CiKlWz#=i(q8h+xHX$+KBF%XDM_dJt<CMA{c(LZn-?C7dka;Z
zprZ}5D$2mIXrs@saq>r$$at8SyKze$8YIFEqK4Qdzsn(h^&|{Q7y}+;gBJe>q>;p}
zUD9llWlzi5VC0n#Tl%cmamae3fA*)5S2Mq;Y~nh_+UV<pH%(&WPMUQQUA&?Q2cWh*
z+n`aA1iqJRoMp{HNdNeQkGmP4Z2xn?KqwWv1P@dpUd_^EUX|Vuu8|b4JlIh5Ia7{M
z^6xUaRc!sT!;N~kk@`FeU{tU`MFu*0s9KH%?>DD#!Y2L_55Y72UrNY!oFD0OXiyDF
ze&7=-w1xXTV+NdgO+x?Ahof(s7byX@%Zx5*K=fS7BWi>x=7cP1mJSSj=_0dV9N;-v
zV6q{QBU)~|Mdw}lgWlQPeumO~&DDhFk-hU>GD8P4{%?)qhsLJZ|96evzshmOd(Zmt
zwmjx?fV)cz0%8O<n5UTCToJP?1@{usi71f)jLbopR?@191CP3)n27e(T}D+`{eAV4
zYRcIjU!}*@L+%ojTzEa)1U~i0z~;T?d+nJlk$pUnulOX3)P}K05E&<{_&{nYoz!rT
zuTB#^;CXA;{&Gotuh6(hf?gsMhLczlWbry)*8)PSr>j`=Bbm?&gS7YY6aXh=3RmCV
za`}7*>JCc$aLoo?8;gt3Lz(siw9HfPV;Xet5WBNki$N1Im(FG$KHrOO3o><geIE4H
zjYj829Jhd;&_%T;O_#HT)_3vb4kUrJGIn!Co@5V8pJqmFrsN+CSTs1fH~Aw3RMK2;
zm()Iu#XFge3QETO8LQD4xWp1NS#pug+;5PU)6)(cP>5Ktp)>0pXL$Ss<$DWI2{dti
zFKT&lK#|1E_h_N}1j|0_ILE}Uix<Eze>HHhm+|<Vb6k+t-R@)-yl;v%@8@8CiN<9!
z(PaBCcqSs~$8^{YeF5HDV98jnh$_&f#2s6}iLXs-%9hFQs!YHOT#6O2Q`{qWvEWQT
z(8<Mc<yQor)Gf78QB9eGdVz34v%n08ELow?VZ^iMm};inj!z<URKawrYX5W0zjJ8d
zDwl{ng@ZO|tOBwO{sTmEZ&!H7eFymgAv=1T%r&+D9{LGjzKA*L4jwH)_V2Vn9Dcky
zO|UTaRou})#4vLmRIg|-<HlfG2296Gr|)G~sybJsgy(ECD9rIH2}mUyWI&mAZN3rx
zrq*bK3ktef)S&G3Z?ZMyiZOYg_R>bDt!h3V8RkUx!aV5Lx+kIOb^8kw;nrFO<~U?2
zwo%X)b5$%$5<YSul^p&;xST2PP#V|MSPF*YakJ79conJC^Mp1PVx98;R{mBF@FCL}
zLc^bkfCeJG@OJ$^9UQu$&&`;bD(GIgR-f+sGah`cy*JS+A3ES~4bAc06OJbE6DlP-
zLsN#9`7|z#s@79Y(-X;eJwnU<WY>)Ti=2p~VcxgcHy)=p-xJ`;q8dr5x7gkrb~hK)
z8qyv|KM`h)$V<Jxc41^F$qgUOTAp$j41Q7|$U!lx%hZ(&0Tn#c3j6KU5^mlGG3(Cz
zX`mpNIGC7vw;lcYXR6UFbTT=01zaGap<nue=CMO>K(ub*T`}wWc4@_>o2AUu(PV*C
zzn8-GdVl@>BCpv^>NEI*<w2cokqy6g!S45I28-hxtF02YmT<YEf3^+uU5+wnR=CR$
zV$JxU?j7lt(FZwA-xpdl&U>=5Lcv`O<IWOq;FNao7PT5x+ZoF<ST;<$x#$+p$MsKR
zs>sT>4F@K%L#3LnTH@VORqjK6?lh&HS)Rv8K0~+pNQ5mX6UoBQdN<KQu8UPUt?N}G
zst~2O)-{Ns0g9UTw<su)hNT@(2j5@-m!Jp3kNph&?)%fWnX^7CjLgr;F8!Zu)5%xO
zHBPYvk7MQ9wd^r5Bs!;vgL4A%TIow!GW>@vX{i1Ommz`X(_7hxe4cSW3Z{*E!K61q
z=KMC;t;`I(;+eJli*J43pp$##hnx#~2Pq-$g*_kL;kO#Lm%Wg(Np7_8;rgA-e)kO(
zUtLGRj9Mz4!KO_Hhv;Y8;Y9Vg|11n^&{Kz_Xvu8e8eb^Dd@Y~@y%r5$s9lw9@&T|0
zJ2GaCblZ1Dqds<cIv~Q%OmAqKP2)0*#PoB*Hk#hZ4}(`o0@0F^h1LTns@xi9Lr3yU
zy7PW9uley*TCBvdoWk(VFJ5k}xU!Y7oyUysdaiVg9LiuX<xvI@U`9MU3zJGUer;*R
z*^}3E4kNSV@P}X=;J^_Dc<ks3)R2RSf586Or}}d$d?DTdnut`MioD+@xx6nCv=PLh
zd0u5JNP79xqi?vSZom{pD9(KnL?)!xf3@GD%KE@aT`){XKi*>CqtkqdJk=u9Aqqr)
z>;2=5R#hSVQ_e>(Xe^@f+43ZsUp;Nr7zVH8?c}HyL1ZIj*T}O+aU&j7;<No0cvbY}
zp|SZg_6HwAI@`R7XKF1(2~yiqMx2(A5W5?y`p?i84WWZ3@>IPvm0tv!jbrs76-=%c
z29D3WUY^mT&=3=&(G3o(B~}D>`mcSwjOI7^a*l?f#IV5EL$S&NWi&}lCCPUi-*`DZ
zAJN;8GPAJ;h}6-RdbFedECVKi1RQ3PAj)ApQS;-Rn0%E5m6pv`m+aSw2>t>+raRT+
z<UM7lV$oU3xE>)5MC!?$B4ZwN!a{)2vzo9x8Q`&{fXBAE+0oH)2bB=Cx`2Pd?8{9g
z(@m-j8nhF1o<;lB^~bgmxp4Kmk`xJ8aGL?S1{MlRj!GZVOhur@pN!Bp+Jnjfk%qI|
zZSAg1$D<jh8`KP!fjoWx&>jutUJe=}0uDeW7PyL{YuZLOMQGL*J$F8Ihz0kvSrZ4|
zj_Atn{@Tuq#dOK0M?^$8Xvg0Mv3uba9bC4or<M5}h+L%%FvwZxzb<0ScaWW=-rAAE
z(ANj!Fz=B7+ojvgc1ji#XAUqxDGmo3vGi+M;gQO{dBjjMeWO!)(}d6o4Dvc=y$)53
z9Q18v6j_`nB1vZ%@B7k$VGMZX1$68UyH<&A3C7|{gU2YbUA@g;DQ$ma#i#=C?)F{+
z(;bY+rFsZPy--D&fDf%qqD#i=uSi$-!SDEVL()bYUu#0PryA@~uM*9}mS?=Fn$g0N
z?|wHGXneQx+4I{|`3c;^qRF;*$U~A^j9l#xfuI@fnt^#qKK}e9=oX2ZXn=L?&&Kc)
z%mrH2TNTK^RlY8K^yew?xC300qL%(5YtBft7!vAb{?R7?`DGlb+ZrjBtKl_jQ|8rt
z+RbHJ#DK{c3I(aEpJ1*7{`?Q-f}0D9A!u0=xt!Dg`?R98UBBuuTmOtnmD}V|TQ=ng
zq-M|$A>NKSykU8_yV<KMy!j@gFt@7o(!Yr;%}y4ys}x>da{{}N(d^7F2Vfs|=e}%p
zpr>Xz7vxZ?fj3UB@5T7Vij}4{yhsq8G|FymM5thlH&OZ#y-eTdjHM}bsZJK;sV?)l
zzC3(`0mQ9kn$OOm={4BCe`f7*e}~OpwI&;WvhMq`=v~`QN?Q<Mt_P_9Ovf8YegKB#
z2Qc@E`~@y$db3mKD=;y%u*6?k#evt1R6>=Atr-G&RrXE#KhCtIM#V+Yj{tAtVo>((
z&1NqMc$3k>(yAKZO@>fJplTqM@N(+w>!w_01Lau&93v7k9xpbjK=bp3SZ@=5jup@5
zr1=|kav_7x)SAOw$IMU|vJ0&?t1hhI<TH8`L45ZkdQhUkm4Ng8bD&>|>|R9TQn%No
zdI|Rua7?6OH30utXWqr$Y;Z%qZntfoVKl3A=~U?jblz4VX-i@>V`N9^$S}}I8Y9h&
zcTOu{J3kyD+<^*ik68wQ%B?8IKRFhY*!Ym)rt?uJ`5@u04}iG$de&PdThfMVs!?}@
zQR7u=!C@0bqUzZK=ee&qX=78|c>ea*?ON=Rr-U8gxkfOo(Ng;&WFCyv#sa15vRT1i
zS*%>I(^M^FQv{kOvSG(mbS7vaO>Rvu2WuH=e91{Q@`<N7ZCD9*=0F9)U1YPYTeKuL
z+WY<tRuXA0#a|qM3!}e71EQU!&F@cRHp^A|Ith0=O@Z^MS=VP;Mj^I8XNoWcXvmvM
z>jPhh@-N59cIbEA(rhMtIxc~(z?f>#)FdxxiSjDFdhYjqkJSa=5q`|wvdtNG#!KjS
zx~<qtx156w^6LBmb@O^}Opzx4<!=vs^U24eeOH*4xdAssA+bWe74N<#%1NualI!Qm
zH`YOy6q`9{KU8DyX#QmMD?3%aPNaWZWJ0!(8LOi^NFRA>FZI>?0+j?e&c6;->-0WE
z+Pa>;Xp3uc1oI>CZOG`@93d9;hg&1%#bVORKR}xi&=Hl>&oL}8s-Dh4YYeni(&Fq)
z@PGvvfG{}dNz*x~;^S0+ARomW%WL|lzvp;wFj1bmn_nd!Y<=6jX8?`^bw|b07=|xj
zKa1)6mqeurt4Zky0JEIvnGKe?zB|6l64t6l!nV7|t|6}SZV#4Q;|rT`q(ZhS4Bw#J
zQvRkgma-LJMSd2rH2V&79|*d%o%gag8gR_RonpCbQ&TGBhA5IHMN0CKriZ~(A=z9`
zpf0M}vM5c~OW^G!<@ifP#5I5Hj0Isdv;h_wh#>(oOOf%4GDbCaP?)85-%;_q_Plzh
z3CXTg=QHfsZALxddSabyB$Bt}7ej`FrK$zQd3F>ZL&1X91jJGXZ}%cqpXR8{j^v=k
zn*Zd8!HDB9z}_N8s|HtK4~yEVK7C&yg4tExY%ow))^1pkaT;;3!h;?30Dy-^NZM2v
zP@E1rhU+W#ZFC?=ka*qhjE~L8@(6+6j0D^M%dqlDhWfY$E!wlE6~hANdxcCvSS1rr
zFW<8A4E27_t+MB=b(5!#GX2~~KUAv<Sle(C<~>cGvgi5YqD{wVWM*;#&NCck>QYg$
z3HyNcy_+-(MHIm43$|Y12?Vac!6pn%E)t7X(Wg#&G@mM$OTB%df&p_&i>WIis4Z(+
z%NkV{9eI!8=MDiu07tSHT`+8SD1i=skXz%V-EgM*!Rm$kPb`)O?C}`hllQY=H58>_
z_PY;T?1g1yK<$>kckTeqgY`^N95bnAnj`&T`Gxb5G#h@*(NEg7cGz-B&(C}0ESK<}
z)>CyhLfl*KABuK0imY1py6bzM6-p2fXt<ltk>%?ECn|$I%!mg&mbgAHW@DhY%Knh9
zGErDYf>D)4&}iBgRH2TAm4W+_LXJmic*%(?cuIffr?D}iIQIZT)u*O_5T9;T=3IC@
z3rE>ufp(uQ|B`|0wRSlULy$o8^asxu&|DS+<Z(xGw$1jR#>--hP%lDLLZ3D{t8X^a
z_R{Rvka5`LH^4(>O)$Ii!d?ipj_DZ?9c5xrZn2`^G2N}Cy<Jn>1;j<AU>KMz<Azil
z?rn6MDo`LF+3e;x{8OY%L~(y=*7mgrfIYs9Iq^7(bH@%L;I<QM*fXpfwM0l9`0R)K
z;Eujj|I=*XH{q?#R~nBDZ1QqUwz`QDPR2gvZ4qYxmQF1xrQiz*@`b~0jSH~VXlrAQ
zgp!;<M0hf2MeVW5Swvxw8=vfm%Zow@`%pj&M${YaTzstskzNea)9V87+~2qdnG*Zt
zW-j-6s%|d;U~TDb1&rM6r5|^}+yt;#C0n+0@wjY&a==9W^PKZk>s+Q9e#e6Hra!MT
zhpT2QC~<>*VgT2rQmXB;fe|__x;t^O0_&7-Op>0x7nlV|J^cMK++<fVWCw8_3!vm9
zRa1Y;j8RrZr`u^EaZ_i8jhP&F1|K+2|HIV&gx}0g{d=hY>0bNLH^Xahe)kvWSO0zU
za9|nfdaKyi8^7R2|Htb#M(obtY$M6kn<*mpP47D`6|8;Z0Q3J!oBIKICRZZe;7txY
zDAH1&VVsKZRq@cUDtj31T(J~7C9b7YQJ#IxMqY>-b&isV1a3Uc-i?HO4G|<-JSi?7
z1&8!@B36$KrkXPkrM|vPQ&!@I;t(Ht>}dI23!lH!hGe&?Yv6TMieGM(awgZ9|2r9i
z&b9HTNu^k$g+@#A)MHhaW%e{0(ndt0kPTR>W8bd7>~DL16q>|Ij{$0Uo=4TvG|An>
zVE2;wza|6$r^6FuFaP4BP2Of(c-p<HP5F{K#(dR97gNH32%P~D>mNnG+SU<C+<Lk;
zgw7>^Ec_zvpHM&D=#7X06NJnRwQpB}LuBkM#DZ@*4?(JPjfD5sMn&Z0T3i|NYgzh4
zOK{+ZaTL3q1viuI!)C{u?j6Kj8?N&{EbyCmrdrG-Jf~9g`2!HO`W6z)756WkyPRBr
z-T{pGbk?)Zg-1cOyWqxq+RGj6jE|{)0D#*a?tT2`U{iI4Y2*>8iSlYia=rE$Wupu*
zbm`&nZeCKZT}KGpMVBF5<c7<e!sed0`O1T|$2*N5oATPy`BpkYBR#N#ZMEN~0iI&7
zXblrfhX5^uX@+;O-gHAkpKW(IhnLhr43d;(;kVXf$#<ILNG4QuSU0Xv4+nbvm0#l&
zkEoE<(yw;Mq@FhX|L%D3n8ur+gp{)*uI}xoV;>n|zAdvsOFpdN8?+(<>%c>)s-r70
zlRz<;BJdO=C08a_ZCk4L;M-7dvlBvNT8*Mu;|N_mc1``azGeUK{f0)Cg3P_LOh${7
zK31rwPXTP}46-YNl@hIHM@9TC9lM+&)<lchG#jy1+~Y_KyO&QA`VJ?)u;Mfgm#)AY
zMIJS)9x?heFI5_tO<S6gYRrkdb$<D5WA6V5_z;;>^(Rr6n!96<1`cJ<r6B2eUQGb|
zdwfP%4KUESxYIvr2h6L|yp!H(q%JlbZhL6iq|4zVYG}nkD#xC739eqdH?zUBYK0<A
zZZ=+3>8EI*#7S!SQ>c!CSDyFT@VB$p|H{sM*6+j^ZZ*Z8amy%Tf9SO#X$iqfb$!3V
z_S$O6d62~U_V)zD7?>4wdiL#iN(%HxdmFNyVfHkWX^$}dHtFx*+7yI=$%EWMK5KN~
zIx+nj`<7>xGEOB%sc|_ed7OI%#~ml`1T?r=153hBC(V4zEOJkH=07E9h3h$5u}ND*
zzpr%vD4Y2HW37bH(&-;WrTa<hi+5dxVl|<@zmy}Bbn-Z(MVu;sh*Elv0r;!!G`{4w
zgGeud63oY_7oL2Yc-CUt`rjxOsJUhZ`r~xS!wjy8B6>;4?cK9DwZJbQfB$e&@1As)
zc0cH0B#H}|<m^{|LFeDTt^lL+UO)|qobNN7L>ew%!I;ap!*vDp@RHB-pMgkJn~`8_
z^UcWMoL`D9Fbr(~fn=~W;wE!1k_x;O<aJUq`(2wX5OH94|3YQ3wB;${%SPD22Gskb
z{O3tyV8D(>f9a<wKKtva>TdkIScD&FSSMK+h;-o6-7mF16t1{vAu`6;o&!MbE3x{)
z_HRthe{eN|gRE{XeKF$U<j1S%&a+qT^c0pex=34D(9MRiL-^D*9WG#3+rp3>+P)o}
zVgWUPp?I7G@>O<di;kAJ9(}m<yYM?0a!@Ewdj^NKABEKSm*)RyPx+_)*zK7g6ZV=_
zwR<1@@IeUvovS$hPJSS!YgHs>`Lc{@4vWt#{9(~g-^4WCOXbiL=8ahMnV>!35o7s`
zc-ev5a{tHRGd0(coM{8uhU}&$9w)l`lg3N+3ONM{rb!kHEgmq?eH)c6`bs;Q%vu>3
z7Bv%Mg2(-w>-i6K_FI19r>{2NaW%^2QF|QZ|2eY^s0)?!Dk-P;H3}>pK)WQy8c_3d
z{{({g>NYc->cngSWHaAJmKf{CL_h%$X;R^s&6hQ#?z;QqqxKO5tYO?E3wSioKf4w#
zO<~LnHu?i$#^{_3E^mOmr}!B9XZDUs!Ao+VxCbLKZ&~3{;lldZlKtHQclVdc-ECVb
z36<h6Jf5X+UevA=Wyx4*)ouSTL|b>Cte&Kigb0sM07eHH$`BRH@vH;2e|qPmrv<@>
zDK-sEm7j!Jl*Ms3&j*41V!^9@o<=&5*YBZK(C)ny?D@78n3VCW4lV4W34gG=@mIQ!
zg%2tsam<nqn~aI}GC}X;=?&v}I};&v4Y`eLmp(XhtKH;2>rn@|Q0!laDW5#Gnawvo
zq=Eq-cPl5iImXE^kqU;t?cRKt9%M5kWAh)xW~T7&<6vEzB2ctHG@=@<jr11=eOBO4
z!WzxaS3kom{WwPEyzT+?@!1LprE`h&S9}p+rr{=u#zCcFhkqt|I!*j6xnGwx@7>V?
zx+vXVTplo-YnmK+!PEt!p`M!`c@kW{NB22=<dwHO{DhG}rAJa832|jc&Bwa1vp2?~
z%tJ}$MSUMBDE%1aX8iWz9>aAT%T#;@Xb#HqtR)>qMQv|A?1z`BqQXH1Fzq6RX`Nt`
zY*$ts&=G6<<)=y!BWJe15pK4I>0cOWV;9}gemr_>T{EzJu7IhD?QY$z$jRKkrR`uS
z|3VC4S-CY8^)8Av=zV^?Z~y!)xirCS-76^7wV?mjvi*K{s%^rR!293{Z15RLMi9iU
zVLN7F;hZ9ErWc$M=(Boa1hbNF>j$suT&e@q=dCAbzCb`T<JRxTw&Xq~Z7D7{CJ;v4
zkPxoPD7XQm>MA0Y6KVTu!NzK!1zeD?6FJsu%q4X#b!%eIaO$qig)gckmUxZ<aaLr^
zbR+Kz>Q5bC{~z|=DlU%gX&Z$kB*B9Om*BzOJ-7_+8r*GgCnP}d;KALU;2PZBJ;B{w
zhOZ}k@4tNSdoItpI(;!en3?YBwYpZVlBb>mCk(#-RauNIK(!x7x33*Q3&=K|_BQ+s
z0HIO&oYXVi^UTp{mRd788k)$_c^*<xvd+Sgz^v&)%V5^1XAl58<kZ;=FZA}nn3r{%
zyZi34vks|W7jPmQ{QJ^fcRmu{TVM8Durz$3&9K9JsJ(%M1(E94A2_+Q5rUr0mVeEs
zJy7>u*Q+cQi8&oFzI5Ix@<DH_@*0?WB1_v{8mr=NU3MAdUn3a%;xS|FjR{(qrJ7B?
zI3BTJS=r$u@bGi<d2pzsJ;;|WtP#ezzAi7OFW-=3B}yRvFfc$8#pet^+ws<5${VDR
zmT5H6`Zg7H72{fkp~l!$s4->N@-pjX0HFwS2T8zmLp1`WTKg6_Vd(w0rdI#aa`*+7
z_{&lN*i|VKUwZyxKaA3*NO+iFnCE=Kv`S+TZMIhDta+>6Whq-6o9<3&$L)`RX*oX(
zyns>txw@xOsxJ@3f+;$At^~$ZH=MgyF{@oz)mg6VJG-aP^UV3(8cEanE`R8MHA~H2
zxtD8*D}x}0-09;?jlb~lW=SvGx#qXfe}aR3G;DXON^4zHI_S?QAVW3g-TVG|GoGkL
z{pm;l-uXw-tNm}>HB!D>g945dt+B*!z-$xIduqL0gN1zZ;Qe^g(X4NZ1%O<%b%D<)
z&`M?N3M`gg0r5!1Y^B`EDloR3PqCTV-=>Q(R^-Cldt(a(|5HX{RGBK}nAS_JrC}K_
z-+K8Fj>T75cLFF!0$$^J9{Gw9^NBpXscK6Otb+xUsO@jo-`5sO<7KZ&b;^ucdq6jM
z%Ac}nemlYDwBKFAY*y;Fr9s(WRL^*>iZ>m5gXM6ZHhOK<H5TV;FV)v>>FvBfyVVL0
z_I~U2ood|jTB_y*#cA(eEfv?k__l}rw%UciMxtETe&p>M+N0GMzP-n%;zYacry9>7
zU46X1FPH9a5Z~0*6GljHo6+RT4Ua=B!^6h(nxZ~UZ(1J4jQ2Wx#JYtEuouW;{F;(T
z+lcL@IHN<rg;Jc}%K#ZZyJ<MUEeS_*k_$<ypUuMNuih<`Rcfg)I5n>~l71XVR$0#r
zidvFNbEi#KEbuGye5xy_HyM8!pyezDof6du4JuJ{N8}K#8hk{T!ykYWn#nT;!Y>1v
z@(v6XI=!lae@h9Cy<CQJDRfr49X0=&6|w!f^Ev{n4O@f~qx4(>G`)0LMg)2@W&O{~
zrfUsl0{!VT@>(q7H4%;UcVay?vrnJv5z@v~uc%s{bf9R#xsg=U{l4NcJjkW~8;$bM
zqk_M$J*^I3HkU@*wc3n`9Mv5)!=#p5wK|bN_s}j9fIf6%ebngl>}sKaegl5ZuZei8
zCsOx4bQ2Ulu`K-7d)1#@^fF1v%y#2!NgsKv=dFvfC=ZnfJ@CHdvshlPVIhku^X1GC
zX9pA=j|VZ^^~0U0@nnCib20RipRyX%tTtMv5c_cOO<QWe1PN~TTz5sYii}=D=RVQO
ziz0<!os#quca!Q|fer%p4DODhMjKJa3x@q!3_xw%BF$8q3A_0(4H$uN0#Wt@!ceN0
zkmj#0$`N$2f@ZAC2U!c#;+ukk`tNv>zT_pZ&DeR!v{~0fF@n2*v9f*6=JVjz(A_q7
zv_xI!W$wMb%!NwP8k~P#b3-g|H0x!-oXp~!zHr+JG&wt>A$iTZjPEX)8BMV0BL8
zH1i0UW;AB0G1owYPCmKe&Ps4aw4Ox}m<Y@gsS0b3xMNz|eWgbI7~GF}dOaf4HRjLb
z!@HE}CxeSbXR;-Wu6BNtyJqV2n>NRf0#&@JIIB-RHMw-JKK^?~@(2pI8r<=M9SF+}
zLOkWk63fIyfv7wOUv7lelQsK7rTLf%U(Wy@ms9q;*J}MBnD+|uKUb(YYi`u<y8?y*
z?{%X4X-I^A?mA-ygZ0UnJpAFjEJ7E{tVg(-O$Hexi5+0;2Y_gw0y?K3E>oz>%a<{0
z=`Ik%liS>JwQo*uvz{&zo>kYjASO+fJvj@XGr|q63f*hPON^cldy(7R!P*&cF1t|k
zK4$mMj?9kwkj&K9ZJ)kEWXwehg}SXNcYS~p!!^ri7+VZKFv&ony+zd#ck3Q%I!>-J
z+po}-!=G7In9KTn$ek`%-L9*$(#_-XcT`bxe>PK5&yE4TE3Bhla^L%{no?kT%4#<0
z7Aue9Nl9G#N^_jmVmPKVOe0KF9mUc_syHaJ;~kVyyhF|vI@%=j!B^!<oy8jsf#Y;Y
zoi)O5tP`4G6y7unn)P)wfYn2vk{{BRtO1}vYy@QJGh|8RXb<i;HFyyBiP9;1>41wB
zWzU8#)DZ~Wve_bOf4OViLoFGuXD;l>Lk2eM%hveAjZa4NP#VBmi(rCTiZ46Z*c-wL
zjwfRkOG@L-&t2V(j!1%l3cEj;G>hOAo5eJ=MSAJoWErw(D@$^D-48$-n15Cd9IO(m
zC7zj-L7<D(AAcKcZ{o9(rO5{GG}owxu&TZ$ZgR!W`ctPRHXpp36~1ZCq(=!UU{UqW
zMm1YONIYFp5oa#9>Ar!~r7Wlqx|B5KN`55-(^{lX=YZG?ET*mLxFix%qid-9v9%)V
zuO`{$D`aN?o3npf5HZ|^4zJ|Q4Mxf7j1OKY?%?5ZqyZL@vR}6}0Y^q>E)3q&91#L<
z1>U^(x;+T<CDXzW{H@lo=pf*86tsb5B!lHETvBsnIT>hB>vUQ(V4`3;RHQrXd#$@)
zvXD`{u-_(mL#vrhxRZc?S+iXz&wwIu0dp!F1CfW6=#kwMa+bpRj=B<k^AtNfJ!(_?
zEU<Ea@S|?dHexz3kC8#wiBL@@O&e`CFMQMDU6hsL(0dh(*!<C0>%{ce>Ra3NNdf3=
zQZqFynCuqvEHe3FlL6b{`8{NDVxoe9JD>0N!-Fasznucu5){WgJ(*jd9xVsCk-BJf
zlm%V-s$Q`GEF4nFmI{+a^BVRM4BG?jzw`v6ceT1G<dfG0qSc6cC@{P)-?tZ4ATgA%
zj+J&W<;1<NAV3j<_y1<1_mzO9)?xCbG}v7RP(+?z$P(MeH;+M-#C+b6C@^Pa#G|R`
zU~7uU-K(yvA8pI+BYcOB2dBh>)K@lR<QHo|g#5b9U6QXi)oZUZ{PLGz+XfHR%2AnQ
zgy{>LZ#nqZxk6IA7!Y4WQnr6G8E$>;>3O(S2@qFmt_MGD&aTiLvHt4XL~hJa89G-T
zKt-)W1JY;!9%AwZ*JTtH_oh^^PN*SNsQA3|j0Yb8wUmycg;|SK{%6(wx<^=kPm`?K
zNVHLinl?Be%&}4{ry`j=DnT-ghk3bTDvS1H3SDZM_5eo5{-DBy(b)g5)t0CFqw*Jl
zZrLUe=GebBt#EWAOA`l+tV3>!6l^~)wZYBLuo*&*D15doMh1Y{{F$j))wUK2%IUuQ
zt-U+S&r<G%itK|Qmpl2iv_BO9L<;-APzK6BouJn~ET|fG@FD8b%g5|NF`wVV@{7lI
z14|v)EVKn%Em|Pb4?IOOZG0J!v99BQkfhdep40sFD$#}w<VW;ufp$Ip&=t7U=US_&
zJw98j(^QAkb_2T)`qdYMh7DPF8?K5;{7g@<zk&D~nJrzDjrM~V+yM}nLARQVxl<Cl
zV)L$q;P&uj_Or7`=QCtdA50O+g9m6sM=NLMBe~JahDWuzLMcC?XwMEPOy%%1AC+%^
znjUngvtV2ZzRAWY=u!2p7B(2h$GTjxl|{q2T!@ah(R2mHyx~1in}JGPQnu;1POmld
zOgtG12)@kSAF9pOsQ_}_!s(Og>=(ryEcF#5LJ6JFsM=hM<Isa29L7f3%-ks>H6u$8
zFoz(VFNx}{L1DwW$!vYPRDfoK@3faZav@KWEYr7;w{85*;4;zdIB$ZzJf^YQD)32u
z@fwI|G4Dd!uTv<>im2G4yoWE-ZRrRx>Q)!?Djga5RAn`kM}m(96QKBbG5#dF{9v$_
z+IAMjY$b;KV&v0~6tiN>K{V@pnx_fFd>mqUxX-STO&a94uQ4a`3I*kEp*$Nz-s?gM
zAR3MO0hSe-r7X9=&DzO%G{Ii|#?=Om0o?y$4(&q!;3UA<`e1#Ugpt6xhs!-Ri#Xa6
zlAfJE*YU~ZAP>K77(g{sGrRjjky%SW7%&`y+B_)%hHK)=Qk5-KG`$5dZPv&cx7%NY
z+gaZn3Mj{FMxV*o4#|I+gf$6o4)~D+Y%Bu`8^FeLb}2!V{f>9MmOsG-`#AP!9Hq#o
zfatA)xhoO^_Cl-5p6X>+75#o0HTsUiL4~&FehFZ=8-R-QjM~eVvE0l0Y5KK}rui=h
zx5uh*>fXXT?WWHXBpAYM0u#<lk5A_cNV&nAly{spA9`j=Ja1e!ku!k&hz-V-vHq=G
zTlhlFdccy^yd&Y0wsGSGq_Lddif-{k%ucago4Rb#<n$3VP4)+Z>j{l%Nx3!_rBSA1
z`*;a7-6khkQ0?z;`JC8u0HJf{26>s2F=MW2UVyhHSpUsf7Z*rr(wKvO)^RX3A^37V
zRNNQVe*EqBSNuh##ldn-3dn@to#=YL(bV=%(o@;qw%-WrbgrY<)~a_}i`CXMmlFt8
zu+3cnd@8Zl%N?M!v#NjmTx;z9#&rJR<&Kv(q)N;h0M=0T*e_$-VqjcT5xWvKT@55#
zZCmXL91Q)CkHw`{N}g;s_eLqfNYYU@a_=z#J56h5fkNdr2sO%1&x{AXOzLli`3Fz5
z{3)q7b9Dxin&nkdX)+k$;;$GNG>Ico6n5;9La}FjL=~Y6rI4{m(^Wp|qs#kL$%+w6
zz-O4Lw&cM|$zB}M3EEtk)iJ96XXnDMd5LH>169qB1VRZK0h{(PA)Nbu5bxWB(?qa3
zBY`4AkVwx^3ZE;QuUyN6^ZM2F;0C21=hN+wQQKo^za4M1FZxphUk=ub7@vm6N*#<N
znFIzxgw7MPhvZ4x&CH04eBWAZ&*SFKIUhUU=bpE;dOeW+wltHcdx^_yqa;}Yi(f4V
z555n)R}r{&aS}g3Nf$KZJ))52(WhHw4$}jit7LsaTD@#3k0Q^h%Ag9z5&fTfS7#-H
z5wGX;JV9&jcD4^$B*{G6sz8)Y1MllXjsO5{MO~8yhj6d)zY3*&)bBT%7-OU>ID+~&
z5>tEI9|7ccm0)VK`XF~k8x7Y3fPZbkMyi0Iwn$u1{d&T0>vePRDe#FVn1sErdR)1B
z=}!0IGBA}GgZuvc{GFp#$Hs`j{R8~|x^sSugY==F==OfWHBrZX-79>2e*NYdG>K7D
z8Nq8sd70Y+o@4)62l`wdW=a+jc7Ts;1Be<Zk<Mm)2i6;td>i#Vi$1Qck3wVzy|NNU
ziB)0g&4O#!`Ol!p-|X(=(y~h!>q4HF6L1`gZx{6*=<gToW(N6d3Ox^o<!&cGZ|aG~
zo-wn~>6sJfWI+yd<EyPBJ4%}hB$;?)<;XS6*r9kfEp|f0k?}*VG7(rC>9c@%GBuEx
zf9V;iC}8QaP-I6q09pE=GImExd~u3uVd{DDfPEl-jv~a-IB)&tMK8T}?SZ>{K`Fox
zn|k0Jvw9co2UlqKO9ZQx$#&uks?8!tvo?_>8r`1uDVwicYHFjUIc%%05D7|>fUm)u
zm-^97T1LHxD-zdcq<P}v;#jA43hD#PWgo0c1)qYR&=%P&kkSIq8XBTN_(ksR6;7;S
zcDP&QY=EPf6KJVdeUvU+F|v-%9VljKutx`?@1)c^CC}%hhMF;4Dj47MR{=Ao9Enr#
z`v{3x@`g&W#mH~O?(;^^gwX9kCNl$NwPBS?`RUrDYwbCGK|SqSkQp?B|5Ur`4lV~t
z-%I_>%B;FmnPTj(HX&Sa+{+^EKTzO_`Zbk__m2kSZqyxgexCu7YW>D#xTHfj#>JQw
zWdvk$<O_|aMak!?to-{5De&^Gp0hLJU{kWp(Y*R8ixa4Rd2z9!Q9R8Ez-g)e&@bN(
zKdF{!304>BACZ5QA}}v@AoDEATBQNZMBS#IEr=BRYyZt^)8wT?nrao<!#!Ks8Fl?u
z&U<=;hSilD58qfjSrGSKj3j%>Tg1Wf=OWDkeBtC^`jAaRoieX9>qPnA!I&n<CdNF2
zCwQP}k#q5Z`h~Qqd%LMvhmQi|0&!zH>NJ-G+HA&XN!n5B(^i8#)n=(#yFv%wC~5v^
zih(E^jm=*F^J;qVxr=~qR-=Ps<xibusJ7VN6BW&gkdaDovdNHn1v5YfP`S2!fPrcL
zKD{WIAZQ{xbAC)9d^fQScuKWeoL=iFkSo)gB@h}zisLVbn9T?b2y%!Iv!9*PFa34?
z#_nY2gzHN+ZWb<A-at5=1&0oNuuX9VAIQ(jGj=DX*bp#M*(lztx4Bn!ZfKB-Xm?cA
zdaKvRlIMH83`tT^$utd1DtZ;aR34pth-1U;8y_Aw!&p|}tDI<LZ$1eCX8|)|3?ofc
zQ!bIcu<)gGY0}8}D3{_uWf}*4cAC|-<7NY;HrRqvU9Nq-iiYts;t{%2>89`-r_JtA
z7?XYY4hpa`t+Dm}clF~akLTuqX}TEx<<k`<qVw9;kNN?kM)6kHy=kxX^4&Nqvlbx8
z9sH%}2U`tY(g~$qviL0=%8)@8T~sE$fo3wWAY-C8R7MpPW5Q@I9dOvq-X6$T&+lX>
z7mmE*mo^zUpZBIaTUD^s3=-zaF~=|rDjkge&~}A0HL~axwLCjso9LCf`Bhb)tX<6=
zZRK1CvFEnPNU`vQ85Z-o`t0-Ad>3c$F)G)|60N$Uo~dbZApT{;=`(S|ez8c=ko-)2
zUL=5hrC7V!!qwf^E{^Q8V;ulh2VycGmj(B~VA|G9X}0!AxJ7HvTB0|^cZYSb3%lRx
zZZqA+^kqCBU_l{tl9U2XTn&FFr6N=<=}|&usPuIhXXqlfVKRwq0b*jy-)FvC>Rd`_
zb2)X8PKA;_4I4*vXX@sc(L`f<JJKG;PD^WdWYjvJSAOAJ#%@4?s|H-CrKat)L}SfC
zLofQU<lq-tE+F#_4f1Wv-Y-Zl7O&m`J;+|Y5AL77{D6}K>_E^VNwldVbTB!hFQ2Pk
zX>TwHiEVoJt_(5Urute8p9bnIp2ey^C^&{i0V@I^)3x&7sW5SbM~M*{Kspp2yhEW*
zl$yDu!{ml9R3a{Nh#7yeo@3bEl;c_bxrn$(f`ZGbkA$Kw>nS`Z-UX*bB!mM{1GUZD
z@{=uipBH9^d(lhI$wY0_p3S3A>n79kj*2DPh;7)_f(TzReJS3D9We57_OAEnJfRl>
zOCSO%BzPmL!^-B3;saVoSErqK;nxRPb}PCo+jSJZxqcye5mxSu?pR{bJ)!I|M--?_
z1E>|h%SjJ64|B`Dq-L!)A>i3zqm-T?&uiWg8;^;mxn$O4-;bRl8ZWzZ3-gG04wgL4
z+VRCGG?E4&o`p?G<lP2!XL6dL!d-4F+HT7&iw_$-SYvr601W{H`oY|=X_^VFO}&-G
z8mY66GzXY2+Ofvy!^@`X=ih9Yp_3i7JAG*|_e0h;uj329ia*dRs>wA%_q}T>pvFqz
zf$o;3bufpq$ElvIyb^>7Q$qwGKWA<3p15CXGO3$ZUe0{u+zB!pT9~YWF+McuN=iAq
zQA>g#pJQ&NfklNv%}yOW!jAlwU#d_9G(+5zC8ZXu5JEsq+{?xo0^$6|&Cpw$I>9DY
z*;#F+t4FSBRwE7KEU}pm(6CQC3f1X-wotu+c%<tzDqB@z!rY3*pgY1oqy;WR{{TUN
z5ku&-|Eh-l4G2rTTUGxP(K7k2XF~k);DXyxF7cgf>ZzTc*;xA5eq9(ryyIp1m9D1d
zRn8#`ct?wTv34Wy=7Jg!$XB3~P0cMdnG=n6Gz5_@XQerqOMe_W3Vnvcj!x|)q}f|p
zFQR3c<p@GNOsc24Jk{hR2078dG;Hu)b1SvHUw0I!X<q+DI2kJyqIHy;%95-^-iZb9
z-YS?$i3+PsB3o%YlVGL%xF3P5w&=%3mw>`W@V2|_w;C!eY0Y{`cYES=lALg3WO!lb
zN4^y!!U(l{>FYwn*>MIwas0{6p1mXIn`NwblS~jMNbj>TqSlr6?VC>-6upkVYbAq&
zFJ^7s2b2@WF&Uf6wXopAMU<a8UZ-Q*7i7wg6J1RE1;>i^mT6dQckvfSRzfP`n_$d0
z`RceQTd1wF1Zv=WuYf0z%FG)JBIh>Y$ORZgYxh2kKzbVDC$}i{B?(SfOasOLTZ#~N
zP=(+E4C=G;^UboC;QXGwb#Uef@bD@FfR|N@pn0WwZ15(HGyD2WA>hjw-*B=n`C&bP
zZ!>>xdbb(KE&Alx+syGWZok%^ptsz#o*}Yhl20C3BJD{=|DON!jPaL(NMcs$`kbGL
zmT8tUNairDzPvacVY^HeQIt3%pGW_qB9?Wz_-@;gf^v`W?$`G`kP?9l<8kbKY?7_U
zVi97<FFuhs4ekhB_Jnpvwpv*BHjJ)<%Q&3-6N1GAxJ!BGFsZaiB8^NAmC(B%-y`N1
zcB@<hnK=GEjK(0l)sfX2<FX&(R`Zd_{<OgS-%=UY!_uqh4DjAo`7i+$-|xTS?M<K&
z>J4gh*PRvu+d#_w07j}Kw@I0p@cLNnPrgTNH`t(lt^rA7YE<+?#9aa<OM=Do!jfP)
zjLS(y-wkv|HD4!;>6_i(eag7y0}o3e8_HfO2kFV`e6Et<A<B_rWm9F#MgyB-hWGyG
zNWy<sHif=9x8Zb((dwEgu6)gxSgo>k$6Y<LUQ%xMsay=0&CGreEs)UBGoeo|f_08)
zPwI8$OCST+zXmo^@6g1^-Y@|08-3TlpN#R^KsGOrZi4YqISBe}>Jq_o+nA9_y|<t?
z@6No(lGZC3?0#1~RP)}PAPM?poFif!2k4-f(f8e@Ryxv#WsUIzDGzLl#GV&X$wlTD
z%t-}z#NHbl4Jqm-K4qp6tChrw&dzL7A)OKY>%k+5ph60yeQjqEv=lQO(%DKs4*J^<
zX$-aY^c(!6i(iw#aU~Sn$EwnB#cM2FzCkcrUkvJbUi-fVW5Xb@ZU0=S3>tb`xO4_=
zjREGCfwYSk2h#GK8M;Gl4F=(y?<L!hdvpi30kHk22_Kv&<c8YW=xqT3=e)c$G1gMI
zEJud?KSPoJl=LxltGdfLb*VN(c6`R+puk9ro|%Gs-K2akCesA;b)g_V=&rNL?Kwvz
zX~9CFARkSAGq#DZo@rVT6bVaWj{Y_%H7SFfcw1fj1Hi>PWH^m1R-x(>*5^!@0g{0l
zjMK*Cb2Na#xUKawdjCeJJ}1Em55^0z)86XMi|5es)OyZ-X9Oe<46p=DU%mkL4yl^r
zgT(%H2hgR{^9Agi>~!ClMG{oljqS&vFJ6MeY(F;Djv$-+@-cZ4%I^B!q*yhEnR~Jz
z{%wYHU?-7Fv{D=^vH{trK{!uHQJmjXMCI{GRx{=}$`<!aj;JmQCt~C<>}``l%n?;3
zCWvg5GRttN?;V-WQ;c8kaOYbBV`7h*7mZdnrL}<%T~J$Ac(IKe_#Y$6F?xz6u(VM2
zx};U2Qt#ISL-cZ0Drgu!pS!-!AVKOzfO`~v^HJb=#)~uYAL6t=^*VoPG4ir1REaR#
z*tPwa)ep(|K4i#nRc`7>XD?Tn@@)k1jX6r-?GG#J_(DGb^$pugeBt1Mi>#g!_z~_e
zczRux5iB6>K{K*6g+tQ)-f1dOF`qlJs-WPhM4du-3j_yw;q*4%-G}A*1vtkN<89D6
z`|p(Hf5G&YI_>Gc<uCIYx!BBl>WK@_Ug?M~u-MMz`l@l8q`~a4gnD{-R2aet!CP`B
znTTIcA>lR+0CeI2DzTshxwEe9SNc-os*~5UU;GQ<3Z5>Ht6wqK*PE>f13V0yT3|Ya
zdY5G$c}h2{(JTdWmDJ6JLQHoVnsa&}H&qTl!u#?QwnNI4vgx>R6ZQmm9=i!<pYEzK
zFQu|~VS0IWuX)Cv>jg7j&Hg#fd5X!)97nn7>SXX-FxTqHJZJ$Oim%LkJ;EE-d@J3b
zMK4jZ5C+$DRTpKZ`OejolC+)SkR7bUi@#kQvC%-_RL6v*etmKA5#@OXIXEtIC^XPw
z`kTXVxBTLLcmP|w(D5sb#ZE}Zf5GEQt_}9Nq}l1w(q?$GK}142npTKgVc%_$2zcgf
z>a8;_$YGnq-EajGpHvQ?*Xe#Q+$EY0tQ8wQ?ZHg6ESo+Raq!Vy!~%Q2R?b}m-JQXQ
zIEhHP@sZK=A;(1AZw%^<l!^{>4Y*M|vGX)F&#^AgUAkpjjnHhg&H|d3t^0KQwB@<<
zYe=2>V%ZpePsFQ61hvC-ebJ~R%24@%va_0?cKfYqw<D^1KhVxeFArKVW3eVL@yzsM
z(Gc>ZCt-(S7(9#n6ezJjKbmdlbM{6^U}ZHuAJgf!J##3Xm}PO|PZW0(S6ik`EK{R^
zM_G9^D^U$FRn(<7w6SvR*JERa22H2k(0X|W*lg%MNF+oe3dI4u<Mz}@zd8>8!lV0#
zVN;IOY&-EwXQk-3L5}R}Y0I%%9TfrrnywV_A5)3R&Im)zC!-n3BQ)4?;cEv%(f(Mt
zxHV@N(`woDq2kIS{I0D-j<AIENS%aicJ0K_;k~HFlsiGkXKNUzE5-Uy90-ITp~?*Z
zdXGOoamg>f?K-j9YIZIT%!v&FMfxavV|ru62Em+JG;`OvC8Xgc9WVQ^YTGu%S^?Y7
zuL&$_UdDFsus=!wat0^?>L~eFbfI`b`7g9M3VTAi<X$MLkt(}ajFb|RNLY&JTOxfx
zm_CxyEQojU6;Sx`C%o4}P`F?$1{C`Z1xJ2zbWyg1@4TB?R=8Mh6c6ijrOWzTp8>0X
zO=mDR8XMllqC84@(VlvoCf8rp0XL;h1m*)y;2Nn2<mUhk48{EA(n#o^I1;BWP6|m~
z3Q##pW4GLhum4)a^rC;{>6Cz{sm`}>X*j_5e6fQ#Xq;2T{q;vDrU!$cgQH0FV$CWW
zUehF1@mBq!X<{VV3IJHsml}9>N?Y1k>?rc^L%b1<1Q?$OWNo-T+AB*53fW)6?U6Jr
z^cOD2)F`y>^fspwyIL+3xUUP<=<HJWhjfk%ztJZSoni^J(Qp-u`sT_-x?1^_V;CUO
ziK+b9d&B`aR|PjkZwuEma~$B)^Pgq#v1zuc**+I)H;1={92!O~f#SBse<%XaO7VB*
z@3`XIe12b?*K^VHEmQ<F*WjqO%DhbnCJWSCcAs|@H~sI}r)K4Gsv^q4lz#~$^Y+iP
z90_pgAXnWW#`#6!`*Zp~Pic!#BK136>|%VZg_DBzXeCLs*6bI9PfLexZ&JH1+1+1H
zE96GfI+Ogszsw+b98N3)%}hDlN>4eC*v4t~R3Pt{y+=Pok=WqHs+6^YADukrYw(B0
zd&EnZylckH2i3EwolBCwwYN3`czYpv`BA(EWEi;QanFvacduV<+7>l)CWQlre;A%o
zZ;P?pbFI7mn6x;7nIBYV$t1QL&(buTtz6wd+kKxziqURZtOAIKi{R~d$sC5s=o^j#
zrV^<yQ<B$$kCsam|2&+0Ay9=$j7Rggb<wz5*9Gtb5-ay)G8{|DhHY>54taSwyx4n*
zw}LKH;X`FOaJhrJq`7`;_GMdLy*=DbaRB5Ce@0gk2tR*CyNwW20+!d-z9l)YJjtw%
z4WPV>uSna=9x*NZdnj}kss{SmabV)Qh2LywVdKLLjW=Ez`KQG}({CcX;1zQYKy5Ht
z0zu`gL*YKG!DW?)(x|%211Vq<XdqvtJo`rP@2_J018v-Qm1NM4Kg(9xexP*QPx_-U
zCbfG>-G-Y@;%?)`N;}xwo#v`E>6r`P*U?i?D&5E585Ws6-E&_hj{1sOA2)1^0yJ|O
zsF!c*%CN|IVCg9FGSSzF6KK4x3uvKBEW@cmCI@q@d1EQ1U^``5da@2Ll%u36Te6}I
zjqL0kOHpb@mY~vdMwuAjgl^y6a^<^~XB3~H?iJrj)hL-$k%{md9nw13j2CMAGe7K|
zSMZ0$fi)!_ciCB7#-fWC^pt&8o5mm2)=%{FjO&~BI;g3;=~`)JXHZ<)LT12v(98@R
z!5C$x_#UfhWNPY%Ib|@__P}=RpRp<XpGV|=!UmU!Sq!KgB6c-bW@e^W(Uvs^q>lNL
z+}9EF4FPn+y;$k4Azti}Qz#s`fpH81QU_Mg&?mwJ9p#o;_L3Yj(t^+zo+T|bBw+@R
z%$F(+YE&C66N-IUAqjpU5UVbPx;-UH;c(iU;%sNYS~H%ivZOI<)9KxwuufcmW-g{a
z<C(k&<7dsWKjWy~8|dlQ(uWN^zP&!*7ur^yxO`=1@<2YK)Z%zkUI*qu^7hlu6SDwQ
zjR;}}EhYLr<3P)2DUExvQ?ZhD1=xtHW|n)JY%yIij+>4HUV~X`zp2~ZdnbM(ZhrU6
zku|BU&EZ)u%=hU(()=MTJ286yD<5EX)M;;Zp5=kkk6(>^zMo5IwI{kSets=uF&1x-
z>G3^*0}(x4cI5Q*z@x}fDzYf6Y??SR?D^y-L(~Ln_wQJ9*0uG7;t$G}xPLv_K3?*u
z^z5%3?6)7*z5*$~kHtD~gPtwKhG{{=u!N0sDl!}lQFz~r$)wT-h1KzX#sSEkD(yM}
zuTPiFQ^!`+DUBZ$S3~k|&!-m37N86UHZN&Rt{}amND>}Lh#E}7L=$UIZY~uS+f2l+
z7G;Q?6_J8R>#JPDGnLO2;5Qu|$+RsC0IbCsOFL<~m`s}UIvzD6&t(iWw@Kd+A7J=<
z&l&_8%+osE(78=vS*<Eib(6MVsjU9h4W63J6W6=x$9QUYX{`@MMt%z8e-eBeO!L}H
zShKvVr!G$(LJ_EsBy{Ma<Je?Wvh7~;e8_9A)y}F&B3!*&vzy2FaVrCg^c#Cu4hmg$
zD6jbn&P2WNifug!jE<D3;n)~AEtSZ(kDQb(sY8K`dyZruFWtYV%lSvti~Xh7QTqz-
z*eN!}it`NHQI@T@rZ~c6%%3aM;9yfhW33r)A5DDUym)*tsP6b_{<%T2poqIAUv`18
z*^T&#bcG5v7qRn)Rq>}A0QreiV|qQJ-{rdl^yD&~Z{#K5wgz&~rs`YxzgC6v2c+(&
zd<9{&!f(m+V@#+S@i@i-Z5QZH54G>0vZZsU0pu{_w7~R;VjY)>N$PaTmfe3zS5TH3
zHB`DVOt=c~Z=f^QQZIAFT<bW?aCNhO;ZkIJelog&r7Qe|c=d4pM0*{*auV3~aq?!J
z@##4IM_W>*{vCNs$mr=J=J|px5Fz-`_6`1ioQI^`kRLPBXjGRb*KaHL0Tj>f6Y7%E
zm?JsB%l2ocR3ASKD(;(-!Ot|Hh3f>7{)cD{724n=NFvL+kJrPJel7*WBy}(?p}lD5
zgJ{sAIuihK>9^w`Q&;}Fb0SEvS-p}(y~pENkvq7TR(7tlz(qk{@KJ;Zn5frvVaV6t
zH_t~^Za63U9~@Ah#4WFqIsNK;2%^e^t75T{lM4d+KX#^kiRTL?P0UP!$@du`&j(^!
zW}5HO%SL1i&E{2T(^6>DmA7~?Yow(BgP7>(%5G_}?6IR1Jj$pQ1fjRPZ94z}m$;st
z`VctJ;k0rrjB{^qyJ$3`m%lPaX+!;27R$IIWA4Q<t}x6yAmD~sxDwyJlN+OuIwk4=
zs3d8Td+vLQ1A_GL>5;UwJnN1pn;Gr_;&-{?X@9mv`r<s&b`)JqJoiuT!BanDu}kmX
z|1FWMC!e%l22(RApLt3)jmkd<l+`ZONr+V25znCyj==aI=!!)*8gW9lTj%ijE;
zi{G?V+cmomzxT|p0OMv|8iMSc$@k)8naN7;?2E)LvCgw$rUsMm31iV_isY;+02EF>
zaU*sR3We+bFb!;+bV<@{{S9Z_Caofh#j&x!Qr5ZGvlA+MiJuylWa?J6A&mH(J}FDZ
zpuXelt=ZYy>jE2PSH0VTgizWY?VKyt?RR^8afOw1TQ!Fxb(e5S6;EH5l=nm6J@eDq
z_#kd;r7F@S!!ixYSa8kA@3LdXc;$4J$Tta-`bHJW+1YOm0lhQ{=%uBv|6_h?kk?rj
ze#Rp7bV~tFENTWNQYX8x4Q@(An;8*6uVM)_KblOsM6~qWHDA8VFx#lOj9jTzBsxQZ
zs_fw8ftUQ({mZf>Mpiu_0>RPk9DZC}+{DrJ)~>b&fC#Z-^82r<@4M2-AouZ#VAJnQ
zEJId6M`FcytZWoNbJJrtVyP&A?8g5#NnyX5jV#+kH%szC@}>J0Z_)<XxG^tQf<`8$
zF|OnUb=@FQUD^$?ACeNlob(88_<q9Fp|vTEZ)MJy*%ZknGN-0CLW6-sSg(I6*hsWY
z<kM7an1|XhR3Xp*k5z+*^-9;tQ4?4$aYILk`U7n)j8RSV2wHUlMG^wf!?gB;>~se|
z)XE;2uIZmfE<G-dF!t-(d3boJo!V9W=wLe75!C(sjBO=QqnTLh&tTz0({^ewD+|8k
zCZYeDasZH#|I0`R<frW4TgGKsSNCVS^Xt)VCygx}OjTVNwz|N{n<VYQ(TDKl2)%N=
z^Rkzg-Y(UV&J0Tcqv<Dr)h(pyP3GgB>o6}%t=@md<3`BMqtu5Q7Ccziark6lLWCRu
zaq=fFJT}a=j$ZhQno;Xg@ul`A%viI2eVm}F^Q1AqP#gi9H{=7PX3?uEZKjmbtqO`J
z<>|Ql9Vc@;8M2TT1@}O_*$rOAhk#r{Q>yhDP+>89FN08NxqWZU`+vNT#@$=U#B`UY
z1TXg%L6yL`3cCDA93zG080=uj44^LKL1tO~l?3e6s)=RL!rGr<`7EV-V)O_S%5)PS
zF&!b_c_~cJ)mXB>ZRIWvhdyhdA<vEfVJ>q~T5BN(aI1+3PxUFj|4$kI&ex!YE2i7K
zw=gT7X7#B)mZvodMiMRX_ujkLd4N-lSzGcFW@k8QMNshd^L0cIk)D}|)Ov}wC3;N2
zH8v_C*5dy8bXpSClz1aJaE_Q%?fc2MlmD~b$oi^Z-E71&;}ghpNrX<yBm?lE%q(5V
zAB*FTIwl275!tY(M2);*ksR%~DKmhF5>Tq7<iAT1v~}smjH<p!OKYxQFo?vY3?;Or
z?d=$68&oMM&>>dd?sC!_G{u|%>aI=lz83E(15nP*DSEMJ<y4%1%PM!$Rm{GEC=?Q2
zL>=VQ%`OdWgR~yzJjvEMv9%c(UkoIzrcf^~E`|^rKel-I(If$RInLK2T6HT5Tx<*%
zZOyePzzDO)7A`LE85@#=DGxWpDVo<;-FE9X?Y36KPV;R(_OC7bSeg%n27=sf8#-ms
zWaMl-cFbkq2R$n7on53sgIj;Bgy^b35rbk(1UV^ta{<+n;=|zD-UaME7AW*^x=3#>
z^Akbg7-fTiX+!i6clu;S*>kXXJgy?SYn)o)CY|Tu6#M@*xEYWHdWoYdL5Q2hY;+Ju
zxJ=H=JfJ8_0`<9n8G7Jvge0n8sL5}i!Tikr>>;l)hEc)$9830FP?hyEVc_`;@~XxH
z;~b!#2?Km4#oA7@R=xT~D6t5pW_?G2g39j@5q9x5`s3YQ%^}YGbGxA%R!+(7JD;}L
zbPQuMy*qP2D+3Cru$-@-TE4eRgH9(kRf$y{$Pbr!NojUwRya!oeT_E8;P~~1I$tE!
zYX3FVX&HB#gxKs)=0zernD4H8$<@Ie$D6Dn7J4InEb5!Zd%?=Hm{M}ihN9sF0ZPg-
zjFb@?oT=I64q34xE&o<G)KEKEqi<OPriKZr?j*kx&>JuP-~H5IdtME{{E_3eM;PN0
zvSvLQ(r5QS1Wc%LKx@S@haFD6i)6-Di-*YIwl*H{tRwGnGPH8gy^j`x?Ecl)D6i&l
z^cQM*Ta@-=xaT~yRQ$H47yvCdZ79}c8Mc!h3x3O)Vz$o0rL5Cwn)$OgP1T6-H0z~B
z2kQ(P{?7+4T`4{I+GXbh4Is=IHt-%KH9n?(grl3$d&pcInDXU<KY>1Ac|kQxa8+-W
z1hO->HpA!~v%V!rgAzCC7BC)*Z59;Cxu=8C*uj|6AW-XfF2iLfx^^DH4;x$58PSTP
zMGcY_R-E|m&|RnSQ(0<ojI&qV=ZG#}Nd6`()+=GARKYNHfH~!<3jrUC^E<NU&4)LE
z=Nw3@hK`CA*nmqOA=-AkKo~(F-Uc5N#BzGA$?E!wafK5aFCG+G0rFUM+p9Sii7CY!
zoJTsfHji(c4uFOWRKAVj%TArT*6Vi$x6(?Al?vpcu1|%Tx4t3fYJK{HQl>wW{P>nH
zldLr9u*yc{3N7Y!pvH`T7G5w<L_q08EHWpzi*O|N1(SpP?gDxtW3-2m<u~rp&5JDL
zO>;UuN{Pa<jmrmj+Sp-FjF*p4f&~*f%rmu8>Y5Z2&k@~F3N4Oe3d@c6u}Mi@%nn_R
zyBhw^KB6QIxMJtkyPwjxl#8m}y~f;cMjtLW?HGPXD$TUK^dr|&G;&#e;Jw}QUqE9u
zz_}<VZmpqjGY5z`oaj!sX$&H#bNGD|X6ig==YmWx(wJ0<P9l7>S5_t`9XWgfD%5({
z)-|8)&{FkQ!Fyis!Yso~7!ydvuZ8%8=IEDuQ(*w5WVdLUNu@?T4{vUsiDRi7xa{Gp
zC?U@;g(V&eiuOvxUAOjH`6Z0(caE1}BAGog{K5H;%_<k;;VRpikiKrKO6veN69p!U
zE*W*v$5?S469AtwTId-f$2G0^#u<z$^Pf_X1(^Bd@=a=_K)8OFXGE#zJwA0VJ&uie
zR+?ao9)qv)9t$4KScuw0##{Cq?1P50Z&6IIN8QZbHZ2>pd4L4HM7E&zF1&`|BW@_4
zvEknLcXNE6PwQp_j~_&-j&6@w5`ymmiocF6B3CW|;lsrN)D`jG&2gW_<p4uB#}b6g
z2?UhRoG7cTE$$XYHIUQqM*_jy?-k1eCO<SY4tPoju}Sg6HBf7fEY_JIL_1&MgLJeA
zcGYPXQGAu;q5EE_v$(L!9tD5GbSTG8<+z*Xv{ALfrS9cPRz=4yg4IZdx%5}z+C);y
zJt<46Adb|=0!eUD_Pf1}_woPr@X2N~B`u}J`HQyzQ*{|IRsUKPfKS3HRN|j0bCSNT
zpMiLGvnBAo;8nrPMjThylQoft;@ag0y}o#xxn|3TC9BZCJV(d9=`M9-`k$`kIMe25
z`~<rVR3AT`;$7C%6wR#xp@W@d+ut{w*HRC{zG;p`Pg{C(ReVe#UjR&w0VTW6d6pQ)
zutI}L)Z({3hdn=WX+IhLeJQyH0L1jSuAHoDP~+oP_xVy4VdSu19ri-s$htk$qbOOE
z($}x)nM|^U=c^PO@85L?$f&*_jtW7k6`Y=XKL<DUG#4Ib+Qzn6L$*EVQ{Jom6k?Up
zs@+`@6U&PH%>gWk|ED<t{6JkBzz;lQTa8ufE`lY+C^^r-=tn#@2SDMwS<MTIef|xS
z?-$x9w~4vYtKvA${m79dUIMB@+LvNd4d^%4*l{Oew<tNdegX=`AAgCV@kWOmsHz^w
zvK7bgZfiK++1+gj5xn}ndJ(x6`S#(H^HHPb`k@s3u`&Shm`LUmV{!COXGFaO2$ZaP
zI;Q8T+3Az5x?}k1E=k{8jR^;>%~0dQM=VSBB6{$+ULIM<<hDw((ALvdR41ChT3Cou
zlypAYQLO4n=6s)aRwZ1^t=OksLzxX=TzQ-oC(-h?n6m=&S_`LCJ~z&QdeOm7E&Vc4
zU&B1ylg`Ifi_u%<!=wM~`vW(ld&Rfe=}!dm>Qj^YBSuw38KeO8>S>J$sf2NX5PrUy
zMs@~T<1w4D>98YS{gD0}%e_y5DIM}<3#eqc-}uyVYIkF`kv`rMc#Lww`OM0)T<E&$
zrtlp*+a^%(#s}b))25=$G|5?YFn%qlOFNCkq2I>Cg?A|XpIxORY?QStBAXBftu1Kr
zyo5(t?&}G?yUTqgVN#u|q+h~{09Izy{fJ?@Mi?yE7;O1)DS`iX1-bL3(g&of;b22O
zS`MgEm~BW7A>RhZEI#s~aCZ+|J3@<-Sf*km5*q!hw!pnCNe(6Fa;ZrK5+3Y)KqSDF
z?*0W~(34JUl;Fs(<nrHw<)iD7st;x^I>xUQ(4ctfGhy#XR5%0sRzBk|N8Y#>VFWWH
z?ECA0m7p77E6U5UWTe(NCl1U+1tX$35l_MOi(+2sZL!sdTO&AMGJDL`T-xK&ynau}
z&t!kw-*=9}?Sj>8og486dLLxctk7gehEKYhb!CApLfDtdyuUQ0RI|_#Ui3e!z+=9W
zcGqoOT5iegWg~4SJm+}CE|F^f-j6*+v|Q)Gichgjod~9)%;j&B!v?f!_m@CoS&pSR
zid1HmQ%;wmw_0H)+Xp`MtCf-~wc%oTV4ELN>WfSaF?iq4vK*c>m$@_1oTj(Tp5=Ex
z2y;fIb`uZYj_%A{r$6^I97pGBatU|$_YJOXA>LHeiV`UqQX~O($g&@{H$8;wlo5h-
z#zT8W&RV2YuRUIxU0<S(BsvbfaIa^30KqQj#K#YDhY0HRppK&pI{nb{cn+Idaw;NR
z<HK_@lN|p6OO+LX7MwW=$TLW7Y||D1dtzeUlgc|+WGN^}!Woakn4H^%0-uqN^5j91
zQ0lRJXe;dbvRLx_3S8ecLWVtq4)*e4>HRVT5nO|pZ-=0AP&+`kKUV9g{x*1|#!xgh
zrCVt-hH)aQYn)HFFGdEziZ{DLwJGWs-(&cRN)Oq5_MS5m-@2ea>S-^<gE01ROW4za
zH_nsh13Kg<$QYJpzyv-e$sVz^OPChi_dE_tcqhF-vwYJLm>GA0JZEB41lN@A>EM_s
zvr%xuiDY$aMBVb~R}*85-Fx$A=`$0>Q+6-gsY27yl|+>Fsv*Lnlp5Ra-{$j19nj^C
zhGXO#D!rT=V#Uv@8Bd-ySV<1>IE`h(=2wT2=O+brO<zGm%v{)6Gs=j}F%K9PWZOZ)
z@N2}Q9fEm*$hco{RhX#&Ub#7%AJrm@05T>?G+zrLOK!xlaJL^j?)QxDS8|Q*9=0<=
zz;8328eFu)E;Di_%vWWN_x4x({Inz+bM6GY!~*%=OI$rQ>;%ymBDT62UA!4|@k4Ob
zeU?v2koA_!oC3u?LkTMGHjiTipO{_)J}Z{L_vd#@Lv2htnmx!CKgRIkZv*`)Pb}zL
zG`*5nG+WaIQP%A~m{SurN`JK3_$2i>-gUYCWvUqn@GM#d_S1JZ0|A~v(!}4?0FX5f
z*aOh$Z&A7psrL}5oe7hyyAe0aNE8W8=zCwLOt3@~G#Tb(1bf<a5Ba?>{^bQF1}5Ja
zi;s*G`;2RV6*pAloR!8^eYm76mZeCH9L&QVnwv)jxT%@XXLKv%#;jt&1&K+3S=iVh
z=`W%lAg*WASubn(aUiq@xvPFlSo<X1wcFTh!{M!<tjyvVoSj`>dcy7KFdMhL`WWM8
z<n+U*Tw54%SglJ#7yYMO=z2)<wW!s77KErCnOfgVr}asMs37J43a*yI^DID#7lhxS
zEIi#JJ(-QnxnM=ml?|lkzyh5kV(ZVnl{Mdtd7iRq&o511AOej7Vf-Te8YJz-tfqrV
znxe;BD3|H<1gj^0cPxW{BY46@9KCo{!jY?M!!4~JGh`{Xk@pRpJTd_TC;oNtz|;G(
zLK!`()mjFPTk7X(w{tbK4gPrBLRrqm8^h+i@|z{&>AzJh5`kX9e;t0~G>s3)M(sN8
zbJBftNe;jcUhaEQtN4SB4d@I`3Y;`b1StP+0diKFaAEWdZx6iIGIr0g2Son40?fy+
z@on>uOQFfPTirmmWGX^SyV<~eQ&Cl~To2j_0Kq-JkHj!Bh3RJEpcRsjy2o$ui4W;t
zr{n$bl;^t0XocMCB(#xa14LO|Kw}Z}4P8=+B<cM{qcx&`zA9v+X7{AiC`gV7M67Xm
zRfuXi@DhFo1D2{#$Q?Sj@EfA;r+&MrAl<LQo{)g;#QBzU>~TDuGk_WNuR};ZEBJ2S
z`<(Oo#erO-U=E*kcv%r@XVj<h`WJEpz<81P>fNW4Dlx9Vk8olK(RpRMOcBw?b^8Ar
z&>cPW!o#@!=SpMYSs0gLaNcJX#hXs5kAc6n)%zY5k%7?$?pS(i-CHizC+Ll8)N2vB
zOt3ht&OYS=9_P?~0Q^4}$f$yncNsVOIRnNwm6z;PsHgnQOp%~bwZ1Ab5MhRq*(YlC
zzQ4;<J=fp@YmAtHEitac=#l*GS-mThmS=q-VJjGUGkO5W`(Fo6%hp92OYqUYBxm$;
zi`4<E6D+M`bE36wUk4$8odSqIfM?&V)&h9%umq*Vt<HzQwe7Z&t7RjCR@Z_V)vzOx
zJ<{?R4KKRGS_`K200-&s3lo}xj?)F9R*i0Z=;)X?;rWszhiP}cedMg&9|Q523t*Y=
zlVXSw!MI+T>CgK68spfUccc=&9OU{yC#JuS0;ERyb*ME(9q)6L{ItqXqTR2-B>3@D
z!aZkTSZUngDw5O087ED9R_j~&^X(}Lt};BWH+>)&N#ViyFl3MTlVP|Tvet2QzV?ME
zzfycbmuoGpG>KN|)w73FkCO-d{Z>Nr=LfdRd|H;EKZ|q8ggwt~dYxu(K8Hcj-tkKT
z0-H>Y>M4eCDa=(Brq=i^h=&N65(I^ZE^pm>3gh-9Cgc5{AE#ElkDOt^1qh)c40N{+
zV8Z717=OWU#2*w0o7^cV)EZx}xYrYW2lx!nF~3qC^}gUYJ7r`=^nQd_2rPX-Tu{UM
z;?x;#(=J(WLYK_+CD;pC;s`))6u6ob$&i>DF=#0A;zITa*WkYDotJW-f4`Xg*la*p
zDSStiOp`-aD-iyKp~$%0c0I)N)C<PArnltmMh?6Y+@SYz#YdDtcOC2L5&+sq^aJV{
zxTx-H$EZU`h2vmvSWk=SB@<zJCK3X8(<;A5&Az$Tuz-ilj8l(>j%fg$X$=LKo(atY
zcOw+s)%AG9YzAk*lzyBCX)1mO=}wyI%a?+v1Q9b^jb=x<Z*N%K%ei0bdA8-n>PtQx
z)jd6S)IQxzpI<*Nwmr>q+TG7SMF+_QRT>@Oc%Cx3gtaX8K#HI4kQdJ`)M~fdgXWT)
z@%7xM89mY4%o(2x9O3Zx=<$US8uiXCJ~v|YIneewBj$yyZ$I5WsJA)wGS&*7C#RlO
zoX~JsnEv!oT7>X>)>@U9T~QPa^Qw9r1v4-qYgEu6ecoqMT>S!c_~Jm@_krW5I$@M^
z>OJ6iykcpKOur{_d61ImZ@F^&)k<8|y{$uXJ@6U0!$W`I#iV17`O2jocwsRB?z!nB
z@RGQ|e6{^_BEdW}Ecg)2_e`iml}z}gG>D^IhXPfo>w99%HpALzVN3viUAZ15C+yD{
zeaPKuDZnH@=X$$7A-KA2BZ#a~&ue^u42#$^@NIzi;j89B_I3*S^@7>J{-17)k<gMu
zibzaU=+NPTmqF;=9Dae`67+6dp!Zc(ZFR7Ijniv7;=|hnv^$xXVGQrf9t;1_!%_9_
z;Q?58sQgS`=fh7xolNhB%$rHFbl8mW9nO&bAhpYLMy>lVzx!aGNI7UA7yIS6oDW4@
zdo4kIrQnYT(|(l+uZ+rHhPCkFc(Q*y*leFGhBabD?a@QV@pNC{z5<$7b(szw#d(gI
z)K-0?UnGoZ_URlO(8T8`*NV8OGKH$zfHCaSKR_n^E);xT+q&<~+d=AmH5;^MZFOzJ
zwl%9jc}$e{=Fk7UG}D#k&$+?RmFmxX^oYV*@R+X7syz=(BC!`f&*W@B=?-_C955l}
zl@n{-0wff)3AT8x&c#Tu&jGzsP~eQZ+L}sve2ZQ830Z{p8W*>7h2Z6H-0BhaL&T5w
zHlBCt;Nr)$+iANEHSYiAhyH8WheY>R(Y=V9uYsUX-yB|R!*|LZPNY(gVW^^=K`n1Y
z;EZw(0z^u2EmK0xHdBx=E7x%c@trqG&Z@?Q9m68~r#6eu$PKmoaggW0m;W5lfBnv1
z(`&Ik=X$i%%~4i9((uX^?C(QQ@7U>~msA6=Fb#4bmGW5vHkAyM@G&Z&fyAlrK1jAz
z=B(;WynxKvKUe{J|9d?`y6^9tRr|AEaa_Hxi?*%#Q^^}UV{!0M8pkEtS3MDxbr`@)
z0^I*1QNYh9=>G`ezvrdX78tP34Za1;H#F?fK(e6m=9rWW!LaxS&8HokfDvxzAIz*&
zqq<$`RMRfg9Jcx`vbN8J#BA;_PY~N4F4whv0a+u4oXtj-ZjN%E_p09-%>(7`ZTOxV
zH}pgtLmCy-NH~u(=Wvh=_O33nk0h(8S6hocJ1EDiPe-4vhvt|MM)j}FfTjWI@ARpA
z&wX#mn(i0(=LEgZEGu;>|EK~$@PTCrXa=JH_}7y&?6E)H)(D%vA;SCV=+!2gIaMQ1
zTBV-=PI$vR$Q=5Y15-$|ZBCFb_7^gJ9l>VHUkQuEO-X{b8<D>Eo8`odlII|{<F56L
z7k}?5g*Cc00RPzwFhF4q9A5uziIpcTLf_2>(o(P!22%erlc1^_D3^tP_SYc<2W>U~
zuOs8xUpn~jHxyt2$^F-dFa<Ec9`WygRbYU}{qKKiVE*hd|Ge?m57_;I!_NhW69U%|
zXX@|uXaDqL_}2$qTUUo*+F@gW?-F|G{_hv~^V$FR_x=C>EX@Cl!tv&>0=YD;Mb#!t
zbK!TKe+21gSKU^bF-bDrwG&F*sKv~HS&g5K*|GLA;~$Ha2H7zRX4gL8kNgUuiFbe=
z+So|aub46itOP6%|2#3FSAWa55E>;&Yvie^rB?a-VHM$@4|ibxmgn~!2Ar)wJ5H1u
z)RBth|2|XVpEDIhS_X)KQX=R%PB?$dJZ5fR@c{tmhrTJ)3xN9n{1Fw>@b9m_`{%2L
zU;q78iN8!G7@_e8J(EbFHU@eg50GB_^GC?mIgO`|F4XRU8LRwz@hQ9%RG(kp^MJnT
zhXVBe{PPFP4Nz$H&t0))!{1XlzN7+dO6Xx9cl^&~Qen=+V5q|=4_psE#%xml`M7?-
zF^(UoY5^`c;^!{zhskD;?n?OAsZ5(Le}4u8v-6D2zz!TYqVo5Obg$R>`<64Fr%dy5
zN!jMLcXy9X%K|M4pcnp$-Sp2uoe*N+|4(~Q{tx9B^^k3nJzIoIRHIDm5EJrEWl4%G
z*~Y$Qo$Mm(SA~#0OQ{HrHOyok%2o^_`_?d%eaXJP_nDUO`yaf&ywB&8ewh25d+xdC
zp8cN3=d;db-LnSVSy{~XYP12K=6@%i4O~T&x}G>*lsgmn0Y#&&C)oQ3*`=8wG^%98
zCQ5yXIO>v-yUg&gU4cGUL6-&tb{o`xCp4kY{~?o(w~7jpf871RZ*s3)n7f<1?tlLe
z&8WxFXq8+~hqyMFQ8=!kt->rTOR`7Vksd-hK$Gwn<-};N0ZAS)HBA94>F_%9`6!xO
zlfg3icSP>NMZpY&;2<_0m<-VCidtiTr%_aTif#}H9d&c%$v}W~iohV&-Q+jrs^0(E
z><nebO4lQ7pq)9c`>a*oiiED#?S#*Q=kUiFAb_y)Y?D7UcC$3zKX5kmUUUk~=q;rv
zjG4N3^Yi-`JM;1>Cw?E0vOb*^duJLKVhcX6e>`mtjsv!A<G@N%HQw7675|Y&vtwWD
z!<T*!d+Ej@{AzhhvTLQpz47s5Izs$Hupj4dv7d1>F%M!3_T>)8Zb?+<$fP&b+m(1}
z%XUUz+O&2*{;#J8Gj*`2r<$4OnQ$0ss`rFpRun=>BX2lqi=h2+K`YHm;NS8xfPk{1
zdQX&~{m*_-I`da!1iUAWgci%&={dr%Cr@MiKOWg5!Lt$zi1A#k?AKO9aMUgHF;R$V
zx&x@xqe-zVa_$93ezP(3tKyqtZ{{D98%^)gmC*?Wq3D1vuo?z8@-I$@h6IEwkGmxw
z)aBrqe%cQ+$~}z{4IRCRPx?d_<%fQ!JprbkT=NF&`$%0cb$e2_GO2R89?mf|U+dR^
z#uzUyXMxb+7_b@d0Jh8XHJtuu){A)zEorBfHY$m{vHY_!5MWzQ<y{e95@qTx{7+#j
zOYrYHR(gAaViG2*-ASGmy@vmEtuNGm$vrQ>fKm`iV=WrMM5KQ85c#lK0FHD9BmJ##
z*tlOUB=<ixt2mW6R13azw>(8BRi9rVX-_V`!^togoAL{3=<f;8%Bbd+8F6S{^X9~<
zq_Z+gj(@iRR>BfE!hlb*dJmNGjr$>$P3alx8{2cehM@lS&qH#)GbeU3<ZN+iRv4Ja
z;Hbp(czTAz+nFCB80SX=zt|QU{I7vndS?aF_SVYc0BrCd@ab6op<}2HVei?8L9yc(
z{=kC;m=c6ri|+4{ZoCH@dB(G1-cZQvdiG1OkzB9u|C+a^b;$o|Jh8W|G>e}>g`aH?
zmUldO+meIBfufmY-z&89=D%nQ;p4yUxBB``w(T-~Z0WJQUt(m&LOCR=N0Yr;>4K7~
zPyStUKx>&r>qzj^LeQKC0kxD>?SpS#kL@|J^K3mz;@oD0;9oqk<-@ya2b<H2=6rgC
zz}0_k`t`h$Y^RT6{>_BUrIm1=x^Lf6X3+31*PC;2Rz1>pUl&II%<_<s>2IzSM!rY~
zVZr|5ZOG*2YEX^w5Ef_YCG_1X4^jyIR0LNBW&7m*_4|unvv8uC=s<;C#K4zv5HUim
z-!{3I|B(cOvDPj30C<6M@D*geru3p&AN+F83r5JC<BYzD@xL=S@<NGoK+@wq5h=ep
zIr=b{d<w!xyJmW5=k?a$5I25G|F8G2d7#<~)R(h<@XY+zC0NS$kZEmB^y%r23ZCz&
zzG-#N7N&Lc?+|2MEOaffU=z>3&M~VrruiLIfjmi(d0-s<9f-_;#pgaFBjmvkfm!Pl
z!F^mHFP~t9mM_o-SE+*CuL~1*M@bQ)RHl-*fv}b_2pMdG>2N^d70|5J?VFqQC1qXB
zHYysE<6F2$&qv~jX7x5U)10qTzycRP2T_eM#im;)=M97EcsZTe<*lpBT{MwImHObp
zNLc_4g}l#L<>@^Vg6+R0IencI8H=#nXq`Oxxgv?iiP$vIIo{@$mjh|(=XBM&N2~07
z`Sf`;MDVJg1gpv{=z_6zWc-EdfqN9M8*|5F-4#~Hlq=eDmI!9sB#;XNe1hd95YiM*
z=FPnB^8_;K^{H)UWU_^3kI)e8FiT+I&OQNXRle|bj%!<m@glw+_A}X<wFBUey8NMT
zWod00ZPBnvrTiAfBj*G!aTm$?EvKx$P;M<2I;}4XiZ17@$E5k!Wbu%bM2BYupbip$
zPHrIgNynwYGB_J4+TfF115N2?@-ryABNGs94Q3%-r6}~aTYMPuT|rT%3QuFn&t|<(
zZ<xS^LTB+rK_-X83i7uNqaDQ!(e7g++g|xE@8GyZBZW&Jtldf8hf--7VsIeeG`3f1
z9PV$nbcul+ww?C=3(b}4VE2u>3;4goy{VV<nqK={E_K+RSIyG}$dNGW-*6P?V*P-9
zNxD5e1g_u#_1^;MBD01}=c~1#6y0|(u6F|;wrwIx-&``%zxP_>#DYh-pnVjRo9(tM
zq0d0XCKveD``(Ef-|qr0>9%ypXbRw}#sq_ipT5@}<6g$LF6HgnSnbm24pZ`El7OtD
zi?JWOmUUY+U?gO6LAAGMz|N~4a<f++P27@3sj9PKp}M#?;d6!G-RG2bHNvE{jR!TR
z5;{AMVVAVLvbk+ye1Vgnx%Oskav{U%j+1}?aA#!clGIM%=<E4IRY}kDvJlIte1lUS
zV781Py*#68zYg)x)jcplU@pONMUdnUD5v@9xa1p^W~q#RP*&viz}shqJPqlGz9l)e
z^909vYu;U6f|BVUBYb`s!xo<^R-3W}&L8Wyybl1F{w&b{@SvAKH{dKH7PJtI{skA7
z+Q>Wg7~D4w#25Qeq+{gblynQ3NB<ooKFi2Ip|<{y&VoFx$<yeyD7&HuS2b%U+HWZ=
zLh2PO04$%8bO5Tbym_Y?n0<(4>o)KM>}JKXGD2Dqu2*gMBOYnW-&u6rwd{-?DxdAg
z*fYKlJvg$)ZN$qHl%#m}1MuF|1);SL3pA?_dz}T)IEvQ&m5B00g;8K=M&$t+ud^cI
zM97-$scrq`J(8}7^i{a>vi9`4mMN)EG9B8!sVjSkmAhpu7(H6$S{&2omj*?_l{B9|
z&!dzA(1T=M@ZE8kBJ)L`-P)urTcvx_UF-mP(^flTS}?{lbwHh$q1MZKnf=a6jySY}
zW9);b=PagA;QJL(SKkrYjJ^jV2P9BJ-bC`3+qrLy0P2JAz%(V$An?A2OkfkRi38ZL
zBJ6{Y)FzM}<9W*oA~nyKgSWft_LM|cHVMtcOU^~ow0OX&Vh}6tRWJOi-sMQW*ca!Q
zX1%&1ZP~oKK1u>{39Ny&38ak!*x#Fn>6Hyxz^i7=8$uGA@4=9!XpANiDf~1=e`1WO
z1j2LBUVPs(6gIVTcj2ex*ElXFU>fxzxeK(;E*N%uUCPeiRpl$Fm!=27tw^?28Hd67
z2s!d0@FMjlHnUfZ)Bc+^E739moNc37x&XQ?xDP(wZ%ork$9azdgv9JJX}-BDC%3w;
zfa2#`e1co|U&bS~e0Y5?%jc9{zwID8^T%XIWhk}5(*UyCfPvMfhmOV*PbW<Y5$u*=
z-8kj~@;M0YSGk_*50pcNNRSM{7DkI*+Y-cbGA)f;^&kA|V`fs*^TM?ZBP@i<iFZFf
zDc=S%oxvcPFh@>#l+qlM3)QAVw~`7^odi3{FXOW@61~-AbDr%n??spx(TlhP)XqUA
z&a{&rx7*ol+GI_gO=waLW@NFI+mO$$FHpVVJqb>C?!pd|h1b91AbUh9;uEWE8<1Qe
z|4<Q|F5bm4f#~LVH`zdAL=R3>GTRr&b3({cLmY_?7XMswx+yM&`OZSt`}dXFJSxU}
zXX`s?>5suM3G5!ey!a0qM^sQJ5MMziMtVW?*bmi0zRB~Wx$mAX2^T%qp-Ao{ssZ@*
zI&OdlMr1st-eUV}pK;EN`KRzWSy+t{4(`)&Rzf~VA<0muIDadQX8jCG><b&eqs&<(
z=vs`WWzB&?L99;@`=Ya*vr8qmj*-{pnWYlNQ7J%T)=lU<9)~yx`w*C<wzl#g7d`iu
zw@WFW!b?u)ds>*P3513(pE=Lh=syBR!AKgPo;W<atM%F#qX69iFMhMymX$jRqT);E
zc5QFrt2S*a2vsjemV`r(?3oJFJr(x>dW+E}X-W)P#lnwDW^6pZ8jMal7j8sMf8yAD
zWw)j<4Kf`SJp?y%5{`Bw**@>QGC0YClL!2GDYgM%ulad)ilc!Et+^VkZm1(#porHt
z7YMd~(w93WJ4cZXZ2dyZ`4|BxRC;hOZyfMCULNLj;G|oG&=-0##gYg_*+YK%9VX*h
zt6Fa;g#a@O?bhYc?<sm>X0sNQZb0Hkx@R{9HCP!-9d_(K02B~rDM!oiO5*O4Fc;HY
z%2~?(iZCErsfU#~_bp0xn(UZ_{Dz$dJ|Jw$=1<5eO@L4JLM9tFDS+{$eG=0Xjrf#%
zyk|-!gn2$2DELkpc-d$pCgqndUY+%Jq@;q2!a!jR=vM|kUh5p_Gkz1nBN<xWz!tzf
zBrzSn753Hub(`@QM8&(oPwsI9&BqLmHco*Usr)2lo<&P5?M*I8AN;1N+Xu9PBC{MM
zvA}&IDUaf5mb!xUJ0njwVNZj;!F@KuuJ0<YPek@bV@uM|esrL(0`%b9W00VfJCoyq
zr>sAw_gc-y#|{*T>-H9Sb&YA%l^qxigBwU?2(1Ev0v3;ih|6}Bo-h}+Kv)L8$q1S^
zK+;%p8os^fy?$GoKV5m4{~>DZCJlHmF}e=$?>_A)HsxI7QjE<}jb@a{)|w#17k)NN
z%{r+?KQ)eDE`74AyS!(O*ju@+Ml-mvEQBV|rl^D=?F(T!b0u^erbG54(&)_VZ=luR
zB3{Kwa+)Jwf~LW~iC^_8k%|~#sIEw*JdFVLqmja&P_eNqba&5u$lvwFLmDx2AVxNb
zhip;#Bji1FR}&VZ-ovxQ5sxcE8L?td>;^iEZu4nEavzW}snbVw%+I*ya;D|}T0<@%
zzS;KHnHp!~+!mB7B}b~X!Ki~HwDb7leWf}CPb@^Er&FOP?ewU8mF)(Hu6kdm)6#DQ
z|4r~V)*dWlL3uCdWiM4=*5VKGtq8wK1|NA!tJSaEb^~7zfx(_!1{M@aV>~Kv#<Gq8
zn%W+B083tItl`_?=)t0WI<8iKse45`Dc0~hv4yNI42VGvKay`dXDb?f$jCOmTuhRM
z7{{!c?zE*0kw&|rw^P^fbGpNh8T(ZCp8eS*xDRocE}w<oJ$&H23)!uW@#kF#AdJd+
z$oy*Kmr!T%9&uD{?8~;Zg9T-hrs3DYm-l#7F1u5vx1)58Mj@Ax_dYP+Um`6otiIBB
z>mmgfgB{zSF%S1~9iL#t?%CU!LW)@jE6J;mqn)$NVd#2=aDBJ(YxwYj!<GS_Rpi<I
zG{|w#5CE6u%N|P1)J3QHuur1ff2u&X2TA4e8;(=^0!LKw_JNLP)LKmTC_Yq8Dv$*A
zZW*X$dpjSMiOxltaxNg+VKvlJA4=l7>)FG49~{?I2C&_kXWEvOQ*?2A>$tjJZHx0g
z8KAYQpcx_BByiZ&^aXsc&n9PB=0_1Vtmq!;_639Mbs@!gBIO!^>!idCI7%>;EXbYU
zWUBRYEjuoZ*KvHvJR{R46zXJDOGGY6E}XMfxInFiwnOnsJDXLInBHLz4TslF8@XZ3
z&V`JMaMjd;tYPAU{|FeVEob&B%Ad=Nnm43WF$%GxfTjEP&c~y4=CR{A&NHQc1Fzt~
z3GyW2mMR{2cIi~SuyjS)ozUoO`+;WU9|A9EGn_<M0xvGrw{wE2SkC6u98h5ITt#7f
z?@R`xUkrbWILI&Z5p{3p#T)|MJ@-xo@4zzaq4aLk<<e`j7Z@;}KM40xAZdoSUG*|n
zsBpDy?9l<UOwNv6QSSqcuMK-=yfENa>*wp{IP-5bf1rqqW#yNUGT77GKX_KiAd*uW
zhVc{_9x#?OvjDiISMH9e<7r_8-t$mZW@Io(<a5217bocqL!ELAXD{E-kp)D%)6pS=
zu=CMU8pfGY&?Od8@(^}MGj$L)HzJ%2zPx2rl9_A1%6y^O2zKoHU0}v;A7^{#50=wi
zoxDGiVKo))chf}~+HY8PAV}U%vx0ka;hKkI#e{z}bS*#7o{<OXD%l#nM7=;sQ-=v0
z#eFFJx(_f(7TfK3U5ke7x*j3-G2;g03siu|7v6UPqe<1Y^ZpcEy!PhHq6`laspsWB
z{7AaJ@9rYaQPPHK94<B`Um`4I%YWTpWC`5|L9B{}<)l$SMS=i}pxTzRDAJugOEES)
zPBX<BAg+D2d3jMIW%F_sCxe-w@JEwYIz_?0ix^ED(4;LrE@mA=c4(=cpc*12jaB3#
zC7{-C9I~%qHarGdzV1%Dw+6+}_|KKbcVA5SFp@n<Ny1HvqFlD*C&T3TbA3)7{HAcs
z0}Rg)D>2>%X?X--<?y*a!)d}1Dx9MZTr-^MT7mm;>vs;VsAg0}Cj^GA@gGhRvV!Ia
zkX(k7LDhg}alpxT^}|l0C`XU#0Y>J`AM%f{Z#r+u4;|@Zlx1a~oT|PB1zQ5FUE#un
zoFR8vTLZ!dGEbj_^dK$~977P2N|e#cv#ln$*Y}9;giu?r_#trh+_KOt<B5Yl#24ol
zk}o_|x8Y6-snvpNCqBqC@Tp9B8G{~Tk%gloVQ(d7_I%<CujYnc$v><%(t}ee43U_!
z{b2_<>R$b@v@XXx%{G6k-Z77R4rCM4jUS<<fZM^+?YrV{VARMsV7Kn1MRV0%AIykS
zsAq5yC<xzrD-E(;S>JotHu2}b6BoA6c_BDtwIYH-j)du0qPK2bYQxB!DZs4Vp`3TH
z2IbTD9Z1S3f*LMWDAYP1xOs5{kOH_`!f=E0-*G30{2WPT*P=_`Hd$cvSus2-!TyE%
z&DA|6JEm4v%)6J1pZLG1P4NeBQ0<z-XBVLop+#<dYKzVD0%@%m6xq38^!1dIRV{#b
zMoMazoM-m)JP&-s_hb-+@oMR2s}NhI%b-6|EPi7EnEq;YNa2x`Qj>GkvPZh7K<hnH
z?<Pm<vjzYkma%`7uY0LO$^`W9I_urK^k(#KSttf6ZGjnUt`wsR%F_Y_kQx!VxLQ-&
zgf;IKW1%LDb;mB#nYWzQ=eVg6mrC2ZJeE3VzTtsea%PrRGTie2*!Op)Gc}n*lZ@y$
zUfu~ZE>|^NzEm>#WlFS?4TV6|*j$3-o}d|~4U=<yGy3Tu?PS5+9&a$U4Ob#O1Dyy`
z4~vhm5O)w^e9s>?c?a0#{0v6pf%Dtl@$&kn&if<Xe2hx-^+5Kr{!)o#Zgx;w!4@>n
zoh(0uESUty>&~*SWjJW)rcX(dm<d9@Ld@KiV5a9&&$zw`4*WTTVu*-$+f})NZgH3x
zMoUk8IK`jEzWfAkZG`u#xgW^}Q@z=Hep-j>_)Tgbj-0|{cQx<bxJhDb`B*ev0%C<G
z)p-lu%O|bUBR}0pE^q<<fm$`U+WE*r2jb14l)R0Oc8b?lV?6Mk9X*!}&qCcQ%B)u#
zE%5!EMW^NJ<JJI?NP(chs!~n8L(b%hs@h0}x0&Y!Tn=+~mDYKTCWz=vvj&<T9me95
zm|jS$=aM6A%evX2scDaPnBiE`ok*F-3Lj3bs;iAWf0#Q-rxK-w0^*33Z`bUKM~%CF
zF5*#lV?#YnF+*=@2WEwPUsfCWY5p1>&~B-HqvxBK-r=lBTzU1g?v{jKwiYz-b+p>>
z^9B_E;?*HME?45lK7Mn!a6r4Ywq_5{`o|}@()7D>F)empAJG!Ce^APBhD^FR-OkO#
zY|*{m#IrbGtZC)|bAXoBSIwSpK=95t7P!B!oa$TAvJVuiSm?^P%3Vmg2F(XO^E1=i
z%+{etYJd24u3Hsx$&s&Q>o)np1YTdXBBn`!!0jlW(sTGy-ceuFoRfGH`%XsgtP|N}
zzWaZXrAH2ugmuv)&yf39EuS06CHLnPekT@7`*@mufN;snl$f59myW$%cAljtCquG=
z3pMp?e(;y)oaTFb$rMqBF*|Qjg!0#W@VfbZh8BBC`WIb^Rjo^BWVMu~Sn{WSVC+wO
z{f&BSxyI}?im}`n{Sek2>Agg6K3Wvc)2pM`ESRx`1?&qnu=}(r>pK56Ldy!dPb;6=
z7SUZKX2>5|a2`lNhj#~QiC{2)fFlSskBIMJZ8SNBxw<8&ekSs&Hm(oWdt63e>+LXQ
zxZU3h_S?TZh8JV0-&m>YX1V<WzWE*_fXlR2AAtl8^kCyR7@!mYHA!KE>ZwExEz(`0
z<l;m`GB!`rD>6^zz&9_-y7==AOD#UOJ-GeJO^vf4;>eNF{Ph)5`GN!M1D?IV0KsnF
za~%&&LZclP8(I2MEP=)k5>GLlw7hYAU)|K&S&2zH!ZlsMLqHzM=o-72eB$qw**N<q
zVyiIDe_`-bx(<p(kjQWG8yIXC@bs&7F~Y5?$kNR~u|#6J?X3qA{^}!9MeYA=zmAnC
z9M{VPNMe_R{SNNZYxooYu^$lCm%C+)jlFA_`iI`8H`G6UyMVA~WhLfHRwciy`3J4I
zN(}acgu>7M+l0v-?2cjm7o|Vb!X~LNKkJ~)iBmvz2At|(2si~QK_cAP1qM9GM^C#b
zq$@}~4*ENKowV4HTZgPzm>i|+;%<VGJ|Zzril>s?_6|R95p93}<?l%KJodo(Zr$qS
zcb{TVAb<w<VZ~a{gOK?bOAJ!dV*2I-mX?V>qXU}n?C62E?gW?by~!;iA~k3@ofwbA
zw3`ehWTj`U!a_0548yPCz6N_B<wSc392$-JmP+Ngtgv?<lYhcSz}+s9rj+iEMcHMz
zZ=T;Dl8};YPI|jAqR}R3d8BwGJ&YUx-=qiIa&2mGb%9xftjQK=%H;pX{OtN4CV}Nd
z#h3P~@+>KF^ov9y39b*QV8O|tykQ<R#JG34<;Qei)5^FBbrt-RT>Z)56O1uCoQ@+k
zTwMQy-h*}x6IAyFA!DEjVh2Pf1?x&p)~Sl}f7toYxc~qE|C$0PA`5ppX4zw(J;f){
Pe5vbKZv2D0Y#R7KgHF+1
literal 0
HcmV?d00001
diff --git a/doc/guides/contributing/stable.rst b/doc/guides/contributing/stable.rst
index 6a5eee9..2b563d4 100644
--- a/doc/guides/contributing/stable.rst
+++ b/doc/guides/contributing/stable.rst
@@ -1,7 +1,7 @@
.. SPDX-License-Identifier: BSD-3-Clause
Copyright 2018 The DPDK contributors
-.. stable_lts_releases:
+.. _stable_lts_releases:
DPDK Stable Releases and Long Term Support
==========================================
@@ -53,6 +53,9 @@ year's November (X.11) release will be maintained as an LTS for 2 years.
After the X.11 release, an LTS branch will be created for it at
http://git.dpdk.org/dpdk-stable where bugfixes will be backported to.
+A LTS release may align with the declaration of a new major ABI version,
+please read the :ref:`abi_policy` for more information.
+
It is anticipated that there will be at least 4 releases per year of the LTS
or approximately 1 every 3 months. However, the cadence can be shorter or
longer depending on the number and criticality of the backported
@@ -119,10 +122,3 @@ A Stable Release will be released by:
list.
Stable releases are available on the `dpdk.org download page <http://core.dpdk.org/download/>`_.
-
-
-ABI
----
-
-The Stable Release should not be seen as a way of breaking or circumventing
-the DPDK ABI policy.
--
2.7.4
^ permalink raw reply [relevance 12%]
* [dpdk-dev] [PATCH v6 1/4] doc: separate versioning.rst into version and policy
2019-09-27 16:54 10% [dpdk-dev] [PATCH v6 0/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
@ 2019-09-27 16:54 13% ` Ray Kinsella
2019-09-27 16:54 12% ` [dpdk-dev] [PATCH v6 2/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
` (2 subsequent siblings)
3 siblings, 0 replies; 200+ results
From: Ray Kinsella @ 2019-09-27 16:54 UTC (permalink / raw)
To: dev
Cc: mdr, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, nhorman,
maxime.coquelin, john.mcnamara, marko.kovacevic, hemant.agrawal,
ktraynor, aconole
Separate versioning.rst into abi versioning and abi policy guidance, in
preparation for adding more detail to the abi policy.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
doc/guides/contributing/abi_policy.rst | 169 +++++++++
doc/guides/contributing/abi_versioning.rst | 427 +++++++++++++++++++++
doc/guides/contributing/index.rst | 3 +-
doc/guides/contributing/versioning.rst | 591 -----------------------------
4 files changed, 598 insertions(+), 592 deletions(-)
create mode 100644 doc/guides/contributing/abi_policy.rst
create mode 100644 doc/guides/contributing/abi_versioning.rst
delete mode 100644 doc/guides/contributing/versioning.rst
diff --git a/doc/guides/contributing/abi_policy.rst b/doc/guides/contributing/abi_policy.rst
new file mode 100644
index 0000000..55bacb4
--- /dev/null
+++ b/doc/guides/contributing/abi_policy.rst
@@ -0,0 +1,169 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+ Copyright 2018 The DPDK contributors
+
+.. abi_api_policy:
+
+DPDK ABI/API policy
+===================
+
+Description
+-----------
+
+This document details some methods for handling ABI management in the DPDK.
+
+General Guidelines
+------------------
+
+#. Whenever possible, ABI should be preserved
+#. ABI/API may be changed with a deprecation process
+#. The modification of symbols can generally be managed with versioning
+#. Libraries or APIs marked in ``experimental`` state may change without constraint
+#. New APIs will be marked as ``experimental`` for at least one release to allow
+ any issues found by users of the new API to be fixed quickly
+#. The addition of symbols is generally not problematic
+#. The removal of symbols generally is an ABI break and requires bumping of the
+ LIBABIVER macro
+#. Updates to the minimum hardware requirements, which drop support for hardware which
+ was previously supported, should be treated as an ABI change.
+
+What is an ABI
+~~~~~~~~~~~~~~
+
+An ABI (Application Binary Interface) is the set of runtime interfaces exposed
+by a library. It is similar to an API (Application Programming Interface) but
+is the result of compilation. It is also effectively cloned when applications
+link to dynamic libraries. That is to say when an application is compiled to
+link against dynamic libraries, it is assumed that the ABI remains constant
+between the time the application is compiled/linked, and the time that it runs.
+Therefore, in the case of dynamic linking, it is critical that an ABI is
+preserved, or (when modified), done in such a way that the application is unable
+to behave improperly or in an unexpected fashion.
+
+
+ABI/API Deprecation
+-------------------
+
+The DPDK ABI policy
+~~~~~~~~~~~~~~~~~~~
+
+ABI versions are set at the time of major release labeling, and the ABI may
+change multiple times, without warning, between the last release label and the
+HEAD label of the git tree.
+
+ABI versions, once released, are available until such time as their
+deprecation has been noted in the Release Notes for at least one major release
+cycle. For example consider the case where the ABI for DPDK 2.0 has been
+shipped and then a decision is made to modify it during the development of
+DPDK 2.1. The decision will be recorded in the Release Notes for the DPDK 2.1
+release and the modification will be made available in the DPDK 2.2 release.
+
+ABI versions may be deprecated in whole or in part as needed by a given
+update.
+
+Some ABI changes may be too significant to reasonably maintain multiple
+versions. In those cases ABI's may be updated without backward compatibility
+being provided. The requirements for doing so are:
+
+#. At least 3 acknowledgments of the need to do so must be made on the
+ dpdk.org mailing list.
+
+ - The acknowledgment of the maintainer of the component is mandatory, or if
+ no maintainer is available for the component, the tree/sub-tree maintainer
+ for that component must acknowledge the ABI change instead.
+
+ - It is also recommended that acknowledgments from different "areas of
+ interest" be sought for each deprecation, for example: from NIC vendors,
+ CPU vendors, end-users, etc.
+
+#. The changes (including an alternative map file) can be included with
+ deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option,
+ to provide more details about oncoming changes.
+ ``RTE_NEXT_ABI`` wrapper will be removed when it become the default ABI.
+ More preferred way to provide this information is sending the feature
+ as a separate patch and reference it in deprecation notice.
+
+#. A full deprecation cycle, as explained above, must be made to offer
+ downstream consumers sufficient warning of the change.
+
+Note that the above process for ABI deprecation should not be undertaken
+lightly. ABI stability is extremely important for downstream consumers of the
+DPDK, especially when distributed in shared object form. Every effort should
+be made to preserve the ABI whenever possible. The ABI should only be changed
+for significant reasons, such as performance enhancements. ABI breakage due to
+changes such as reorganizing public structure fields for aesthetic or
+readability purposes should be avoided.
+
+.. note::
+
+ Updates to the minimum hardware requirements, which drop support for hardware
+ which was previously supported, should be treated as an ABI change, and
+ follow the relevant deprecation policy procedures as above: 3 acks and
+ announcement at least one release in advance.
+
+Examples of Deprecation Notices
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The following are some examples of ABI deprecation notices which would be
+added to the Release Notes:
+
+* The Macro ``#RTE_FOO`` is deprecated and will be removed with version 2.0,
+ to be replaced with the inline function ``rte_foo()``.
+
+* The function ``rte_mbuf_grok()`` has been updated to include a new parameter
+ in version 2.0. Backwards compatibility will be maintained for this function
+ until the release of version 2.1
+
+* The members of ``struct rte_foo`` have been reorganized in release 2.0 for
+ performance reasons. Existing binary applications will have backwards
+ compatibility in release 2.0, while newly built binaries will need to
+ reference the new structure variant ``struct rte_foo2``. Compatibility will
+ be removed in release 2.2, and all applications will require updating and
+ rebuilding to the new structure at that time, which will be renamed to the
+ original ``struct rte_foo``.
+
+* Significant ABI changes are planned for the ``librte_dostuff`` library. The
+ upcoming release 2.0 will not contain these changes, but release 2.1 will,
+ and no backwards compatibility is planned due to the extensive nature of
+ these changes. Binaries using this library built prior to version 2.1 will
+ require updating and recompilation.
+
+New API replacing previous one
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If a new API proposed functionally replaces an existing one, when the new API
+becomes non-experimental then the old one is marked with ``__rte_deprecated``.
+Deprecated APIs are removed completely just after the next LTS.
+
+Reminder that old API should follow deprecation process to be removed.
+
+
+Experimental APIs
+-----------------
+
+APIs marked as ``experimental`` are not considered part of the ABI and may
+change without warning at any time. Since changes to APIs are most likely
+immediately after their introduction, as users begin to take advantage of
+those new APIs and start finding issues with them, new DPDK APIs will be
+automatically marked as ``experimental`` to allow for a period of stabilization
+before they become part of a tracked ABI.
+
+Note that marking an API as experimental is a multi step process.
+To mark an API as experimental, the symbols which are desired to be exported
+must be placed in an EXPERIMENTAL version block in the corresponding libraries'
+version map script.
+Secondly, the corresponding prototypes of those exported functions (in the
+development header files), must be marked with the ``__rte_experimental`` tag
+(see ``rte_compat.h``).
+The DPDK build makefiles perform a check to ensure that the map file and the
+C code reflect the same list of symbols.
+This check can be circumvented by defining ``ALLOW_EXPERIMENTAL_API``
+during compilation in the corresponding library Makefile.
+
+In addition to tagging the code with ``__rte_experimental``,
+the doxygen markup must also contain the EXPERIMENTAL string,
+and the MAINTAINERS file should note the EXPERIMENTAL libraries.
+
+For removing the experimental tag associated with an API, deprecation notice
+is not required. Though, an API should remain in experimental state for at least
+one release. Thereafter, normal process of posting patch for review to mailing
+list can be followed.
diff --git a/doc/guides/contributing/abi_versioning.rst b/doc/guides/contributing/abi_versioning.rst
new file mode 100644
index 0000000..53e6ac0
--- /dev/null
+++ b/doc/guides/contributing/abi_versioning.rst
@@ -0,0 +1,427 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+ Copyright 2018 The DPDK contributors
+
+.. library_versioning:
+
+Library versioning
+------------------
+
+Downstreams might want to provide different DPDK releases at the same time to
+support multiple consumers of DPDK linked against older and newer sonames.
+
+Also due to the interdependencies that DPDK libraries can have applications
+might end up with an executable space in which multiple versions of a library
+are mapped by ld.so.
+
+Think of LibA that got an ABI bump and LibB that did not get an ABI bump but is
+depending on LibA.
+
+.. note::
+
+ Application
+ \-> LibA.old
+ \-> LibB.new -> LibA.new
+
+That is a conflict which can be avoided by setting ``CONFIG_RTE_MAJOR_ABI``.
+If set, the value of ``CONFIG_RTE_MAJOR_ABI`` overwrites all - otherwise per
+library - versions defined in the libraries ``LIBABIVER``.
+An example might be ``CONFIG_RTE_MAJOR_ABI=16.11`` which will make all libraries
+``librte<?>.so.16.11`` instead of ``librte<?>.so.<LIBABIVER>``.
+
+
+ABI versioning
+--------------
+
+Versioning Macros
+~~~~~~~~~~~~~~~~~
+
+When a symbol is exported from a library to provide an API, it also provides a
+calling convention (ABI) that is embodied in its name, return type and
+arguments. Occasionally that function may need to change to accommodate new
+functionality or behavior. When that occurs, it is desirable to allow for
+backward compatibility for a time with older binaries that are dynamically
+linked to the DPDK.
+
+To support backward compatibility the ``rte_compat.h``
+header file provides macros to use when updating exported functions. These
+macros are used in conjunction with the ``rte_<library>_version.map`` file for
+a given library to allow multiple versions of a symbol to exist in a shared
+library so that older binaries need not be immediately recompiled.
+
+The macros exported are:
+
+* ``VERSION_SYMBOL(b, e, n)``: Creates a symbol version table entry binding
+ versioned symbol ``b@DPDK_n`` to the internal function ``b_e``.
+
+* ``BIND_DEFAULT_SYMBOL(b, e, n)``: Creates a symbol version entry instructing
+ the linker to bind references to symbol ``b`` to the internal symbol
+ ``b_e``.
+
+* ``MAP_STATIC_SYMBOL(f, p)``: Declare the prototype ``f``, and map it to the
+ fully qualified function ``p``, so that if a symbol becomes versioned, it
+ can still be mapped back to the public symbol name.
+
+Examples of ABI Macro use
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Updating a public API
+_____________________
+
+Assume we have a function as follows
+
+.. code-block:: c
+
+ /*
+ * Create an acl context object for apps to
+ * manipulate
+ */
+ struct rte_acl_ctx *
+ rte_acl_create(const struct rte_acl_param *param)
+ {
+ ...
+ }
+
+
+Assume that struct rte_acl_ctx is a private structure, and that a developer
+wishes to enhance the acl api so that a debugging flag can be enabled on a
+per-context basis. This requires an addition to the structure (which, being
+private, is safe), but it also requires modifying the code as follows
+
+.. code-block:: c
+
+ /*
+ * Create an acl context object for apps to
+ * manipulate
+ */
+ struct rte_acl_ctx *
+ rte_acl_create(const struct rte_acl_param *param, int debug)
+ {
+ ...
+ }
+
+
+Note also that, being a public function, the header file prototype must also be
+changed, as must all the call sites, to reflect the new ABI footprint. We will
+maintain previous ABI versions that are accessible only to previously compiled
+binaries
+
+The addition of a parameter to the function is ABI breaking as the function is
+public, and existing application may use it in its current form. However, the
+compatibility macros in DPDK allow a developer to use symbol versioning so that
+multiple functions can be mapped to the same public symbol based on when an
+application was linked to it. To see how this is done, we start with the
+requisite libraries version map file. Initially the version map file for the
+acl library looks like this
+
+.. code-block:: none
+
+ DPDK_2.0 {
+ global:
+
+ rte_acl_add_rules;
+ rte_acl_build;
+ rte_acl_classify;
+ rte_acl_classify_alg;
+ rte_acl_classify_scalar;
+ rte_acl_create;
+ rte_acl_dump;
+ rte_acl_find_existing;
+ rte_acl_free;
+ rte_acl_ipv4vlan_add_rules;
+ rte_acl_ipv4vlan_build;
+ rte_acl_list_dump;
+ rte_acl_reset;
+ rte_acl_reset_rules;
+ rte_acl_set_ctx_classify;
+
+ local: *;
+ };
+
+This file needs to be modified as follows
+
+.. code-block:: none
+
+ DPDK_2.0 {
+ global:
+
+ rte_acl_add_rules;
+ rte_acl_build;
+ rte_acl_classify;
+ rte_acl_classify_alg;
+ rte_acl_classify_scalar;
+ rte_acl_create;
+ rte_acl_dump;
+ rte_acl_find_existing;
+ rte_acl_free;
+ rte_acl_ipv4vlan_add_rules;
+ rte_acl_ipv4vlan_build;
+ rte_acl_list_dump;
+ rte_acl_reset;
+ rte_acl_reset_rules;
+ rte_acl_set_ctx_classify;
+
+ local: *;
+ };
+
+ DPDK_2.1 {
+ global:
+ rte_acl_create;
+
+ } DPDK_2.0;
+
+The addition of the new block tells the linker that a new version node is
+available (DPDK_2.1), which contains the symbol rte_acl_create, and inherits the
+symbols from the DPDK_2.0 node. This list is directly translated into a list of
+exported symbols when DPDK is compiled as a shared library
+
+Next, we need to specify in the code which function map to the rte_acl_create
+symbol at which versions. First, at the site of the initial symbol definition,
+we need to update the function so that it is uniquely named, and not in conflict
+with the public symbol name
+
+.. code-block:: c
+
+ struct rte_acl_ctx *
+ -rte_acl_create(const struct rte_acl_param *param)
+ +rte_acl_create_v20(const struct rte_acl_param *param)
+ {
+ size_t sz;
+ struct rte_acl_ctx *ctx;
+ ...
+
+Note that the base name of the symbol was kept intact, as this is conducive to
+the macros used for versioning symbols. That is our next step, mapping this new
+symbol name to the initial symbol name at version node 2.0. Immediately after
+the function, we add this line of code
+
+.. code-block:: c
+
+ VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
+
+Remembering to also add the rte_compat.h header to the requisite c file where
+these changes are being made. The above macro instructs the linker to create a
+new symbol ``rte_acl_create@DPDK_2.0``, which matches the symbol created in older
+builds, but now points to the above newly named function. We have now mapped
+the original rte_acl_create symbol to the original function (but with a new
+name)
+
+Next, we need to create the 2.1 version of the symbol. We create a new function
+name, with a different suffix, and implement it appropriately
+
+.. code-block:: c
+
+ struct rte_acl_ctx *
+ rte_acl_create_v21(const struct rte_acl_param *param, int debug);
+ {
+ struct rte_acl_ctx *ctx = rte_acl_create_v20(param);
+
+ ctx->debug = debug;
+
+ return ctx;
+ }
+
+This code serves as our new API call. Its the same as our old call, but adds
+the new parameter in place. Next we need to map this function to the symbol
+``rte_acl_create@DPDK_2.1``. To do this, we modify the public prototype of the call
+in the header file, adding the macro there to inform all including applications,
+that on re-link, the default rte_acl_create symbol should point to this
+function. Note that we could do this by simply naming the function above
+rte_acl_create, and the linker would chose the most recent version tag to apply
+in the version script, but we can also do this in the header file
+
+.. code-block:: c
+
+ struct rte_acl_ctx *
+ -rte_acl_create(const struct rte_acl_param *param);
+ +rte_acl_create(const struct rte_acl_param *param, int debug);
+ +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
+
+The BIND_DEFAULT_SYMBOL macro explicitly tells applications that include this
+header, to link to the rte_acl_create_v21 function and apply the DPDK_2.1
+version node to it. This method is more explicit and flexible than just
+re-implementing the exact symbol name, and allows for other features (such as
+linking to the old symbol version by default, when the new ABI is to be opt-in
+for a period.
+
+One last thing we need to do. Note that we've taken what was a public symbol,
+and duplicated it into two uniquely and differently named symbols. We've then
+mapped each of those back to the public symbol ``rte_acl_create`` with different
+version tags. This only applies to dynamic linking, as static linking has no
+notion of versioning. That leaves this code in a position of no longer having a
+symbol simply named ``rte_acl_create`` and a static build will fail on that
+missing symbol.
+
+To correct this, we can simply map a function of our choosing back to the public
+symbol in the static build with the ``MAP_STATIC_SYMBOL`` macro. Generally the
+assumption is that the most recent version of the symbol is the one you want to
+map. So, back in the C file where, immediately after ``rte_acl_create_v21`` is
+defined, we add this
+
+.. code-block:: c
+
+ struct rte_acl_ctx *
+ rte_acl_create_v21(const struct rte_acl_param *param, int debug)
+ {
+ ...
+ }
+ MAP_STATIC_SYMBOL(struct rte_acl_ctx *rte_acl_create(const struct rte_acl_param *param, int debug), rte_acl_create_v21);
+
+That tells the compiler that, when building a static library, any calls to the
+symbol ``rte_acl_create`` should be linked to ``rte_acl_create_v21``
+
+That's it, on the next shared library rebuild, there will be two versions of
+rte_acl_create, an old DPDK_2.0 version, used by previously built applications,
+and a new DPDK_2.1 version, used by future built applications.
+
+
+Deprecating part of a public API
+________________________________
+
+Lets assume that you've done the above update, and after a few releases have
+passed you decide you would like to retire the old version of the function.
+After having gone through the ABI deprecation announcement process, removal is
+easy. Start by removing the symbol from the requisite version map file:
+
+.. code-block:: none
+
+ DPDK_2.0 {
+ global:
+
+ rte_acl_add_rules;
+ rte_acl_build;
+ rte_acl_classify;
+ rte_acl_classify_alg;
+ rte_acl_classify_scalar;
+ rte_acl_dump;
+ - rte_acl_create
+ rte_acl_find_existing;
+ rte_acl_free;
+ rte_acl_ipv4vlan_add_rules;
+ rte_acl_ipv4vlan_build;
+ rte_acl_list_dump;
+ rte_acl_reset;
+ rte_acl_reset_rules;
+ rte_acl_set_ctx_classify;
+
+ local: *;
+ };
+
+ DPDK_2.1 {
+ global:
+ rte_acl_create;
+ } DPDK_2.0;
+
+
+Next remove the corresponding versioned export.
+
+.. code-block:: c
+
+ -VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
+
+
+Note that the internal function definition could also be removed, but its used
+in our example by the newer version _v21, so we leave it in place. This is a
+coding style choice.
+
+Lastly, we need to bump the LIBABIVER number for this library in the Makefile to
+indicate to applications doing dynamic linking that this is a later, and
+possibly incompatible library version:
+
+.. code-block:: c
+
+ -LIBABIVER := 1
+ +LIBABIVER := 2
+
+Deprecating an entire ABI version
+_________________________________
+
+While removing a symbol from and ABI may be useful, it is often more practical
+to remove an entire version node at once. If a version node completely
+specifies an API, then removing part of it, typically makes it incomplete. In
+those cases it is better to remove the entire node
+
+To do this, start by modifying the version map file, such that all symbols from
+the node to be removed are merged into the next node in the map
+
+In the case of our map above, it would transform to look as follows
+
+.. code-block:: none
+
+ DPDK_2.1 {
+ global:
+
+ rte_acl_add_rules;
+ rte_acl_build;
+ rte_acl_classify;
+ rte_acl_classify_alg;
+ rte_acl_classify_scalar;
+ rte_acl_dump;
+ rte_acl_create
+ rte_acl_find_existing;
+ rte_acl_free;
+ rte_acl_ipv4vlan_add_rules;
+ rte_acl_ipv4vlan_build;
+ rte_acl_list_dump;
+ rte_acl_reset;
+ rte_acl_reset_rules;
+ rte_acl_set_ctx_classify;
+
+ local: *;
+ };
+
+Then any uses of BIND_DEFAULT_SYMBOL that pointed to the old node should be
+updated to point to the new version node in any header files for all affected
+symbols.
+
+.. code-block:: c
+
+ -BIND_DEFAULT_SYMBOL(rte_acl_create, _v20, 2.0);
+ +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
+
+Lastly, any VERSION_SYMBOL macros that point to the old version node should be
+removed, taking care to keep, where need old code in place to support newer
+versions of the symbol.
+
+
+Running the ABI Validator
+-------------------------
+
+The ``devtools`` directory in the DPDK source tree contains a utility program,
+``validate-abi.sh``, for validating the DPDK ABI based on the Linux `ABI
+Compliance Checker
+<http://ispras.linuxbase.org/index.php/ABI_compliance_checker>`_.
+
+This has a dependency on the ``abi-compliance-checker`` and ``and abi-dumper``
+utilities which can be installed via a package manager. For example::
+
+ sudo yum install abi-compliance-checker
+ sudo yum install abi-dumper
+
+The syntax of the ``validate-abi.sh`` utility is::
+
+ ./devtools/validate-abi.sh <REV1> <REV2>
+
+Where ``REV1`` and ``REV2`` are valid gitrevisions(7)
+https://www.kernel.org/pub/software/scm/git/docs/gitrevisions.html
+on the local repo.
+
+For example::
+
+ # Check between the previous and latest commit:
+ ./devtools/validate-abi.sh HEAD~1 HEAD
+
+ # Check on a specific compilation target:
+ ./devtools/validate-abi.sh -t x86_64-native-linux-gcc HEAD~1 HEAD
+
+ # Check between two tags:
+ ./devtools/validate-abi.sh v2.0.0 v2.1.0
+
+ # Check between git master and local topic-branch "vhost-hacking":
+ ./devtools/validate-abi.sh master vhost-hacking
+
+After the validation script completes (it can take a while since it need to
+compile both tags) it will create compatibility reports in the
+``./abi-check/compat_report`` directory. Listed incompatibilities can be found
+as follows::
+
+ grep -lr Incompatible abi-check/compat_reports/
diff --git a/doc/guides/contributing/index.rst b/doc/guides/contributing/index.rst
index e2608d3..2fefd91 100644
--- a/doc/guides/contributing/index.rst
+++ b/doc/guides/contributing/index.rst
@@ -10,7 +10,8 @@ Contributor's Guidelines
coding_style
design
- versioning
+ abi_policy
+ abi_versioning
documentation
patches
vulnerability
diff --git a/doc/guides/contributing/versioning.rst b/doc/guides/contributing/versioning.rst
deleted file mode 100644
index 3ab2c43..0000000
--- a/doc/guides/contributing/versioning.rst
+++ /dev/null
@@ -1,591 +0,0 @@
-.. SPDX-License-Identifier: BSD-3-Clause
- Copyright 2018 The DPDK contributors
-
-DPDK ABI/API policy
-===================
-
-Description
------------
-
-This document details some methods for handling ABI management in the DPDK.
-
-General Guidelines
-------------------
-
-#. Whenever possible, ABI should be preserved
-#. ABI/API may be changed with a deprecation process
-#. The modification of symbols can generally be managed with versioning
-#. Libraries or APIs marked in ``experimental`` state may change without constraint
-#. New APIs will be marked as ``experimental`` for at least one release to allow
- any issues found by users of the new API to be fixed quickly
-#. The addition of symbols is generally not problematic
-#. The removal of symbols generally is an ABI break and requires bumping of the
- LIBABIVER macro
-#. Updates to the minimum hardware requirements, which drop support for hardware which
- was previously supported, should be treated as an ABI change.
-
-What is an ABI
-~~~~~~~~~~~~~~
-
-An ABI (Application Binary Interface) is the set of runtime interfaces exposed
-by a library. It is similar to an API (Application Programming Interface) but
-is the result of compilation. It is also effectively cloned when applications
-link to dynamic libraries. That is to say when an application is compiled to
-link against dynamic libraries, it is assumed that the ABI remains constant
-between the time the application is compiled/linked, and the time that it runs.
-Therefore, in the case of dynamic linking, it is critical that an ABI is
-preserved, or (when modified), done in such a way that the application is unable
-to behave improperly or in an unexpected fashion.
-
-
-ABI/API Deprecation
--------------------
-
-The DPDK ABI policy
-~~~~~~~~~~~~~~~~~~~
-
-ABI versions are set at the time of major release labeling, and the ABI may
-change multiple times, without warning, between the last release label and the
-HEAD label of the git tree.
-
-ABI versions, once released, are available until such time as their
-deprecation has been noted in the Release Notes for at least one major release
-cycle. For example consider the case where the ABI for DPDK 2.0 has been
-shipped and then a decision is made to modify it during the development of
-DPDK 2.1. The decision will be recorded in the Release Notes for the DPDK 2.1
-release and the modification will be made available in the DPDK 2.2 release.
-
-ABI versions may be deprecated in whole or in part as needed by a given
-update.
-
-Some ABI changes may be too significant to reasonably maintain multiple
-versions. In those cases ABI's may be updated without backward compatibility
-being provided. The requirements for doing so are:
-
-#. At least 3 acknowledgments of the need to do so must be made on the
- dpdk.org mailing list.
-
- - The acknowledgment of the maintainer of the component is mandatory, or if
- no maintainer is available for the component, the tree/sub-tree maintainer
- for that component must acknowledge the ABI change instead.
-
- - It is also recommended that acknowledgments from different "areas of
- interest" be sought for each deprecation, for example: from NIC vendors,
- CPU vendors, end-users, etc.
-
-#. The changes (including an alternative map file) can be included with
- deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option,
- to provide more details about oncoming changes.
- ``RTE_NEXT_ABI`` wrapper will be removed when it become the default ABI.
- More preferred way to provide this information is sending the feature
- as a separate patch and reference it in deprecation notice.
-
-#. A full deprecation cycle, as explained above, must be made to offer
- downstream consumers sufficient warning of the change.
-
-Note that the above process for ABI deprecation should not be undertaken
-lightly. ABI stability is extremely important for downstream consumers of the
-DPDK, especially when distributed in shared object form. Every effort should
-be made to preserve the ABI whenever possible. The ABI should only be changed
-for significant reasons, such as performance enhancements. ABI breakage due to
-changes such as reorganizing public structure fields for aesthetic or
-readability purposes should be avoided.
-
-.. note::
-
- Updates to the minimum hardware requirements, which drop support for hardware
- which was previously supported, should be treated as an ABI change, and
- follow the relevant deprecation policy procedures as above: 3 acks and
- announcement at least one release in advance.
-
-Examples of Deprecation Notices
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The following are some examples of ABI deprecation notices which would be
-added to the Release Notes:
-
-* The Macro ``#RTE_FOO`` is deprecated and will be removed with version 2.0,
- to be replaced with the inline function ``rte_foo()``.
-
-* The function ``rte_mbuf_grok()`` has been updated to include a new parameter
- in version 2.0. Backwards compatibility will be maintained for this function
- until the release of version 2.1
-
-* The members of ``struct rte_foo`` have been reorganized in release 2.0 for
- performance reasons. Existing binary applications will have backwards
- compatibility in release 2.0, while newly built binaries will need to
- reference the new structure variant ``struct rte_foo2``. Compatibility will
- be removed in release 2.2, and all applications will require updating and
- rebuilding to the new structure at that time, which will be renamed to the
- original ``struct rte_foo``.
-
-* Significant ABI changes are planned for the ``librte_dostuff`` library. The
- upcoming release 2.0 will not contain these changes, but release 2.1 will,
- and no backwards compatibility is planned due to the extensive nature of
- these changes. Binaries using this library built prior to version 2.1 will
- require updating and recompilation.
-
-New API replacing previous one
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-If a new API proposed functionally replaces an existing one, when the new API
-becomes non-experimental then the old one is marked with ``__rte_deprecated``.
-Deprecated APIs are removed completely just after the next LTS.
-
-Reminder that old API should follow deprecation process to be removed.
-
-
-Experimental APIs
------------------
-
-APIs marked as ``experimental`` are not considered part of the ABI and may
-change without warning at any time. Since changes to APIs are most likely
-immediately after their introduction, as users begin to take advantage of
-those new APIs and start finding issues with them, new DPDK APIs will be
-automatically marked as ``experimental`` to allow for a period of stabilization
-before they become part of a tracked ABI.
-
-Note that marking an API as experimental is a multi step process.
-To mark an API as experimental, the symbols which are desired to be exported
-must be placed in an EXPERIMENTAL version block in the corresponding libraries'
-version map script.
-Secondly, the corresponding prototypes of those exported functions (in the
-development header files), must be marked with the ``__rte_experimental`` tag
-(see ``rte_compat.h``).
-The DPDK build makefiles perform a check to ensure that the map file and the
-C code reflect the same list of symbols.
-This check can be circumvented by defining ``ALLOW_EXPERIMENTAL_API``
-during compilation in the corresponding library Makefile.
-
-In addition to tagging the code with ``__rte_experimental``,
-the doxygen markup must also contain the EXPERIMENTAL string,
-and the MAINTAINERS file should note the EXPERIMENTAL libraries.
-
-For removing the experimental tag associated with an API, deprecation notice
-is not required. Though, an API should remain in experimental state for at least
-one release. Thereafter, normal process of posting patch for review to mailing
-list can be followed.
-
-
-Library versioning
-------------------
-
-Downstreams might want to provide different DPDK releases at the same time to
-support multiple consumers of DPDK linked against older and newer sonames.
-
-Also due to the interdependencies that DPDK libraries can have applications
-might end up with an executable space in which multiple versions of a library
-are mapped by ld.so.
-
-Think of LibA that got an ABI bump and LibB that did not get an ABI bump but is
-depending on LibA.
-
-.. note::
-
- Application
- \-> LibA.old
- \-> LibB.new -> LibA.new
-
-That is a conflict which can be avoided by setting ``CONFIG_RTE_MAJOR_ABI``.
-If set, the value of ``CONFIG_RTE_MAJOR_ABI`` overwrites all - otherwise per
-library - versions defined in the libraries ``LIBABIVER``.
-An example might be ``CONFIG_RTE_MAJOR_ABI=16.11`` which will make all libraries
-``librte<?>.so.16.11`` instead of ``librte<?>.so.<LIBABIVER>``.
-
-
-ABI versioning
---------------
-
-Versioning Macros
-~~~~~~~~~~~~~~~~~
-
-When a symbol is exported from a library to provide an API, it also provides a
-calling convention (ABI) that is embodied in its name, return type and
-arguments. Occasionally that function may need to change to accommodate new
-functionality or behavior. When that occurs, it is desirable to allow for
-backward compatibility for a time with older binaries that are dynamically
-linked to the DPDK.
-
-To support backward compatibility the ``rte_compat.h``
-header file provides macros to use when updating exported functions. These
-macros are used in conjunction with the ``rte_<library>_version.map`` file for
-a given library to allow multiple versions of a symbol to exist in a shared
-library so that older binaries need not be immediately recompiled.
-
-The macros exported are:
-
-* ``VERSION_SYMBOL(b, e, n)``: Creates a symbol version table entry binding
- versioned symbol ``b@DPDK_n`` to the internal function ``b_e``.
-
-* ``BIND_DEFAULT_SYMBOL(b, e, n)``: Creates a symbol version entry instructing
- the linker to bind references to symbol ``b`` to the internal symbol
- ``b_e``.
-
-* ``MAP_STATIC_SYMBOL(f, p)``: Declare the prototype ``f``, and map it to the
- fully qualified function ``p``, so that if a symbol becomes versioned, it
- can still be mapped back to the public symbol name.
-
-Examples of ABI Macro use
-^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Updating a public API
-_____________________
-
-Assume we have a function as follows
-
-.. code-block:: c
-
- /*
- * Create an acl context object for apps to
- * manipulate
- */
- struct rte_acl_ctx *
- rte_acl_create(const struct rte_acl_param *param)
- {
- ...
- }
-
-
-Assume that struct rte_acl_ctx is a private structure, and that a developer
-wishes to enhance the acl api so that a debugging flag can be enabled on a
-per-context basis. This requires an addition to the structure (which, being
-private, is safe), but it also requires modifying the code as follows
-
-.. code-block:: c
-
- /*
- * Create an acl context object for apps to
- * manipulate
- */
- struct rte_acl_ctx *
- rte_acl_create(const struct rte_acl_param *param, int debug)
- {
- ...
- }
-
-
-Note also that, being a public function, the header file prototype must also be
-changed, as must all the call sites, to reflect the new ABI footprint. We will
-maintain previous ABI versions that are accessible only to previously compiled
-binaries
-
-The addition of a parameter to the function is ABI breaking as the function is
-public, and existing application may use it in its current form. However, the
-compatibility macros in DPDK allow a developer to use symbol versioning so that
-multiple functions can be mapped to the same public symbol based on when an
-application was linked to it. To see how this is done, we start with the
-requisite libraries version map file. Initially the version map file for the
-acl library looks like this
-
-.. code-block:: none
-
- DPDK_2.0 {
- global:
-
- rte_acl_add_rules;
- rte_acl_build;
- rte_acl_classify;
- rte_acl_classify_alg;
- rte_acl_classify_scalar;
- rte_acl_create;
- rte_acl_dump;
- rte_acl_find_existing;
- rte_acl_free;
- rte_acl_ipv4vlan_add_rules;
- rte_acl_ipv4vlan_build;
- rte_acl_list_dump;
- rte_acl_reset;
- rte_acl_reset_rules;
- rte_acl_set_ctx_classify;
-
- local: *;
- };
-
-This file needs to be modified as follows
-
-.. code-block:: none
-
- DPDK_2.0 {
- global:
-
- rte_acl_add_rules;
- rte_acl_build;
- rte_acl_classify;
- rte_acl_classify_alg;
- rte_acl_classify_scalar;
- rte_acl_create;
- rte_acl_dump;
- rte_acl_find_existing;
- rte_acl_free;
- rte_acl_ipv4vlan_add_rules;
- rte_acl_ipv4vlan_build;
- rte_acl_list_dump;
- rte_acl_reset;
- rte_acl_reset_rules;
- rte_acl_set_ctx_classify;
-
- local: *;
- };
-
- DPDK_2.1 {
- global:
- rte_acl_create;
-
- } DPDK_2.0;
-
-The addition of the new block tells the linker that a new version node is
-available (DPDK_2.1), which contains the symbol rte_acl_create, and inherits the
-symbols from the DPDK_2.0 node. This list is directly translated into a list of
-exported symbols when DPDK is compiled as a shared library
-
-Next, we need to specify in the code which function map to the rte_acl_create
-symbol at which versions. First, at the site of the initial symbol definition,
-we need to update the function so that it is uniquely named, and not in conflict
-with the public symbol name
-
-.. code-block:: c
-
- struct rte_acl_ctx *
- -rte_acl_create(const struct rte_acl_param *param)
- +rte_acl_create_v20(const struct rte_acl_param *param)
- {
- size_t sz;
- struct rte_acl_ctx *ctx;
- ...
-
-Note that the base name of the symbol was kept intact, as this is conducive to
-the macros used for versioning symbols. That is our next step, mapping this new
-symbol name to the initial symbol name at version node 2.0. Immediately after
-the function, we add this line of code
-
-.. code-block:: c
-
- VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
-
-Remembering to also add the rte_compat.h header to the requisite c file where
-these changes are being made. The above macro instructs the linker to create a
-new symbol ``rte_acl_create@DPDK_2.0``, which matches the symbol created in older
-builds, but now points to the above newly named function. We have now mapped
-the original rte_acl_create symbol to the original function (but with a new
-name)
-
-Next, we need to create the 2.1 version of the symbol. We create a new function
-name, with a different suffix, and implement it appropriately
-
-.. code-block:: c
-
- struct rte_acl_ctx *
- rte_acl_create_v21(const struct rte_acl_param *param, int debug);
- {
- struct rte_acl_ctx *ctx = rte_acl_create_v20(param);
-
- ctx->debug = debug;
-
- return ctx;
- }
-
-This code serves as our new API call. Its the same as our old call, but adds
-the new parameter in place. Next we need to map this function to the symbol
-``rte_acl_create@DPDK_2.1``. To do this, we modify the public prototype of the call
-in the header file, adding the macro there to inform all including applications,
-that on re-link, the default rte_acl_create symbol should point to this
-function. Note that we could do this by simply naming the function above
-rte_acl_create, and the linker would chose the most recent version tag to apply
-in the version script, but we can also do this in the header file
-
-.. code-block:: c
-
- struct rte_acl_ctx *
- -rte_acl_create(const struct rte_acl_param *param);
- +rte_acl_create(const struct rte_acl_param *param, int debug);
- +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
-
-The BIND_DEFAULT_SYMBOL macro explicitly tells applications that include this
-header, to link to the rte_acl_create_v21 function and apply the DPDK_2.1
-version node to it. This method is more explicit and flexible than just
-re-implementing the exact symbol name, and allows for other features (such as
-linking to the old symbol version by default, when the new ABI is to be opt-in
-for a period.
-
-One last thing we need to do. Note that we've taken what was a public symbol,
-and duplicated it into two uniquely and differently named symbols. We've then
-mapped each of those back to the public symbol ``rte_acl_create`` with different
-version tags. This only applies to dynamic linking, as static linking has no
-notion of versioning. That leaves this code in a position of no longer having a
-symbol simply named ``rte_acl_create`` and a static build will fail on that
-missing symbol.
-
-To correct this, we can simply map a function of our choosing back to the public
-symbol in the static build with the ``MAP_STATIC_SYMBOL`` macro. Generally the
-assumption is that the most recent version of the symbol is the one you want to
-map. So, back in the C file where, immediately after ``rte_acl_create_v21`` is
-defined, we add this
-
-.. code-block:: c
-
- struct rte_acl_ctx *
- rte_acl_create_v21(const struct rte_acl_param *param, int debug)
- {
- ...
- }
- MAP_STATIC_SYMBOL(struct rte_acl_ctx *rte_acl_create(const struct rte_acl_param *param, int debug), rte_acl_create_v21);
-
-That tells the compiler that, when building a static library, any calls to the
-symbol ``rte_acl_create`` should be linked to ``rte_acl_create_v21``
-
-That's it, on the next shared library rebuild, there will be two versions of
-rte_acl_create, an old DPDK_2.0 version, used by previously built applications,
-and a new DPDK_2.1 version, used by future built applications.
-
-
-Deprecating part of a public API
-________________________________
-
-Lets assume that you've done the above update, and after a few releases have
-passed you decide you would like to retire the old version of the function.
-After having gone through the ABI deprecation announcement process, removal is
-easy. Start by removing the symbol from the requisite version map file:
-
-.. code-block:: none
-
- DPDK_2.0 {
- global:
-
- rte_acl_add_rules;
- rte_acl_build;
- rte_acl_classify;
- rte_acl_classify_alg;
- rte_acl_classify_scalar;
- rte_acl_dump;
- - rte_acl_create
- rte_acl_find_existing;
- rte_acl_free;
- rte_acl_ipv4vlan_add_rules;
- rte_acl_ipv4vlan_build;
- rte_acl_list_dump;
- rte_acl_reset;
- rte_acl_reset_rules;
- rte_acl_set_ctx_classify;
-
- local: *;
- };
-
- DPDK_2.1 {
- global:
- rte_acl_create;
- } DPDK_2.0;
-
-
-Next remove the corresponding versioned export.
-
-.. code-block:: c
-
- -VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
-
-
-Note that the internal function definition could also be removed, but its used
-in our example by the newer version _v21, so we leave it in place. This is a
-coding style choice.
-
-Lastly, we need to bump the LIBABIVER number for this library in the Makefile to
-indicate to applications doing dynamic linking that this is a later, and
-possibly incompatible library version:
-
-.. code-block:: c
-
- -LIBABIVER := 1
- +LIBABIVER := 2
-
-Deprecating an entire ABI version
-_________________________________
-
-While removing a symbol from and ABI may be useful, it is often more practical
-to remove an entire version node at once. If a version node completely
-specifies an API, then removing part of it, typically makes it incomplete. In
-those cases it is better to remove the entire node
-
-To do this, start by modifying the version map file, such that all symbols from
-the node to be removed are merged into the next node in the map
-
-In the case of our map above, it would transform to look as follows
-
-.. code-block:: none
-
- DPDK_2.1 {
- global:
-
- rte_acl_add_rules;
- rte_acl_build;
- rte_acl_classify;
- rte_acl_classify_alg;
- rte_acl_classify_scalar;
- rte_acl_dump;
- rte_acl_create
- rte_acl_find_existing;
- rte_acl_free;
- rte_acl_ipv4vlan_add_rules;
- rte_acl_ipv4vlan_build;
- rte_acl_list_dump;
- rte_acl_reset;
- rte_acl_reset_rules;
- rte_acl_set_ctx_classify;
-
- local: *;
- };
-
-Then any uses of BIND_DEFAULT_SYMBOL that pointed to the old node should be
-updated to point to the new version node in any header files for all affected
-symbols.
-
-.. code-block:: c
-
- -BIND_DEFAULT_SYMBOL(rte_acl_create, _v20, 2.0);
- +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
-
-Lastly, any VERSION_SYMBOL macros that point to the old version node should be
-removed, taking care to keep, where need old code in place to support newer
-versions of the symbol.
-
-
-Running the ABI Validator
--------------------------
-
-The ``devtools`` directory in the DPDK source tree contains a utility program,
-``validate-abi.sh``, for validating the DPDK ABI based on the Linux `ABI
-Compliance Checker
-<http://ispras.linuxbase.org/index.php/ABI_compliance_checker>`_.
-
-This has a dependency on the ``abi-compliance-checker`` and ``and abi-dumper``
-utilities which can be installed via a package manager. For example::
-
- sudo yum install abi-compliance-checker
- sudo yum install abi-dumper
-
-The syntax of the ``validate-abi.sh`` utility is::
-
- ./devtools/validate-abi.sh <REV1> <REV2>
-
-Where ``REV1`` and ``REV2`` are valid gitrevisions(7)
-https://www.kernel.org/pub/software/scm/git/docs/gitrevisions.html
-on the local repo.
-
-For example::
-
- # Check between the previous and latest commit:
- ./devtools/validate-abi.sh HEAD~1 HEAD
-
- # Check on a specific compilation target:
- ./devtools/validate-abi.sh -t x86_64-native-linux-gcc HEAD~1 HEAD
-
- # Check between two tags:
- ./devtools/validate-abi.sh v2.0.0 v2.1.0
-
- # Check between git master and local topic-branch "vhost-hacking":
- ./devtools/validate-abi.sh master vhost-hacking
-
-After the validation script completes (it can take a while since it need to
-compile both tags) it will create compatibility reports in the
-``./abi-check/compat_report`` directory. Listed incompatibilities can be found
-as follows::
-
- grep -lr Incompatible abi-check/compat_reports/
--
2.7.4
^ permalink raw reply [relevance 13%]
* [dpdk-dev] [PATCH v6 0/4] doc: changes to abi policy introducing major abi versions
@ 2019-09-27 16:54 10% Ray Kinsella
2019-09-27 16:54 13% ` [dpdk-dev] [PATCH v6 1/4] doc: separate versioning.rst into version and policy Ray Kinsella
` (3 more replies)
0 siblings, 4 replies; 200+ results
From: Ray Kinsella @ 2019-09-27 16:54 UTC (permalink / raw)
To: dev
Cc: mdr, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, nhorman,
maxime.coquelin, john.mcnamara, marko.kovacevic, hemant.agrawal,
ktraynor, aconole
TL;DR abbreviation:
A major ABI version that all DPDK releases during a one year period
support. ABI versioning is managed at a project-level, in place of library-level
management. ABI changes to add new features are permitted, as long as ABI
compatibility with the major ABI version is maintained.
Detail:
This patch introduces major ABI versions, supported for one year and released
aligned with the LTS release. This ABI version is then supported by all
subsequent releases within that one year period. The intention is that the one
year support period, will then be reviewed after the initial year with the
intention of lengthing the support period for the next ABI version.
ABI changes that preserve ABI compatibility with the major ABI version are
permitted in subsequent releases. ABI changes, follow similar approval rules as
before with the additional gate of now requiring technical board approval. The
merging and release of ABI breaking changes would now be pushed to the
declaration of the next major ABI version.
This change encourages developers to maintain ABI compatibility with the major
ABI version, by promoting a permissive culture around those changes that
preserve ABI compatibility. This approach begins to align DPDK with those
projects that declare major ABI versions (e.g. version 2.x, 3.x) and support
those versions for some period, typically two years or more.
To provide an example of how this might work in practice:
* DPDK v20 is declared as the supported ABI version for one year, aligned with
the DPDK v19.11 (LTS) release. All library sonames are updated to reflect the
new ABI version, e.g. librte_eal.so.20, librte_acl.so.20...
* DPDK v20.02 .. v20.08 releases are ABI compatible with the DPDK v20 ABI. ABI
changes are permitted from DPDK v20.02 onwards, with the condition that ABI
compatibility with DPDK v20 is preserved.
* DPDK v21 is declared as the new supported ABI version for two years, aligned
with the DPDK v20.11 (LTS) release. The DPDK v20 ABI is now depreciated,
library sonames are updated to v21 and ABI compatibility breaking changes may
be introduced.
v6
* Added figure to abi_policy.rst, comparing and contrasting the DPDK abi and
api. (as suggested by Aaron Conole)
v5
* Added figure to abi_policy.rst, mapping abi versions and abi compatibility to
DPDK releases. (as suggested by Neil Horman)
v4
* Removed changes to stable.rst, fixed typos and clarified the ABI policy
"warning".
v3
* Added myself as the maintainer of the ABI policy.
* Updated the policy and versioning guides to use the year of the LTS+1 (e.g.
v20), as the abi major version number.
v2
* Restructured the patch into 3 patches:
1. Splits the original versioning document into an ABI policy document
and ABI versioning document.
2. Add changes to the policy document introducing major ABI versions.
3. Fixes up the versioning document in light of major ABI versioning.
* Reduces the initial ABI stability from two years to one year, with a review
after the first year.
* Adds detail around ABI version handling for experimental libraries.
* Adds detail around chain of responsility for removing deprecated symbols.
Ray Kinsella (4):
doc: separate versioning.rst into version and policy
doc: changes to abi policy introducing major abi versions
doc: updates to versioning guide for abi versions
doc: add maintainer for abi policy
MAINTAINERS | 4 +
doc/guides/contributing/abi_policy.rst | 322 +++++++++++
doc/guides/contributing/abi_versioning.rst | 515 ++++++++++++++++++
.../contributing/img/abi_stability_policy.png | Bin 0 -> 61277 bytes
doc/guides/contributing/img/what_is_an_abi.png | Bin 0 -> 151683 bytes
doc/guides/contributing/index.rst | 3 +-
doc/guides/contributing/patches.rst | 6 +-
doc/guides/contributing/stable.rst | 12 +-
doc/guides/contributing/versioning.rst | 591 ---------------------
doc/guides/rel_notes/deprecation.rst | 2 +-
10 files changed, 851 insertions(+), 604 deletions(-)
create mode 100644 doc/guides/contributing/abi_policy.rst
create mode 100644 doc/guides/contributing/abi_versioning.rst
create mode 100644 doc/guides/contributing/img/abi_stability_policy.png
create mode 100644 doc/guides/contributing/img/what_is_an_abi.png
delete mode 100644 doc/guides/contributing/versioning.rst
--
2.7.4
^ permalink raw reply [relevance 10%]
* Re: [dpdk-dev] [PATCH v4 1/4] doc: separate versioning.rst into version and policy
2019-09-27 15:43 4% ` Aaron Conole
@ 2019-09-27 16:43 4% ` Ray Kinsella
0 siblings, 0 replies; 200+ results
From: Ray Kinsella @ 2019-09-27 16:43 UTC (permalink / raw)
To: Aaron Conole
Cc: dev, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, nhorman,
maxime.coquelin, john.mcnamara, marko.kovacevic, hemant.agrawal,
ktraynor
On 27/09/2019 16:43, Aaron Conole wrote:
> Ray Kinsella <mdr@ashroe.eu> writes:
>
>> Separate versioning.rst into abi versioning and abi policy guidance, in
>> preparation for adding more detail to the abi policy.
>>
>> Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
>> ---
>> doc/guides/contributing/abi_policy.rst | 169 +++++++++
>> doc/guides/contributing/abi_versioning.rst | 427 +++++++++++++++++++++
>> doc/guides/contributing/index.rst | 3 +-
>> doc/guides/contributing/versioning.rst | 591 -----------------------------
>> 4 files changed, 598 insertions(+), 592 deletions(-)
>> create mode 100644 doc/guides/contributing/abi_policy.rst
>> create mode 100644 doc/guides/contributing/abi_versioning.rst
>> delete mode 100644 doc/guides/contributing/versioning.rst
>>
>> diff --git a/doc/guides/contributing/abi_policy.rst b/doc/guides/contributing/abi_policy.rst
>> new file mode 100644
>> index 0000000..55bacb4
>> --- /dev/null
>> +++ b/doc/guides/contributing/abi_policy.rst
>> @@ -0,0 +1,169 @@
>> +.. SPDX-License-Identifier: BSD-3-Clause
>> + Copyright 2018 The DPDK contributors
>> +
>> +.. abi_api_policy:
>> +
>> +DPDK ABI/API policy
>> +===================
>> +
>> +Description
>> +-----------
>> +
>> +This document details some methods for handling ABI management in the DPDK.
>> +
>> +General Guidelines
>> +------------------
>> +
>> +#. Whenever possible, ABI should be preserved
>> +#. ABI/API may be changed with a deprecation process
>> +#. The modification of symbols can generally be managed with versioning
>> +#. Libraries or APIs marked in ``experimental`` state may change without constraint
>> +#. New APIs will be marked as ``experimental`` for at least one release to allow
>> + any issues found by users of the new API to be fixed quickly
>> +#. The addition of symbols is generally not problematic
>> +#. The removal of symbols generally is an ABI break and requires bumping of the
>> + LIBABIVER macro
>> +#. Updates to the minimum hardware requirements, which drop support for hardware which
>> + was previously supported, should be treated as an ABI change.
>> +
>> +What is an ABI
>> +~~~~~~~~~~~~~~
>> +
>> +An ABI (Application Binary Interface) is the set of runtime interfaces exposed
>> +by a library. It is similar to an API (Application Programming Interface) but
>> +is the result of compilation. It is also effectively cloned when applications
>> +link to dynamic libraries. That is to say when an application is compiled to
>> +link against dynamic libraries, it is assumed that the ABI remains constant
>> +between the time the application is compiled/linked, and the time that it runs.
>> +Therefore, in the case of dynamic linking, it is critical that an ABI is
>> +preserved, or (when modified), done in such a way that the application is unable
>> +to behave improperly or in an unexpected fashion.
>> +
>
> This section probably needs a bit more details. People still are
> confused what exactly constitutes ABI vs. API (see
> http://mails.dpdk.org/archives/dev/2018-January/085209.html for a
> confusing example).
>
> It's important that people know not just function signatures, but also
> return codes, and even data structure layouts are all part of the ABI.
Hi Aaron, just saw this now after I sent the v5.
This text is actually from the original policy, this patch is just
separating the abi policy from the abi versioning.
That said, I think another diagram would also be helpful here.
I have one we can use.
>
>> +
>> +ABI/API Deprecation
>> +-------------------
>> +
>> +The DPDK ABI policy
>> +~~~~~~~~~~~~~~~~~~~
>> +
>> +ABI versions are set at the time of major release labeling, and the ABI may
>> +change multiple times, without warning, between the last release label and the
>> +HEAD label of the git tree.
>> +
>> +ABI versions, once released, are available until such time as their
>> +deprecation has been noted in the Release Notes for at least one major release
>> +cycle. For example consider the case where the ABI for DPDK 2.0 has been
>> +shipped and then a decision is made to modify it during the development of
>> +DPDK 2.1. The decision will be recorded in the Release Notes for the DPDK 2.1
>> +release and the modification will be made available in the DPDK 2.2 release.
>> +
>> +ABI versions may be deprecated in whole or in part as needed by a given
>> +update.
>> +
>> +Some ABI changes may be too significant to reasonably maintain multiple
>> +versions. In those cases ABI's may be updated without backward compatibility
>> +being provided. The requirements for doing so are:
>> +
>> +#. At least 3 acknowledgments of the need to do so must be made on the
>> + dpdk.org mailing list.
>> +
>> + - The acknowledgment of the maintainer of the component is mandatory, or if
>> + no maintainer is available for the component, the tree/sub-tree maintainer
>> + for that component must acknowledge the ABI change instead.
>> +
>> + - It is also recommended that acknowledgments from different "areas of
>> + interest" be sought for each deprecation, for example: from NIC vendors,
>> + CPU vendors, end-users, etc.
>> +
>> +#. The changes (including an alternative map file) can be included with
>> + deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option,
>> + to provide more details about oncoming changes.
>> + ``RTE_NEXT_ABI`` wrapper will be removed when it become the default ABI.
>> + More preferred way to provide this information is sending the feature
>> + as a separate patch and reference it in deprecation notice.
>> +
>> +#. A full deprecation cycle, as explained above, must be made to offer
>> + downstream consumers sufficient warning of the change.
>> +
>> +Note that the above process for ABI deprecation should not be undertaken
>> +lightly. ABI stability is extremely important for downstream consumers of the
>> +DPDK, especially when distributed in shared object form. Every effort should
>> +be made to preserve the ABI whenever possible. The ABI should only be changed
>> +for significant reasons, such as performance enhancements. ABI breakage due to
>> +changes such as reorganizing public structure fields for aesthetic or
>> +readability purposes should be avoided.
>> +
>> +.. note::
>> +
>> + Updates to the minimum hardware requirements, which drop support for hardware
>> + which was previously supported, should be treated as an ABI change, and
>> + follow the relevant deprecation policy procedures as above: 3 acks and
>> + announcement at least one release in advance.
>> +
>> +Examples of Deprecation Notices
>> +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
>> +
>> +The following are some examples of ABI deprecation notices which would be
>> +added to the Release Notes:
>> +
>> +* The Macro ``#RTE_FOO`` is deprecated and will be removed with version 2.0,
>> + to be replaced with the inline function ``rte_foo()``.
>> +
>> +* The function ``rte_mbuf_grok()`` has been updated to include a new parameter
>> + in version 2.0. Backwards compatibility will be maintained for this function
>> + until the release of version 2.1
>> +
>> +* The members of ``struct rte_foo`` have been reorganized in release 2.0 for
>> + performance reasons. Existing binary applications will have backwards
>> + compatibility in release 2.0, while newly built binaries will need to
>> + reference the new structure variant ``struct rte_foo2``. Compatibility will
>> + be removed in release 2.2, and all applications will require updating and
>> + rebuilding to the new structure at that time, which will be renamed to the
>> + original ``struct rte_foo``.
>> +
>> +* Significant ABI changes are planned for the ``librte_dostuff`` library. The
>> + upcoming release 2.0 will not contain these changes, but release 2.1 will,
>> + and no backwards compatibility is planned due to the extensive nature of
>> + these changes. Binaries using this library built prior to version 2.1 will
>> + require updating and recompilation.
>> +
>> +New API replacing previous one
>> +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
>> +
>> +If a new API proposed functionally replaces an existing one, when the new API
>> +becomes non-experimental then the old one is marked with ``__rte_deprecated``.
>> +Deprecated APIs are removed completely just after the next LTS.
>> +
>> +Reminder that old API should follow deprecation process to be removed.
>> +
>> +
>> +Experimental APIs
>> +-----------------
>> +
>> +APIs marked as ``experimental`` are not considered part of the ABI and may
>> +change without warning at any time. Since changes to APIs are most likely
>> +immediately after their introduction, as users begin to take advantage of
>> +those new APIs and start finding issues with them, new DPDK APIs will be
>> +automatically marked as ``experimental`` to allow for a period of stabilization
>> +before they become part of a tracked ABI.
>> +
>> +Note that marking an API as experimental is a multi step process.
>> +To mark an API as experimental, the symbols which are desired to be exported
>> +must be placed in an EXPERIMENTAL version block in the corresponding libraries'
>> +version map script.
>> +Secondly, the corresponding prototypes of those exported functions (in the
>> +development header files), must be marked with the ``__rte_experimental`` tag
>> +(see ``rte_compat.h``).
>> +The DPDK build makefiles perform a check to ensure that the map file and the
>> +C code reflect the same list of symbols.
>> +This check can be circumvented by defining ``ALLOW_EXPERIMENTAL_API``
>> +during compilation in the corresponding library Makefile.
>> +
>> +In addition to tagging the code with ``__rte_experimental``,
>> +the doxygen markup must also contain the EXPERIMENTAL string,
>> +and the MAINTAINERS file should note the EXPERIMENTAL libraries.
>> +
>> +For removing the experimental tag associated with an API, deprecation notice
>> +is not required. Though, an API should remain in experimental state for at least
>> +one release. Thereafter, normal process of posting patch for review to mailing
>> +list can be followed.
>> diff --git a/doc/guides/contributing/abi_versioning.rst b/doc/guides/contributing/abi_versioning.rst
>> new file mode 100644
>> index 0000000..53e6ac0
>> --- /dev/null
>> +++ b/doc/guides/contributing/abi_versioning.rst
>> @@ -0,0 +1,427 @@
>> +.. SPDX-License-Identifier: BSD-3-Clause
>> + Copyright 2018 The DPDK contributors
>> +
>> +.. library_versioning:
>> +
>> +Library versioning
>> +------------------
>> +
>> +Downstreams might want to provide different DPDK releases at the same time to
>> +support multiple consumers of DPDK linked against older and newer sonames.
>> +
>> +Also due to the interdependencies that DPDK libraries can have applications
>> +might end up with an executable space in which multiple versions of a library
>> +are mapped by ld.so.
>> +
>> +Think of LibA that got an ABI bump and LibB that did not get an ABI bump but is
>> +depending on LibA.
>> +
>> +.. note::
>> +
>> + Application
>> + \-> LibA.old
>> + \-> LibB.new -> LibA.new
>> +
>> +That is a conflict which can be avoided by setting ``CONFIG_RTE_MAJOR_ABI``.
>> +If set, the value of ``CONFIG_RTE_MAJOR_ABI`` overwrites all - otherwise per
>> +library - versions defined in the libraries ``LIBABIVER``.
>> +An example might be ``CONFIG_RTE_MAJOR_ABI=16.11`` which will make all libraries
>> +``librte<?>.so.16.11`` instead of ``librte<?>.so.<LIBABIVER>``.
>> +
>> +
>> +ABI versioning
>> +--------------
>> +
>> +Versioning Macros
>> +~~~~~~~~~~~~~~~~~
>> +
>> +When a symbol is exported from a library to provide an API, it also provides a
>> +calling convention (ABI) that is embodied in its name, return type and
>> +arguments. Occasionally that function may need to change to accommodate new
>> +functionality or behavior. When that occurs, it is desirable to allow for
>> +backward compatibility for a time with older binaries that are dynamically
>> +linked to the DPDK.
>> +
>> +To support backward compatibility the ``rte_compat.h``
>> +header file provides macros to use when updating exported functions. These
>> +macros are used in conjunction with the ``rte_<library>_version.map`` file for
>> +a given library to allow multiple versions of a symbol to exist in a shared
>> +library so that older binaries need not be immediately recompiled.
>> +
>> +The macros exported are:
>> +
>> +* ``VERSION_SYMBOL(b, e, n)``: Creates a symbol version table entry binding
>> + versioned symbol ``b@DPDK_n`` to the internal function ``b_e``.
>> +
>> +* ``BIND_DEFAULT_SYMBOL(b, e, n)``: Creates a symbol version entry instructing
>> + the linker to bind references to symbol ``b`` to the internal symbol
>> + ``b_e``.
>> +
>> +* ``MAP_STATIC_SYMBOL(f, p)``: Declare the prototype ``f``, and map it to the
>> + fully qualified function ``p``, so that if a symbol becomes versioned, it
>> + can still be mapped back to the public symbol name.
>> +
>> +Examples of ABI Macro use
>> +^^^^^^^^^^^^^^^^^^^^^^^^^
>> +
>> +Updating a public API
>> +_____________________
>> +
>> +Assume we have a function as follows
>> +
>> +.. code-block:: c
>> +
>> + /*
>> + * Create an acl context object for apps to
>> + * manipulate
>> + */
>> + struct rte_acl_ctx *
>> + rte_acl_create(const struct rte_acl_param *param)
>> + {
>> + ...
>> + }
>> +
>> +
>> +Assume that struct rte_acl_ctx is a private structure, and that a developer
>> +wishes to enhance the acl api so that a debugging flag can be enabled on a
>> +per-context basis. This requires an addition to the structure (which, being
>> +private, is safe), but it also requires modifying the code as follows
>> +
>> +.. code-block:: c
>> +
>> + /*
>> + * Create an acl context object for apps to
>> + * manipulate
>> + */
>> + struct rte_acl_ctx *
>> + rte_acl_create(const struct rte_acl_param *param, int debug)
>> + {
>> + ...
>> + }
>> +
>> +
>> +Note also that, being a public function, the header file prototype must also be
>> +changed, as must all the call sites, to reflect the new ABI footprint. We will
>> +maintain previous ABI versions that are accessible only to previously compiled
>> +binaries
>> +
>> +The addition of a parameter to the function is ABI breaking as the function is
>> +public, and existing application may use it in its current form. However, the
>> +compatibility macros in DPDK allow a developer to use symbol versioning so that
>> +multiple functions can be mapped to the same public symbol based on when an
>> +application was linked to it. To see how this is done, we start with the
>> +requisite libraries version map file. Initially the version map file for the
>> +acl library looks like this
>> +
>> +.. code-block:: none
>> +
>> + DPDK_2.0 {
>> + global:
>> +
>> + rte_acl_add_rules;
>> + rte_acl_build;
>> + rte_acl_classify;
>> + rte_acl_classify_alg;
>> + rte_acl_classify_scalar;
>> + rte_acl_create;
>> + rte_acl_dump;
>> + rte_acl_find_existing;
>> + rte_acl_free;
>> + rte_acl_ipv4vlan_add_rules;
>> + rte_acl_ipv4vlan_build;
>> + rte_acl_list_dump;
>> + rte_acl_reset;
>> + rte_acl_reset_rules;
>> + rte_acl_set_ctx_classify;
>> +
>> + local: *;
>> + };
>> +
>> +This file needs to be modified as follows
>> +
>> +.. code-block:: none
>> +
>> + DPDK_2.0 {
>> + global:
>> +
>> + rte_acl_add_rules;
>> + rte_acl_build;
>> + rte_acl_classify;
>> + rte_acl_classify_alg;
>> + rte_acl_classify_scalar;
>> + rte_acl_create;
>> + rte_acl_dump;
>> + rte_acl_find_existing;
>> + rte_acl_free;
>> + rte_acl_ipv4vlan_add_rules;
>> + rte_acl_ipv4vlan_build;
>> + rte_acl_list_dump;
>> + rte_acl_reset;
>> + rte_acl_reset_rules;
>> + rte_acl_set_ctx_classify;
>> +
>> + local: *;
>> + };
>> +
>> + DPDK_2.1 {
>> + global:
>> + rte_acl_create;
>> +
>> + } DPDK_2.0;
>> +
>> +The addition of the new block tells the linker that a new version node is
>> +available (DPDK_2.1), which contains the symbol rte_acl_create, and inherits the
>> +symbols from the DPDK_2.0 node. This list is directly translated into a list of
>> +exported symbols when DPDK is compiled as a shared library
>> +
>> +Next, we need to specify in the code which function map to the rte_acl_create
>> +symbol at which versions. First, at the site of the initial symbol definition,
>> +we need to update the function so that it is uniquely named, and not in conflict
>> +with the public symbol name
>> +
>> +.. code-block:: c
>> +
>> + struct rte_acl_ctx *
>> + -rte_acl_create(const struct rte_acl_param *param)
>> + +rte_acl_create_v20(const struct rte_acl_param *param)
>> + {
>> + size_t sz;
>> + struct rte_acl_ctx *ctx;
>> + ...
>> +
>> +Note that the base name of the symbol was kept intact, as this is conducive to
>> +the macros used for versioning symbols. That is our next step, mapping this new
>> +symbol name to the initial symbol name at version node 2.0. Immediately after
>> +the function, we add this line of code
>> +
>> +.. code-block:: c
>> +
>> + VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
>> +
>> +Remembering to also add the rte_compat.h header to the requisite c file where
>> +these changes are being made. The above macro instructs the linker to create a
>> +new symbol ``rte_acl_create@DPDK_2.0``, which matches the symbol created in older
>> +builds, but now points to the above newly named function. We have now mapped
>> +the original rte_acl_create symbol to the original function (but with a new
>> +name)
>> +
>> +Next, we need to create the 2.1 version of the symbol. We create a new function
>> +name, with a different suffix, and implement it appropriately
>> +
>> +.. code-block:: c
>> +
>> + struct rte_acl_ctx *
>> + rte_acl_create_v21(const struct rte_acl_param *param, int debug);
>> + {
>> + struct rte_acl_ctx *ctx = rte_acl_create_v20(param);
>> +
>> + ctx->debug = debug;
>> +
>> + return ctx;
>> + }
>> +
>> +This code serves as our new API call. Its the same as our old call, but adds
>> +the new parameter in place. Next we need to map this function to the symbol
>> +``rte_acl_create@DPDK_2.1``. To do this, we modify the public prototype of the call
>> +in the header file, adding the macro there to inform all including applications,
>> +that on re-link, the default rte_acl_create symbol should point to this
>> +function. Note that we could do this by simply naming the function above
>> +rte_acl_create, and the linker would chose the most recent version tag to apply
>> +in the version script, but we can also do this in the header file
>> +
>> +.. code-block:: c
>> +
>> + struct rte_acl_ctx *
>> + -rte_acl_create(const struct rte_acl_param *param);
>> + +rte_acl_create(const struct rte_acl_param *param, int debug);
>> + +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
>> +
>> +The BIND_DEFAULT_SYMBOL macro explicitly tells applications that include this
>> +header, to link to the rte_acl_create_v21 function and apply the DPDK_2.1
>> +version node to it. This method is more explicit and flexible than just
>> +re-implementing the exact symbol name, and allows for other features (such as
>> +linking to the old symbol version by default, when the new ABI is to be opt-in
>> +for a period.
>> +
>> +One last thing we need to do. Note that we've taken what was a public symbol,
>> +and duplicated it into two uniquely and differently named symbols. We've then
>> +mapped each of those back to the public symbol ``rte_acl_create`` with different
>> +version tags. This only applies to dynamic linking, as static linking has no
>> +notion of versioning. That leaves this code in a position of no longer having a
>> +symbol simply named ``rte_acl_create`` and a static build will fail on that
>> +missing symbol.
>> +
>> +To correct this, we can simply map a function of our choosing back to the public
>> +symbol in the static build with the ``MAP_STATIC_SYMBOL`` macro. Generally the
>> +assumption is that the most recent version of the symbol is the one you want to
>> +map. So, back in the C file where, immediately after ``rte_acl_create_v21`` is
>> +defined, we add this
>> +
>> +.. code-block:: c
>> +
>> + struct rte_acl_ctx *
>> + rte_acl_create_v21(const struct rte_acl_param *param, int debug)
>> + {
>> + ...
>> + }
>> + MAP_STATIC_SYMBOL(struct rte_acl_ctx *rte_acl_create(const struct rte_acl_param *param, int debug), rte_acl_create_v21);
>> +
>> +That tells the compiler that, when building a static library, any calls to the
>> +symbol ``rte_acl_create`` should be linked to ``rte_acl_create_v21``
>> +
>> +That's it, on the next shared library rebuild, there will be two versions of
>> +rte_acl_create, an old DPDK_2.0 version, used by previously built applications,
>> +and a new DPDK_2.1 version, used by future built applications.
>> +
>> +
>> +Deprecating part of a public API
>> +________________________________
>> +
>> +Lets assume that you've done the above update, and after a few releases have
>> +passed you decide you would like to retire the old version of the function.
>> +After having gone through the ABI deprecation announcement process, removal is
>> +easy. Start by removing the symbol from the requisite version map file:
>> +
>> +.. code-block:: none
>> +
>> + DPDK_2.0 {
>> + global:
>> +
>> + rte_acl_add_rules;
>> + rte_acl_build;
>> + rte_acl_classify;
>> + rte_acl_classify_alg;
>> + rte_acl_classify_scalar;
>> + rte_acl_dump;
>> + - rte_acl_create
>> + rte_acl_find_existing;
>> + rte_acl_free;
>> + rte_acl_ipv4vlan_add_rules;
>> + rte_acl_ipv4vlan_build;
>> + rte_acl_list_dump;
>> + rte_acl_reset;
>> + rte_acl_reset_rules;
>> + rte_acl_set_ctx_classify;
>> +
>> + local: *;
>> + };
>> +
>> + DPDK_2.1 {
>> + global:
>> + rte_acl_create;
>> + } DPDK_2.0;
>> +
>> +
>> +Next remove the corresponding versioned export.
>> +
>> +.. code-block:: c
>> +
>> + -VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
>> +
>> +
>> +Note that the internal function definition could also be removed, but its used
>> +in our example by the newer version _v21, so we leave it in place. This is a
>> +coding style choice.
>> +
>> +Lastly, we need to bump the LIBABIVER number for this library in the Makefile to
>> +indicate to applications doing dynamic linking that this is a later, and
>> +possibly incompatible library version:
>> +
>> +.. code-block:: c
>> +
>> + -LIBABIVER := 1
>> + +LIBABIVER := 2
>> +
>> +Deprecating an entire ABI version
>> +_________________________________
>> +
>> +While removing a symbol from and ABI may be useful, it is often more practical
>> +to remove an entire version node at once. If a version node completely
>> +specifies an API, then removing part of it, typically makes it incomplete. In
>> +those cases it is better to remove the entire node
>> +
>> +To do this, start by modifying the version map file, such that all symbols from
>> +the node to be removed are merged into the next node in the map
>> +
>> +In the case of our map above, it would transform to look as follows
>> +
>> +.. code-block:: none
>> +
>> + DPDK_2.1 {
>> + global:
>> +
>> + rte_acl_add_rules;
>> + rte_acl_build;
>> + rte_acl_classify;
>> + rte_acl_classify_alg;
>> + rte_acl_classify_scalar;
>> + rte_acl_dump;
>> + rte_acl_create
>> + rte_acl_find_existing;
>> + rte_acl_free;
>> + rte_acl_ipv4vlan_add_rules;
>> + rte_acl_ipv4vlan_build;
>> + rte_acl_list_dump;
>> + rte_acl_reset;
>> + rte_acl_reset_rules;
>> + rte_acl_set_ctx_classify;
>> +
>> + local: *;
>> + };
>> +
>> +Then any uses of BIND_DEFAULT_SYMBOL that pointed to the old node should be
>> +updated to point to the new version node in any header files for all affected
>> +symbols.
>> +
>> +.. code-block:: c
>> +
>> + -BIND_DEFAULT_SYMBOL(rte_acl_create, _v20, 2.0);
>> + +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
>> +
>> +Lastly, any VERSION_SYMBOL macros that point to the old version node should be
>> +removed, taking care to keep, where need old code in place to support newer
>> +versions of the symbol.
>> +
>> +
>> +Running the ABI Validator
>> +-------------------------
>> +
>> +The ``devtools`` directory in the DPDK source tree contains a utility program,
>> +``validate-abi.sh``, for validating the DPDK ABI based on the Linux `ABI
>> +Compliance Checker
>> +<http://ispras.linuxbase.org/index.php/ABI_compliance_checker>`_.
>> +
>> +This has a dependency on the ``abi-compliance-checker`` and ``and abi-dumper``
>> +utilities which can be installed via a package manager. For example::
>> +
>> + sudo yum install abi-compliance-checker
>> + sudo yum install abi-dumper
>> +
>> +The syntax of the ``validate-abi.sh`` utility is::
>> +
>> + ./devtools/validate-abi.sh <REV1> <REV2>
>> +
>> +Where ``REV1`` and ``REV2`` are valid gitrevisions(7)
>> +https://www.kernel.org/pub/software/scm/git/docs/gitrevisions.html
>> +on the local repo.
>> +
>> +For example::
>> +
>> + # Check between the previous and latest commit:
>> + ./devtools/validate-abi.sh HEAD~1 HEAD
>> +
>> + # Check on a specific compilation target:
>> + ./devtools/validate-abi.sh -t x86_64-native-linux-gcc HEAD~1 HEAD
>> +
>> + # Check between two tags:
>> + ./devtools/validate-abi.sh v2.0.0 v2.1.0
>> +
>> + # Check between git master and local topic-branch "vhost-hacking":
>> + ./devtools/validate-abi.sh master vhost-hacking
>> +
>> +After the validation script completes (it can take a while since it need to
>> +compile both tags) it will create compatibility reports in the
>> +``./abi-check/compat_report`` directory. Listed incompatibilities can be found
>> +as follows::
>> +
>> + grep -lr Incompatible abi-check/compat_reports/
>> diff --git a/doc/guides/contributing/index.rst b/doc/guides/contributing/index.rst
>> index e2608d3..2fefd91 100644
>> --- a/doc/guides/contributing/index.rst
>> +++ b/doc/guides/contributing/index.rst
>> @@ -10,7 +10,8 @@ Contributor's Guidelines
>>
>> coding_style
>> design
>> - versioning
>> + abi_policy
>> + abi_versioning
>> documentation
>> patches
>> vulnerability
>> diff --git a/doc/guides/contributing/versioning.rst b/doc/guides/contributing/versioning.rst
>> deleted file mode 100644
>> index 3ab2c43..0000000
>> --- a/doc/guides/contributing/versioning.rst
>> +++ /dev/null
>> @@ -1,591 +0,0 @@
>> -.. SPDX-License-Identifier: BSD-3-Clause
>> - Copyright 2018 The DPDK contributors
>> -
>> -DPDK ABI/API policy
>> -===================
>> -
>> -Description
>> ------------
>> -
>> -This document details some methods for handling ABI management in the DPDK.
>> -
>> -General Guidelines
>> -------------------
>> -
>> -#. Whenever possible, ABI should be preserved
>> -#. ABI/API may be changed with a deprecation process
>> -#. The modification of symbols can generally be managed with versioning
>> -#. Libraries or APIs marked in ``experimental`` state may change without constraint
>> -#. New APIs will be marked as ``experimental`` for at least one release to allow
>> - any issues found by users of the new API to be fixed quickly
>> -#. The addition of symbols is generally not problematic
>> -#. The removal of symbols generally is an ABI break and requires bumping of the
>> - LIBABIVER macro
>> -#. Updates to the minimum hardware requirements, which drop support for hardware which
>> - was previously supported, should be treated as an ABI change.
>> -
>> -What is an ABI
>> -~~~~~~~~~~~~~~
>> -
>> -An ABI (Application Binary Interface) is the set of runtime interfaces exposed
>> -by a library. It is similar to an API (Application Programming Interface) but
>> -is the result of compilation. It is also effectively cloned when applications
>> -link to dynamic libraries. That is to say when an application is compiled to
>> -link against dynamic libraries, it is assumed that the ABI remains constant
>> -between the time the application is compiled/linked, and the time that it runs.
>> -Therefore, in the case of dynamic linking, it is critical that an ABI is
>> -preserved, or (when modified), done in such a way that the application is unable
>> -to behave improperly or in an unexpected fashion.
>> -
>> -
>> -ABI/API Deprecation
>> --------------------
>> -
>> -The DPDK ABI policy
>> -~~~~~~~~~~~~~~~~~~~
>> -
>> -ABI versions are set at the time of major release labeling, and the ABI may
>> -change multiple times, without warning, between the last release label and the
>> -HEAD label of the git tree.
>> -
>> -ABI versions, once released, are available until such time as their
>> -deprecation has been noted in the Release Notes for at least one major release
>> -cycle. For example consider the case where the ABI for DPDK 2.0 has been
>> -shipped and then a decision is made to modify it during the development of
>> -DPDK 2.1. The decision will be recorded in the Release Notes for the DPDK 2.1
>> -release and the modification will be made available in the DPDK 2.2 release.
>> -
>> -ABI versions may be deprecated in whole or in part as needed by a given
>> -update.
>> -
>> -Some ABI changes may be too significant to reasonably maintain multiple
>> -versions. In those cases ABI's may be updated without backward compatibility
>> -being provided. The requirements for doing so are:
>> -
>> -#. At least 3 acknowledgments of the need to do so must be made on the
>> - dpdk.org mailing list.
>> -
>> - - The acknowledgment of the maintainer of the component is mandatory, or if
>> - no maintainer is available for the component, the tree/sub-tree maintainer
>> - for that component must acknowledge the ABI change instead.
>> -
>> - - It is also recommended that acknowledgments from different "areas of
>> - interest" be sought for each deprecation, for example: from NIC vendors,
>> - CPU vendors, end-users, etc.
>> -
>> -#. The changes (including an alternative map file) can be included with
>> - deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option,
>> - to provide more details about oncoming changes.
>> - ``RTE_NEXT_ABI`` wrapper will be removed when it become the default ABI.
>> - More preferred way to provide this information is sending the feature
>> - as a separate patch and reference it in deprecation notice.
>> -
>> -#. A full deprecation cycle, as explained above, must be made to offer
>> - downstream consumers sufficient warning of the change.
>> -
>> -Note that the above process for ABI deprecation should not be undertaken
>> -lightly. ABI stability is extremely important for downstream consumers of the
>> -DPDK, especially when distributed in shared object form. Every effort should
>> -be made to preserve the ABI whenever possible. The ABI should only be changed
>> -for significant reasons, such as performance enhancements. ABI breakage due to
>> -changes such as reorganizing public structure fields for aesthetic or
>> -readability purposes should be avoided.
>> -
>> -.. note::
>> -
>> - Updates to the minimum hardware requirements, which drop support for hardware
>> - which was previously supported, should be treated as an ABI change, and
>> - follow the relevant deprecation policy procedures as above: 3 acks and
>> - announcement at least one release in advance.
>> -
>> -Examples of Deprecation Notices
>> -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
>> -
>> -The following are some examples of ABI deprecation notices which would be
>> -added to the Release Notes:
>> -
>> -* The Macro ``#RTE_FOO`` is deprecated and will be removed with version 2.0,
>> - to be replaced with the inline function ``rte_foo()``.
>> -
>> -* The function ``rte_mbuf_grok()`` has been updated to include a new parameter
>> - in version 2.0. Backwards compatibility will be maintained for this function
>> - until the release of version 2.1
>> -
>> -* The members of ``struct rte_foo`` have been reorganized in release 2.0 for
>> - performance reasons. Existing binary applications will have backwards
>> - compatibility in release 2.0, while newly built binaries will need to
>> - reference the new structure variant ``struct rte_foo2``. Compatibility will
>> - be removed in release 2.2, and all applications will require updating and
>> - rebuilding to the new structure at that time, which will be renamed to the
>> - original ``struct rte_foo``.
>> -
>> -* Significant ABI changes are planned for the ``librte_dostuff`` library. The
>> - upcoming release 2.0 will not contain these changes, but release 2.1 will,
>> - and no backwards compatibility is planned due to the extensive nature of
>> - these changes. Binaries using this library built prior to version 2.1 will
>> - require updating and recompilation.
>> -
>> -New API replacing previous one
>> -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
>> -
>> -If a new API proposed functionally replaces an existing one, when the new API
>> -becomes non-experimental then the old one is marked with ``__rte_deprecated``.
>> -Deprecated APIs are removed completely just after the next LTS.
>> -
>> -Reminder that old API should follow deprecation process to be removed.
>> -
>> -
>> -Experimental APIs
>> ------------------
>> -
>> -APIs marked as ``experimental`` are not considered part of the ABI and may
>> -change without warning at any time. Since changes to APIs are most likely
>> -immediately after their introduction, as users begin to take advantage of
>> -those new APIs and start finding issues with them, new DPDK APIs will be
>> -automatically marked as ``experimental`` to allow for a period of stabilization
>> -before they become part of a tracked ABI.
>> -
>> -Note that marking an API as experimental is a multi step process.
>> -To mark an API as experimental, the symbols which are desired to be exported
>> -must be placed in an EXPERIMENTAL version block in the corresponding libraries'
>> -version map script.
>> -Secondly, the corresponding prototypes of those exported functions (in the
>> -development header files), must be marked with the ``__rte_experimental`` tag
>> -(see ``rte_compat.h``).
>> -The DPDK build makefiles perform a check to ensure that the map file and the
>> -C code reflect the same list of symbols.
>> -This check can be circumvented by defining ``ALLOW_EXPERIMENTAL_API``
>> -during compilation in the corresponding library Makefile.
>> -
>> -In addition to tagging the code with ``__rte_experimental``,
>> -the doxygen markup must also contain the EXPERIMENTAL string,
>> -and the MAINTAINERS file should note the EXPERIMENTAL libraries.
>> -
>> -For removing the experimental tag associated with an API, deprecation notice
>> -is not required. Though, an API should remain in experimental state for at least
>> -one release. Thereafter, normal process of posting patch for review to mailing
>> -list can be followed.
>> -
>> -
>> -Library versioning
>> -------------------
>> -
>> -Downstreams might want to provide different DPDK releases at the same time to
>> -support multiple consumers of DPDK linked against older and newer sonames.
>> -
>> -Also due to the interdependencies that DPDK libraries can have applications
>> -might end up with an executable space in which multiple versions of a library
>> -are mapped by ld.so.
>> -
>> -Think of LibA that got an ABI bump and LibB that did not get an ABI bump but is
>> -depending on LibA.
>> -
>> -.. note::
>> -
>> - Application
>> - \-> LibA.old
>> - \-> LibB.new -> LibA.new
>> -
>> -That is a conflict which can be avoided by setting ``CONFIG_RTE_MAJOR_ABI``.
>> -If set, the value of ``CONFIG_RTE_MAJOR_ABI`` overwrites all - otherwise per
>> -library - versions defined in the libraries ``LIBABIVER``.
>> -An example might be ``CONFIG_RTE_MAJOR_ABI=16.11`` which will make all libraries
>> -``librte<?>.so.16.11`` instead of ``librte<?>.so.<LIBABIVER>``.
>> -
>> -
>> -ABI versioning
>> ---------------
>> -
>> -Versioning Macros
>> -~~~~~~~~~~~~~~~~~
>> -
>> -When a symbol is exported from a library to provide an API, it also provides a
>> -calling convention (ABI) that is embodied in its name, return type and
>> -arguments. Occasionally that function may need to change to accommodate new
>> -functionality or behavior. When that occurs, it is desirable to allow for
>> -backward compatibility for a time with older binaries that are dynamically
>> -linked to the DPDK.
>> -
>> -To support backward compatibility the ``rte_compat.h``
>> -header file provides macros to use when updating exported functions. These
>> -macros are used in conjunction with the ``rte_<library>_version.map`` file for
>> -a given library to allow multiple versions of a symbol to exist in a shared
>> -library so that older binaries need not be immediately recompiled.
>> -
>> -The macros exported are:
>> -
>> -* ``VERSION_SYMBOL(b, e, n)``: Creates a symbol version table entry binding
>> - versioned symbol ``b@DPDK_n`` to the internal function ``b_e``.
>> -
>> -* ``BIND_DEFAULT_SYMBOL(b, e, n)``: Creates a symbol version entry instructing
>> - the linker to bind references to symbol ``b`` to the internal symbol
>> - ``b_e``.
>> -
>> -* ``MAP_STATIC_SYMBOL(f, p)``: Declare the prototype ``f``, and map it to the
>> - fully qualified function ``p``, so that if a symbol becomes versioned, it
>> - can still be mapped back to the public symbol name.
>> -
>> -Examples of ABI Macro use
>> -^^^^^^^^^^^^^^^^^^^^^^^^^
>> -
>> -Updating a public API
>> -_____________________
>> -
>> -Assume we have a function as follows
>> -
>> -.. code-block:: c
>> -
>> - /*
>> - * Create an acl context object for apps to
>> - * manipulate
>> - */
>> - struct rte_acl_ctx *
>> - rte_acl_create(const struct rte_acl_param *param)
>> - {
>> - ...
>> - }
>> -
>> -
>> -Assume that struct rte_acl_ctx is a private structure, and that a developer
>> -wishes to enhance the acl api so that a debugging flag can be enabled on a
>> -per-context basis. This requires an addition to the structure (which, being
>> -private, is safe), but it also requires modifying the code as follows
>> -
>> -.. code-block:: c
>> -
>> - /*
>> - * Create an acl context object for apps to
>> - * manipulate
>> - */
>> - struct rte_acl_ctx *
>> - rte_acl_create(const struct rte_acl_param *param, int debug)
>> - {
>> - ...
>> - }
>> -
>> -
>> -Note also that, being a public function, the header file prototype must also be
>> -changed, as must all the call sites, to reflect the new ABI footprint. We will
>> -maintain previous ABI versions that are accessible only to previously compiled
>> -binaries
>> -
>> -The addition of a parameter to the function is ABI breaking as the function is
>> -public, and existing application may use it in its current form. However, the
>> -compatibility macros in DPDK allow a developer to use symbol versioning so that
>> -multiple functions can be mapped to the same public symbol based on when an
>> -application was linked to it. To see how this is done, we start with the
>> -requisite libraries version map file. Initially the version map file for the
>> -acl library looks like this
>> -
>> -.. code-block:: none
>> -
>> - DPDK_2.0 {
>> - global:
>> -
>> - rte_acl_add_rules;
>> - rte_acl_build;
>> - rte_acl_classify;
>> - rte_acl_classify_alg;
>> - rte_acl_classify_scalar;
>> - rte_acl_create;
>> - rte_acl_dump;
>> - rte_acl_find_existing;
>> - rte_acl_free;
>> - rte_acl_ipv4vlan_add_rules;
>> - rte_acl_ipv4vlan_build;
>> - rte_acl_list_dump;
>> - rte_acl_reset;
>> - rte_acl_reset_rules;
>> - rte_acl_set_ctx_classify;
>> -
>> - local: *;
>> - };
>> -
>> -This file needs to be modified as follows
>> -
>> -.. code-block:: none
>> -
>> - DPDK_2.0 {
>> - global:
>> -
>> - rte_acl_add_rules;
>> - rte_acl_build;
>> - rte_acl_classify;
>> - rte_acl_classify_alg;
>> - rte_acl_classify_scalar;
>> - rte_acl_create;
>> - rte_acl_dump;
>> - rte_acl_find_existing;
>> - rte_acl_free;
>> - rte_acl_ipv4vlan_add_rules;
>> - rte_acl_ipv4vlan_build;
>> - rte_acl_list_dump;
>> - rte_acl_reset;
>> - rte_acl_reset_rules;
>> - rte_acl_set_ctx_classify;
>> -
>> - local: *;
>> - };
>> -
>> - DPDK_2.1 {
>> - global:
>> - rte_acl_create;
>> -
>> - } DPDK_2.0;
>> -
>> -The addition of the new block tells the linker that a new version node is
>> -available (DPDK_2.1), which contains the symbol rte_acl_create, and inherits the
>> -symbols from the DPDK_2.0 node. This list is directly translated into a list of
>> -exported symbols when DPDK is compiled as a shared library
>> -
>> -Next, we need to specify in the code which function map to the rte_acl_create
>> -symbol at which versions. First, at the site of the initial symbol definition,
>> -we need to update the function so that it is uniquely named, and not in conflict
>> -with the public symbol name
>> -
>> -.. code-block:: c
>> -
>> - struct rte_acl_ctx *
>> - -rte_acl_create(const struct rte_acl_param *param)
>> - +rte_acl_create_v20(const struct rte_acl_param *param)
>> - {
>> - size_t sz;
>> - struct rte_acl_ctx *ctx;
>> - ...
>> -
>> -Note that the base name of the symbol was kept intact, as this is conducive to
>> -the macros used for versioning symbols. That is our next step, mapping this new
>> -symbol name to the initial symbol name at version node 2.0. Immediately after
>> -the function, we add this line of code
>> -
>> -.. code-block:: c
>> -
>> - VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
>> -
>> -Remembering to also add the rte_compat.h header to the requisite c file where
>> -these changes are being made. The above macro instructs the linker to create a
>> -new symbol ``rte_acl_create@DPDK_2.0``, which matches the symbol created in older
>> -builds, but now points to the above newly named function. We have now mapped
>> -the original rte_acl_create symbol to the original function (but with a new
>> -name)
>> -
>> -Next, we need to create the 2.1 version of the symbol. We create a new function
>> -name, with a different suffix, and implement it appropriately
>> -
>> -.. code-block:: c
>> -
>> - struct rte_acl_ctx *
>> - rte_acl_create_v21(const struct rte_acl_param *param, int debug);
>> - {
>> - struct rte_acl_ctx *ctx = rte_acl_create_v20(param);
>> -
>> - ctx->debug = debug;
>> -
>> - return ctx;
>> - }
>> -
>> -This code serves as our new API call. Its the same as our old call, but adds
>> -the new parameter in place. Next we need to map this function to the symbol
>> -``rte_acl_create@DPDK_2.1``. To do this, we modify the public prototype of the call
>> -in the header file, adding the macro there to inform all including applications,
>> -that on re-link, the default rte_acl_create symbol should point to this
>> -function. Note that we could do this by simply naming the function above
>> -rte_acl_create, and the linker would chose the most recent version tag to apply
>> -in the version script, but we can also do this in the header file
>> -
>> -.. code-block:: c
>> -
>> - struct rte_acl_ctx *
>> - -rte_acl_create(const struct rte_acl_param *param);
>> - +rte_acl_create(const struct rte_acl_param *param, int debug);
>> - +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
>> -
>> -The BIND_DEFAULT_SYMBOL macro explicitly tells applications that include this
>> -header, to link to the rte_acl_create_v21 function and apply the DPDK_2.1
>> -version node to it. This method is more explicit and flexible than just
>> -re-implementing the exact symbol name, and allows for other features (such as
>> -linking to the old symbol version by default, when the new ABI is to be opt-in
>> -for a period.
>> -
>> -One last thing we need to do. Note that we've taken what was a public symbol,
>> -and duplicated it into two uniquely and differently named symbols. We've then
>> -mapped each of those back to the public symbol ``rte_acl_create`` with different
>> -version tags. This only applies to dynamic linking, as static linking has no
>> -notion of versioning. That leaves this code in a position of no longer having a
>> -symbol simply named ``rte_acl_create`` and a static build will fail on that
>> -missing symbol.
>> -
>> -To correct this, we can simply map a function of our choosing back to the public
>> -symbol in the static build with the ``MAP_STATIC_SYMBOL`` macro. Generally the
>> -assumption is that the most recent version of the symbol is the one you want to
>> -map. So, back in the C file where, immediately after ``rte_acl_create_v21`` is
>> -defined, we add this
>> -
>> -.. code-block:: c
>> -
>> - struct rte_acl_ctx *
>> - rte_acl_create_v21(const struct rte_acl_param *param, int debug)
>> - {
>> - ...
>> - }
>> - MAP_STATIC_SYMBOL(struct rte_acl_ctx *rte_acl_create(const struct rte_acl_param *param, int debug), rte_acl_create_v21);
>> -
>> -That tells the compiler that, when building a static library, any calls to the
>> -symbol ``rte_acl_create`` should be linked to ``rte_acl_create_v21``
>> -
>> -That's it, on the next shared library rebuild, there will be two versions of
>> -rte_acl_create, an old DPDK_2.0 version, used by previously built applications,
>> -and a new DPDK_2.1 version, used by future built applications.
>> -
>> -
>> -Deprecating part of a public API
>> -________________________________
>> -
>> -Lets assume that you've done the above update, and after a few releases have
>> -passed you decide you would like to retire the old version of the function.
>> -After having gone through the ABI deprecation announcement process, removal is
>> -easy. Start by removing the symbol from the requisite version map file:
>> -
>> -.. code-block:: none
>> -
>> - DPDK_2.0 {
>> - global:
>> -
>> - rte_acl_add_rules;
>> - rte_acl_build;
>> - rte_acl_classify;
>> - rte_acl_classify_alg;
>> - rte_acl_classify_scalar;
>> - rte_acl_dump;
>> - - rte_acl_create
>> - rte_acl_find_existing;
>> - rte_acl_free;
>> - rte_acl_ipv4vlan_add_rules;
>> - rte_acl_ipv4vlan_build;
>> - rte_acl_list_dump;
>> - rte_acl_reset;
>> - rte_acl_reset_rules;
>> - rte_acl_set_ctx_classify;
>> -
>> - local: *;
>> - };
>> -
>> - DPDK_2.1 {
>> - global:
>> - rte_acl_create;
>> - } DPDK_2.0;
>> -
>> -
>> -Next remove the corresponding versioned export.
>> -
>> -.. code-block:: c
>> -
>> - -VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
>> -
>> -
>> -Note that the internal function definition could also be removed, but its used
>> -in our example by the newer version _v21, so we leave it in place. This is a
>> -coding style choice.
>> -
>> -Lastly, we need to bump the LIBABIVER number for this library in the Makefile to
>> -indicate to applications doing dynamic linking that this is a later, and
>> -possibly incompatible library version:
>> -
>> -.. code-block:: c
>> -
>> - -LIBABIVER := 1
>> - +LIBABIVER := 2
>> -
>> -Deprecating an entire ABI version
>> -_________________________________
>> -
>> -While removing a symbol from and ABI may be useful, it is often more practical
>> -to remove an entire version node at once. If a version node completely
>> -specifies an API, then removing part of it, typically makes it incomplete. In
>> -those cases it is better to remove the entire node
>> -
>> -To do this, start by modifying the version map file, such that all symbols from
>> -the node to be removed are merged into the next node in the map
>> -
>> -In the case of our map above, it would transform to look as follows
>> -
>> -.. code-block:: none
>> -
>> - DPDK_2.1 {
>> - global:
>> -
>> - rte_acl_add_rules;
>> - rte_acl_build;
>> - rte_acl_classify;
>> - rte_acl_classify_alg;
>> - rte_acl_classify_scalar;
>> - rte_acl_dump;
>> - rte_acl_create
>> - rte_acl_find_existing;
>> - rte_acl_free;
>> - rte_acl_ipv4vlan_add_rules;
>> - rte_acl_ipv4vlan_build;
>> - rte_acl_list_dump;
>> - rte_acl_reset;
>> - rte_acl_reset_rules;
>> - rte_acl_set_ctx_classify;
>> -
>> - local: *;
>> - };
>> -
>> -Then any uses of BIND_DEFAULT_SYMBOL that pointed to the old node should be
>> -updated to point to the new version node in any header files for all affected
>> -symbols.
>> -
>> -.. code-block:: c
>> -
>> - -BIND_DEFAULT_SYMBOL(rte_acl_create, _v20, 2.0);
>> - +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
>> -
>> -Lastly, any VERSION_SYMBOL macros that point to the old version node should be
>> -removed, taking care to keep, where need old code in place to support newer
>> -versions of the symbol.
>> -
>> -
>> -Running the ABI Validator
>> --------------------------
>> -
>> -The ``devtools`` directory in the DPDK source tree contains a utility program,
>> -``validate-abi.sh``, for validating the DPDK ABI based on the Linux `ABI
>> -Compliance Checker
>> -<http://ispras.linuxbase.org/index.php/ABI_compliance_checker>`_.
>> -
>> -This has a dependency on the ``abi-compliance-checker`` and ``and abi-dumper``
>> -utilities which can be installed via a package manager. For example::
>> -
>> - sudo yum install abi-compliance-checker
>> - sudo yum install abi-dumper
>> -
>> -The syntax of the ``validate-abi.sh`` utility is::
>> -
>> - ./devtools/validate-abi.sh <REV1> <REV2>
>> -
>> -Where ``REV1`` and ``REV2`` are valid gitrevisions(7)
>> -https://www.kernel.org/pub/software/scm/git/docs/gitrevisions.html
>> -on the local repo.
>> -
>> -For example::
>> -
>> - # Check between the previous and latest commit:
>> - ./devtools/validate-abi.sh HEAD~1 HEAD
>> -
>> - # Check on a specific compilation target:
>> - ./devtools/validate-abi.sh -t x86_64-native-linux-gcc HEAD~1 HEAD
>> -
>> - # Check between two tags:
>> - ./devtools/validate-abi.sh v2.0.0 v2.1.0
>> -
>> - # Check between git master and local topic-branch "vhost-hacking":
>> - ./devtools/validate-abi.sh master vhost-hacking
>> -
>> -After the validation script completes (it can take a while since it need to
>> -compile both tags) it will create compatibility reports in the
>> -``./abi-check/compat_report`` directory. Listed incompatibilities can be found
>> -as follows::
>> -
>> - grep -lr Incompatible abi-check/compat_reports/
^ permalink raw reply [relevance 4%]
* [dpdk-dev] [PATCH v5 2/4] doc: changes to abi policy introducing major abi versions
2019-09-27 16:30 10% [dpdk-dev] [PATCH v5 0/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
2019-09-27 16:30 13% ` [dpdk-dev] [PATCH v5 1/4] doc: separate versioning.rst into version and policy Ray Kinsella
@ 2019-09-27 16:30 19% ` Ray Kinsella
2019-09-27 16:30 30% ` [dpdk-dev] [PATCH v5 3/4] doc: updates to versioning guide for " Ray Kinsella
2019-09-27 16:30 13% ` [dpdk-dev] [PATCH v5 4/4] doc: add maintainer for abi policy Ray Kinsella
3 siblings, 0 replies; 200+ results
From: Ray Kinsella @ 2019-09-27 16:30 UTC (permalink / raw)
To: dev
Cc: mdr, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, nhorman,
maxime.coquelin, john.mcnamara, marko.kovacevic, hemant.agrawal,
ktraynor
This policy change introduces major ABI versions, these are
declared every year, typically aligned with the LTS release
and are supported by subsequent releases in the following year.
This change is intended to improve ABI stabilty for those projects
consuming DPDK.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
doc/guides/contributing/abi_policy.rst | 314 +++++++++++++++------
.../contributing/img/abi_stability_policy.png | Bin 0 -> 61277 bytes
doc/guides/contributing/stable.rst | 12 +-
3 files changed, 234 insertions(+), 92 deletions(-)
create mode 100644 doc/guides/contributing/img/abi_stability_policy.png
diff --git a/doc/guides/contributing/abi_policy.rst b/doc/guides/contributing/abi_policy.rst
index 55bacb4..f1353b6 100644
--- a/doc/guides/contributing/abi_policy.rst
+++ b/doc/guides/contributing/abi_policy.rst
@@ -1,33 +1,46 @@
.. SPDX-License-Identifier: BSD-3-Clause
- Copyright 2018 The DPDK contributors
+ Copyright 2019 The DPDK contributors
-.. abi_api_policy:
+.. _abi_policy:
-DPDK ABI/API policy
-===================
+ABI Policy
+==========
Description
-----------
-This document details some methods for handling ABI management in the DPDK.
+This document details the management policy that ensures the long-term stability
+of the DPDK ABI and API.
General Guidelines
------------------
-#. Whenever possible, ABI should be preserved
-#. ABI/API may be changed with a deprecation process
-#. The modification of symbols can generally be managed with versioning
-#. Libraries or APIs marked in ``experimental`` state may change without constraint
-#. New APIs will be marked as ``experimental`` for at least one release to allow
- any issues found by users of the new API to be fixed quickly
-#. The addition of symbols is generally not problematic
-#. The removal of symbols generally is an ABI break and requires bumping of the
- LIBABIVER macro
-#. Updates to the minimum hardware requirements, which drop support for hardware which
- was previously supported, should be treated as an ABI change.
-
-What is an ABI
-~~~~~~~~~~~~~~
+#. Major ABI versions are declared every **year** and are then supported for one
+ year, typically aligned with the :ref:`LTS release <stable_lts_releases>`.
+#. The ABI version is managed at a project level in DPDK, with the ABI version
+ reflected in all :ref:`library's soname <what_is_soname>`.
+#. The ABI should be preserved and not changed lightly. ABI changes must follow
+ the outlined :ref:`deprecation process <abi_changes>`.
+#. The addition of symbols is generally not problematic. The modification of
+ symbols is managed with :ref:`ABI Versioning <abi_versioning>`.
+#. The removal of symbols is considered an :ref:`ABI breakage <abi_breakages>`,
+ once approved these will form part of the next ABI version.
+#. Libraries or APIs marked as :ref:`Experimental <experimental_apis>` are not
+ considered part of an ABI version and may change without constraint.
+#. Updates to the :ref:`minimum hardware requirements <hw_rqmts>`, which drop
+ support for hardware which was previously supported, should be treated as an
+ ABI change.
+
+.. note::
+
+ In 2019, the DPDK community stated it's intention to move to ABI stable
+ releases, over a number of release cycles. Beginning with maintaining ABI
+ stability through one year of DPDK releases starting from DPDK 19.11. This
+ policy will be reviewed in 2020, with intention of lengthening the stability
+ period.
+
+What is an ABI?
+~~~~~~~~~~~~~~~
An ABI (Application Binary Interface) is the set of runtime interfaces exposed
by a library. It is similar to an API (Application Programming Interface) but
@@ -39,30 +52,73 @@ Therefore, in the case of dynamic linking, it is critical that an ABI is
preserved, or (when modified), done in such a way that the application is unable
to behave improperly or in an unexpected fashion.
+What is an ABI version?
+~~~~~~~~~~~~~~~~~~~~~~~
-ABI/API Deprecation
--------------------
+An ABI version is an instance of a library's ABI at a specific release. Certain
+releases are considered by the community to be milestone releases, the yearly
+LTS for example. Supporting those milestone release's ABI for some number of
+subsequent releases is desirable to facilitate application upgrade. Those ABI
+version's aligned with milestones release are therefore called 'ABI major
+versions' and are supported for some number of releases.
+
+More details on major ABI version can be found in the :ref:`ABI versioning
+<major_abi_versions>` guide.
The DPDK ABI policy
-~~~~~~~~~~~~~~~~~~~
+-------------------
+
+A major ABI version is declared every year, aligned with that year's LTS
+release, e.g. v19.11. This ABI version is then supported for one year by all
+subsequent releases within that time period, until the next LTS release, e.g.
+v20.11.
+
+At the declaration of a major ABI version, major version numbers encoded in
+libraries soname's are bumped to indicate the new version, with the minor
+version reset to ``0``. An example would be ``librte_eal.so.20.3`` would become
+``librte_eal.so.21.0``.
+
+The ABI may then change multiple times, without warning, between the last major
+ABI version increment and the HEAD label of the git tree, with the condition
+that ABI compatibility with the major ABI version is preserved and therefore
+soname's do not change.
+
+Minor versions are incremented to indicate the release of a new ABI compatible
+DPDK release, typically the DPDK quarterly releases. An example of this, might
+be that ``librte_eal.so.20.1`` would indicate the first ABI compatible DPDK
+release, following the declaration of the new major ABI version ``20``.
+
+ABI versions, are supported by each release until such time as the next major
+ABI version is declared. At that time, the deprecation of the previous major ABI
+version will be noted in the Release Notes with guidance on individual symbol
+depreciation and upgrade notes provided.
-ABI versions are set at the time of major release labeling, and the ABI may
-change multiple times, without warning, between the last release label and the
-HEAD label of the git tree.
+.. _figure_abi_stability_policy:
-ABI versions, once released, are available until such time as their
-deprecation has been noted in the Release Notes for at least one major release
-cycle. For example consider the case where the ABI for DPDK 2.0 has been
-shipped and then a decision is made to modify it during the development of
-DPDK 2.1. The decision will be recorded in the Release Notes for the DPDK 2.1
-release and the modification will be made available in the DPDK 2.2 release.
+.. figure:: img/abi_stability_policy.*
-ABI versions may be deprecated in whole or in part as needed by a given
-update.
+*Figure 1. Mapping of new ABI versions and ABI version compatibility to DPDK
+releases.*
-Some ABI changes may be too significant to reasonably maintain multiple
-versions. In those cases ABI's may be updated without backward compatibility
-being provided. The requirements for doing so are:
+.. _abi_changes:
+
+ABI Changes
+~~~~~~~~~~~
+
+The ABI may still change after the declaration of a major ABI version, that is
+new APIs may be still added or existing APIs may be modified.
+
+.. Warning::
+
+ Note that, this policy details the method by which the ABI may be changed,
+ with due regard to preserving compatibility and observing depreciation
+ notices. This process however should not be undertaken lightly, as a general
+ rule ABI stability is extremely important for downstream consumers of DPDK.
+ The ABI should only be changed for significant reasons, such as performance
+ enhancements. ABI breakages due to changes such as reorganizing public
+ structure fields for aesthetic or readability purposes should be avoided.
+
+The requirements for changing the ABI are:
#. At least 3 acknowledgments of the need to do so must be made on the
dpdk.org mailing list.
@@ -71,34 +127,119 @@ being provided. The requirements for doing so are:
no maintainer is available for the component, the tree/sub-tree maintainer
for that component must acknowledge the ABI change instead.
+ - The acknowledgment of a member of the technical board, as a delegate of the
+ `technical board <https://core.dpdk.org/techboard/>`_ acknowledging the
+ need for the ABI change, is also mandatory.
+
- It is also recommended that acknowledgments from different "areas of
interest" be sought for each deprecation, for example: from NIC vendors,
CPU vendors, end-users, etc.
-#. The changes (including an alternative map file) can be included with
- deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option,
- to provide more details about oncoming changes.
- ``RTE_NEXT_ABI`` wrapper will be removed when it become the default ABI.
- More preferred way to provide this information is sending the feature
- as a separate patch and reference it in deprecation notice.
+#. Backward compatibility with the major ABI version must be maintained through
+ :ref:`abi_versioning`, with :ref:`forward-only <forward-only>` compatibility
+ offered for any ABI changes that are indicated to be part of the next ABI
+ version.
-#. A full deprecation cycle, as explained above, must be made to offer
- downstream consumers sufficient warning of the change.
+ - In situations where backward compatibility is not possible, read the
+ section on :ref:`abi_breakages`.
-Note that the above process for ABI deprecation should not be undertaken
-lightly. ABI stability is extremely important for downstream consumers of the
-DPDK, especially when distributed in shared object form. Every effort should
-be made to preserve the ABI whenever possible. The ABI should only be changed
-for significant reasons, such as performance enhancements. ABI breakage due to
-changes such as reorganizing public structure fields for aesthetic or
-readability purposes should be avoided.
+ - No backward or forward compatibility is offered for API changes marked as
+ ``experimental``, as described in the section on :ref:`Experimental APIs
+ and Libraries <experimental_apis>`.
-.. note::
+#. If a newly proposed API functionally replaces an existing one, when the new
+ API becomes non-experimental, then the old one is marked with
+ ``__rte_deprecated``.
+
+ - The depreciated API should follow the notification process to be removed,
+ see :ref:`deprecation_notices`.
+
+ - At the declaration of the next major ABI version, those ABI changes then
+ become a formal part of the new ABI and the requirement to preserve ABI
+ compatibility with the last major ABI version is then dropped.
+
+ - The responsibility for removing redundant ABI compatibility code rests
+ with the original contributor of the ABI changes, failing that, then with
+ the contributor's company and then finally with the maintainer.
+
+.. _forward-only:
+
+.. Note::
+
+ Note that forward-only compatibility is offered for those changes made
+ between major ABI versions. As a library's soname can only describe
+ compatibility with the last major ABI version, until the next major ABI
+ version is declared, these changes therefore cannot be resolved as a runtime
+ dependency through the soname. Therefore any application wishing to make use
+ of these ABI changes can only ensure that it's runtime dependencies are met
+ through Operating System package versioning.
+
+.. _hw_rqmts:
+
+.. Note::
Updates to the minimum hardware requirements, which drop support for hardware
which was previously supported, should be treated as an ABI change, and
- follow the relevant deprecation policy procedures as above: 3 acks and
- announcement at least one release in advance.
+ follow the relevant deprecation policy procedures as above: 3 acks, technical
+ board approval and announcement at least one release in advance.
+
+.. _abi_breakages:
+
+ABI Breakages
+~~~~~~~~~~~~~
+
+For those ABI changes that are too significant to reasonably maintain multiple
+symbol versions, there is an amended process. In these cases, ABIs may be
+updated without the requirement of backward compatibility being provided. These
+changes must follow the `same process :ref:`described above <abi_changes>` as non-breaking
+changes, however with the following additional requirements:
+
+#. ABI breaking changes (including an alternative map file) can be included with
+ deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option, to provide
+ more details about oncoming changes. ``RTE_NEXT_ABI`` wrapper will be removed
+ at the declaration of the next major ABI version.
+
+#. Once approved, and after the depreciation notice has been observed these
+ changes will form part of the next declared major ABI version.
+
+Examples of ABI Changes
+~~~~~~~~~~~~~~~~~~~~~~~
+
+The following are examples of allowable ABI changes occurring between
+declarations of major ABI versions.
+
+* DPDK 19.11 release, defines the function ``rte_foo()``, and ``rte_foo()``
+ as part of the major ABI version ``20``.
+
+* DPDK 20.02 release defines a new function ``rte_foo(uint8_t bar)``, and
+ this is not a problem as long as the symbol ``rte_foo@DPDK20`` is
+ preserved through :ref:`abi_versioning`.
+
+ - The new function may be marked with the ``__rte_experimental`` tag for a
+ number of releases, as described in the section :ref:`experimental_apis`.
+
+ - Once ``rte_foo(uint8_t bar)`` becomes non-experimental ``rte_foo()`` is then
+ declared as ``__rte_depreciated``, with an associated deprecation notice
+ provided.
+
+* DPDK 19.11 is not re-released to include ``rte_foo(uint8_t bar)``, the new
+ version of ``rte_foo`` only exists from DPDK 20.02 onwards as described in the
+ :ref:`note on forward-only compatibility<forward-only>`.
+
+* DPDK 20.02 release defines the experimental function ``__rte_experimental
+ rte_baz()``. This function may or may not exist in the DPDK 20.05 release.
+
+* An application ``dPacket`` wishes to use ``rte_foo(uint8_t bar)``, before the
+ declaration of the DPDK ``21`` major API version. The application can only
+ ensure it's runtime dependencies are met by specifying ``DPDK (>= 20.2)`` as
+ an explicit package dependency, as the soname only may only indicate the
+ supported major ABI version.
+
+* At the release of DPDK 20.11, the function ``rte_foo(uint8_t bar)`` becomes
+ formally part of then new major ABI version DPDK 21.0 and ``rte_foo()`` may be
+ removed.
+
+.. _deprecation_notices:
Examples of Deprecation Notices
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -106,46 +247,42 @@ Examples of Deprecation Notices
The following are some examples of ABI deprecation notices which would be
added to the Release Notes:
-* The Macro ``#RTE_FOO`` is deprecated and will be removed with version 2.0,
- to be replaced with the inline function ``rte_foo()``.
+* The Macro ``#RTE_FOO`` is deprecated and will be removed with ABI version
+ 21, to be replaced with the inline function ``rte_foo()``.
* The function ``rte_mbuf_grok()`` has been updated to include a new parameter
- in version 2.0. Backwards compatibility will be maintained for this function
- until the release of version 2.1
+ in version 20.2. Backwards compatibility will be maintained for this function
+ until the release of the new DPDK major ABI version 21, in DPDK version
+ 20.11.
-* The members of ``struct rte_foo`` have been reorganized in release 2.0 for
+* The members of ``struct rte_foo`` have been reorganized in DPDK 20.02 for
performance reasons. Existing binary applications will have backwards
- compatibility in release 2.0, while newly built binaries will need to
- reference the new structure variant ``struct rte_foo2``. Compatibility will
- be removed in release 2.2, and all applications will require updating and
+ compatibility in release 20.02, while newly built binaries will need to
+ reference the new structure variant ``struct rte_foo2``. Compatibility will be
+ removed in release 20.11, and all applications will require updating and
rebuilding to the new structure at that time, which will be renamed to the
original ``struct rte_foo``.
* Significant ABI changes are planned for the ``librte_dostuff`` library. The
- upcoming release 2.0 will not contain these changes, but release 2.1 will,
+ upcoming release 20.02 will not contain these changes, but release 20.11 will,
and no backwards compatibility is planned due to the extensive nature of
- these changes. Binaries using this library built prior to version 2.1 will
+ these changes. Binaries using this library built prior to ABI version 21 will
require updating and recompilation.
-New API replacing previous one
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-If a new API proposed functionally replaces an existing one, when the new API
-becomes non-experimental then the old one is marked with ``__rte_deprecated``.
-Deprecated APIs are removed completely just after the next LTS.
-
-Reminder that old API should follow deprecation process to be removed.
+.. _experimental_apis:
+Experimental
+------------
-Experimental APIs
------------------
+APIs
+~~~~
-APIs marked as ``experimental`` are not considered part of the ABI and may
-change without warning at any time. Since changes to APIs are most likely
-immediately after their introduction, as users begin to take advantage of
-those new APIs and start finding issues with them, new DPDK APIs will be
-automatically marked as ``experimental`` to allow for a period of stabilization
-before they become part of a tracked ABI.
+APIs marked as ``experimental`` are not considered part of an ABI version and
+may change without warning at any time. Since changes to APIs are most likely
+immediately after their introduction, as users begin to take advantage of those
+new APIs and start finding issues with them, new DPDK APIs will be automatically
+marked as ``experimental`` to allow for a period of stabilization before they
+become part of a tracked ABI version.
Note that marking an API as experimental is a multi step process.
To mark an API as experimental, the symbols which are desired to be exported
@@ -163,7 +300,16 @@ In addition to tagging the code with ``__rte_experimental``,
the doxygen markup must also contain the EXPERIMENTAL string,
and the MAINTAINERS file should note the EXPERIMENTAL libraries.
-For removing the experimental tag associated with an API, deprecation notice
-is not required. Though, an API should remain in experimental state for at least
-one release. Thereafter, normal process of posting patch for review to mailing
-list can be followed.
+For removing the experimental tag associated with an API, deprecation notice is
+not required. Though, an API should remain in experimental state for at least
+one release. Thereafter, the normal process of posting patch for review to
+mailing list can be followed.
+
+Libraries
+~~~~~~~~~
+
+Libraries marked as ``experimental`` are entirely not considered part of an ABI
+version, and may change without warning at any time. Experimental libraries
+always have a major version of ``0`` to indicate they exist outside of
+:ref:`abi_versioning` , with the minor version incremented with each ABI change
+to library.
diff --git a/doc/guides/contributing/img/abi_stability_policy.png b/doc/guides/contributing/img/abi_stability_policy.png
new file mode 100644
index 0000000000000000000000000000000000000000..af9e8e0b559a8724ee0fa83f08f6a1578316739f
GIT binary patch
literal 61277
zcmd42d0dj|_djgYOirI%hMLJWoxohm-1kJs2{lVID^pT2Q#AL~Pyv-18>KYKC3ms1
z(sB*=1+6qUR5BOD1<MTsm6VVa!QXAo_w&s2`{#Ln|2(gU*Q>yr>%KYXT<1FHT<`O~
z?wmVoEBPP!|A>i+N!p!)ofi`mFB22{Vf*Jjz?Js|(__Hfu88xte~MKMD1HL|_|ebG
z!AeZ*C1Kx|*H6ITd#|5@M~I0@weS4f)fHOeBPO=8W(Tvn7>!sGN`CQif&b2OF+7C#
zdG*Tu*DulsWPfXT<?!=h9_6PaI+_hyuWu*CKfH2z9s>T++DWnNUHtw(&i&P!D0Sl|
zB}-5BjmOW&ie7_SK<W2SHQcN=aLSbT+LOKS_5GC_!s+@Kk$(J7o1%(MDD`qE?YUW7
z1S`LqN_Qqtj|*x5+5Y`nPmbIM?Odpv_Gd<IUJw&|Wj1-^(?!$8SfTB|2l}Q#%Xp+c
zVmC+fR+}4?M*sP)?C;x#|9lKQ>?QgA<EzW3Tz7x}Saop!|K~#?<|6U9ndD><pv{$A
zDrLI>QLfxRTa0qD+13yftNP=+NII}KKVG982EJSD)r{RYfnWWCq6UzG8?LPWbKA`q
zQK&~*fQp*g-xZb6DY30Z06JB<et*n6ornK3<o@>|2FWbD@^AN~e7|tf=1YavmmTB%
zlv=gpK{tC<pe5g|`S$+z3$1%GE;8Rl`Qf|m{+Kp^76U_W@)m0@tJ&t`YDSt&W_oS5
zFdf{hBC-G{71Z$WscRw_dzhh`UDmT#Zd@*0v@AH)JcarCTN^)tz3I5?<`J{;fPzEP
zGB)`N%3su?JQWYO{u-k$TImO6@jZXY^zY=%T)>5i@5pw~(Du#dv^h^rD=E<~VRxm6
zsBoH}z}*~rRe|2VpfqfTF<}2KZj(P6d%V`oxIQJ)Lg7MI{eWP%<>gyPD2(?YnXW{^
zhA`{ddq?-}iyeNO+Pf&fe{+pjL}PCVU6Wk8k6h*@?y)Y^zMCXy)a5;7?1N1GiW3k=
z2a75L3i#rlD{Xr;y9Y0^@2q}a`Fh2&mfssRv+qs$%ana!t>v~KHOG^u-%hp5?}!Tx
zB?_FHvkaH+Tbgi|M(3GvO(P-O81p<8c){7EyC0a|Iga&OmK<?G<*t42KHZB6r?>R5
zer0zb?vS1rmSZc$^8S4D{Yl5ME?w<4CvVXPmMeFU`DVp#Q!W<_Y#+FguKFTxWhwy0
zm}=UZJSk+pQhedV8^by~LPq(WMdhPOn~t|6|3}9<ywZCNygjpdYtl2cb|R@Mg-YOg
zj5e*=9?bQLCqEeqC^-1CPZ{SPQ!`@LY$;hrzc`z?vs_-;<gK=%<G1pPMX*p0$?c8U
z(09-FG^?bKP(B5t(d<bImF>9ug2{M*&~nwQdH8mCO!xG!oYfC_H-YvpZn$(H@DMSv
z>^I2i?d-2n@RPd>71+j0I`$fgY(yl<>@C(udnHNWc*`*Do70J1;cT?wJ}}B71ye#l
z%3z4+WYdyhYBo-PnGlVGt?!9}x4zo6_L@7lX-UJ@?WZR3TcU9Cxx)vJfxj>HSB!|c
z>#A=v3rft;xA>h21vLL!A`Rj7gDHbO7fp3m?Y_0^pM}JUzStoHAr-%Kfvo(dJ8%HJ
z#u0y83}NgKyWF8;DW8hNI&^q&|JLoD#zAMA*Z$s{fmEF;cJ3j<D+ym+JCe9R8Nt>o
zLV(9(NjFRlzM5g+gywaGLMpfT&O+0QLMk)*xr^S~cr3P~5j3PeaYCI`(eq@cq$y$Q
zGep?sy!l!*`(n7T6iv1lY@G8v7H@GZYNOf~`m*SM1T@i}Q6l$@YJdoAhm<en!Y(VN
zUsLaO72eA>Xfz*q0P?)s5YfS3*yruj9%Gm60b2c=F=x~5!;rQ$*PD?8rK>E#|3+|F
zikN7_$m%SVNNiyfF9XZK9GR~b^x^q7nlBLDO$!e|a9WvJWB1(~m&-3ql#lk}CR3qJ
zvt*ZzRvt3|p)C$k6~1&HbwkCbQ7j)Cqx%rTB`3!q-M!x?cT9nlDtHd6hbi{TyrJ6I
zkKd0ChN(7AIGgJ}R>_){eTlbOrCP_-7}GK5I~q{F?J=qYAxRA;o@2jn%Q~&kOG8>m
z^L%#OPY9wAbjs$i=*zQiViu2=pL(aV<=M~>B$ulUN@a$Ps&SILVLUa6lDv~>!40g1
z??%w7uZ<e2Sr4Q7amiy(Qh#;LclOMxHSfB#tPesXm>Z(47_T34Znk=MBtk@5p0Zc8
zW7bR5B9p(zz$HmPYad9|`k!%Vm=Vi2O~d;V{fbDR+|x*NR=m^DVr)8SM!nPhgf1x@
zQ>nKxHw7yU;dDK(>~7iCpzjMkbh6Xp*>=Oe)9*EL>T9#0!n^8v@bzbS0W8McGyB5m
zUN~*8HshJy%1djr^3m<_LifgK4(F3&#d+1VGohdC;vPElD+WLk3G}2wDC^E~weJp$
zJt-wlB&9ZgxjKM2l`hc?Jq^7b72*0G?qH@F4ORRq4*qh*w`C&K)7G)~Qt1@J_Rk=?
zl<nkLo_kX%B!Q=nG%n*Z+SwS+8am1sW>+V5Auc+ToHSjief<YsN9Ol+c5^Cf8^0T5
zs`ypedYGK8_GzRxAV=(2dnRGL6=U*QdSRa=;ojGCu;RM}n1p<!QTlX_Sa(|w)^njr
zKlNND77k0r30m+$vf5nBL#h1b@)34fR;YYUL1Sw@OycxogL=clHZVBxiWa2}I&&Cg
zupC%fD#N4;yFAj`Kl=(v8qXRR)C`x+xR^c!v%TVz7mxmgcRE;3(0rDyXoU0N6!eVf
z#qftV3Gs6qt7!}$zWC)0w1vG;Zg6YqIkTU;q03>8mQcr#7aW7GK>x|}&6N^)1?{(R
zhdqBD`ofZbTGp)6E;COO&6aq*PLvc!$ah9yiNB=oUA9a*MW-0jpfAF<CcqVhlEmv4
z<n3&>NQiIoi(*iXIaU}4n3R6Oy{#dGCT0^t*{QBNlPws;Q*C@^MLLb;g@kB-m{Jm0
zp|V<r5rrxYOFYYF2%kH<(F9s2q5pYPXj2AWb;4vlYgq~4BDnE=qya_JMkORp?J+`?
z;N1AE^6oW;ThurP38=hxH1b{zjnCNFO>*I@Hbd@8lvY!uYu2jQ>(~DNDSMvl?NsR2
zZsE;*N^EGqG;-j3mam#1y(X!ov#^Lvuh&B*2#+EkYA@}JYBS-8T<gzBSE^V0Uo`$&
zkd(OS1%4@~;9fO7Z(R$tjfF9&)e_<b6sf>-RBQ}Ws9pG5D#b_pec|0x^fN}Z0=szo
zCzd8)g>G)BR!X)_2y-7My}aIL4!a;gnrl5)`ElDDR{gO_ePWX<>>?46bbHu!^S$hi
zdG(*?MLaYSIWwybIr!kl3*@0p=zsKA)534y`Cn8CIs9B@pHWmC^szeunS-^NJAPoT
zDaIAXWi$<q9;1v+JKX1OA`DizT0$QfH_?xq3uvBN=oW9`?e~;uGWajeSqM0t@~kBB
zCtOTNRl+<Nd)4pcM%QL)d!onzX7o@w+4Y8Yfg6ND%Ud#PFiPywB$tp5*C|Q0NZ^Q-
zmS!`vpjv6$t;?a|K95DVV8&qHl2U0Npe1L~a^dCIi=m`r+Y2&aRj3s5#08}`<)o7I
z6`W!!w1o80Ora)El4eb(24!nem=>8nGb+etT3O)Yw8xoFB<Y1p{drZ_ilpuio;kFl
zx}s<Eg@i{Qzh^+{Zy+3CMdx6_PPCZdQIs$F0SXoiJ;*DU%Dp4>XFWhJah^3aZa0L7
zJ&tX(Y?MMX=9quTe?e!Vx-LLSeZ@krVfCT7I89c>F-bvkK?Y_<>V%Wy_sZ;5h{DLR
zQVDjRB!#XYkZ*e_m90ri94NtR4IIS=KeKAjETrN3d{9%roCwPPd>PKE@(D5OFvj}+
zh8w~4A+B>(=2?%16D2ujJ>(XB^k5MFEb>G()7i*9BGLX}m2A$Jug~zlGiH@9t!aDP
z_7B?Me`@W{b)dNV9kqM5_0l0W=-G3Z%-Z0*D+PA5sMaWXR9=c7Y|d2wMW3JUGPCQk
zMZHaI=fD~0U^R+&`=97r@&4Gck@j0MLlXOl{OZ1rNU0<688<LOHg%1d#94CCvX@ge
z9GNf<Xj6KR3tIoVvPeSCEUBf^N6HEz9dCYSrYXpe!PZyUPSlX3Nhf#DT0U<hsPBcP
z7zaDHTEl^b=(fH>=|D;O<yppToPy=no-JYLVoD}(GP>7(j7Pe2xfS|}YH>jwc@P$|
z_xP{5J4$$Tl@FO!rlLFx+bsxOt`_ds%s9ixIe3o*uR>*tXfihebG3`L_n+M7fgD@P
zkYx0C_FCTQ{`ls?`q)MVW3EG_J7Wwc8DHp47_w&Ud#R*ayRz%z`Z0w*w%0%&OX(xZ
zr?M#H-pWw#+~kFwM7BrxKF@$-yAc5zT37GCy>)c|FOyy#Pz~Bte8g|zr#`e?(tn=j
z``je}Su%0HYYvLCEf0OiBkEEOEw9DOwU2Jx##3#5roQCkttbZlmn(<a72ZUlM=@JK
zVBc@2S6(3Qu3T|Yikh<DMAx6rTW@?y9J_`2USo<yy2u%wfT~};7>`^Z>#<pbXFRgM
z&`JeM%Gb3lxva|vNJStdSRh7n3vHol-nPeP*4TO&dYb!{tao)>>1qFAOL$$RKpzNx
z6C+Ba4yc*ql}X}XRGDiPc>A>ZO+7ioZoP^Pe_D>E*ycIdXek=UP(0fxnn*jR-~+_J
z9Mt6Hjy;}?Z|Tu;(Xrx!ob!_yB|quOEZCDJRI>)M@p+*e>&}5-hyRB8=SoI`PoQ{t
zjWb)9Ve*k}Jz=C`o(m>@A5BpeHL1famAzY&35bYDeR>Q1Ug>Fi2=mFwD!u1Lwh>=h
zBkUVShY!~U)LU;kgobOi+qj*gG%Wae_-*JqsNNyU=&}jF+5pR`V)l}%MREwM>|^4g
zRU$fW{yK$ysgP-TAoWg%eaHgp%WFJp;tTaSuj3%?e?oE|5v2P`=dtM%AIvq&XU@{l
z)<s4sJ+I_T6J^h0(EUTTp79u;%0H(lG5dE`-f~P_=-Q8THtUa2(uoezqjs5(!x!{*
zYg0ZpQpg4TztZlTDu#K7hKKNKJv{XRDVZM3s>A7Lz@g{kzeNmHxUDu@x)$3r;BYQ0
zbkv!?&p3TVnc!L<{B`mLp%8QRrJ(uZKj{J+E<4y#cthSWDAGy#q*;(`ztvhSQEpYV
zQjG|Gq(?r5YG>ns8m{W2lfG$W2~e-O!AzxU1+W${xIR|Cv8_#Ju*JovRTK-#am_Zc
z%DRIOFPv^^`>Jg_%<dDoAr7Rn`(`gExcVZPEz6^qnolUQk9yF|IMR{GQ=b**KZHUU
zN~tuLSVm?bySSNp@I-PLbo@`Kd<Wnja2gzAXk>f2+iZphnr|3>R_>As?<)6b9>F-b
zb#Z$;u$hR)yoVOlB&(u3W$i{WA<xw?brr;a_Nr46(|AQ21BdtT3VLqwd8&@GHq|Ej
zV60)(i|)=r`^(6apVT14-(Ds&F81%)3TrPCHRHoQyOyBDiOrXBb6+fBK37Gaw)Si-
z5W2G4{9s{9H)<iQ+h*a*EKo%+l9*rH?x~YQBJ$X?ZY!ZVUA#LWjEnaB!GY4v3#Egf
z%JdH#Esh^g?Rjdpp?l9he2(W_+3fE2sT`!ucF=;whAvGOs{xrkGpwIV8rW27!YQSq
zV4~mE*+4nI1@HKMC;yerj=q=*m{OER6BrNh@hEf)6D2I((Rqjz_p?zFFs0`mC%sDl
z8+@Q>$MztD0h|6XAd1qnbI#uFJ`7E2<5)^@)j+0O96IPJ?cHNTh0mdjT{<~8sU29U
zc=(FTR(IUo{do@F1TupJ4RtR^pYHexl(UFE2dmIOu-G`Dcvc7nr5~NhkredrVZ{(F
z#kyykN|avw$`o3umRvIY^0~kKf};nDu!8L68sUE7R)hMvb?*Dnu@3|<w9c!dPLnh_
z{+^n%xrNArMtfD9R!X<iVNy!B;Fm0n9@f^V|9qvUH_q5*J$mKKe%hJeIAMV;G)G=x
zNZO|Pbj^fSMnq`3W&Q90sGGB6NA1r$8^{wgHQeg1Q>Cb(rxEp7;s3TB{XpVa76ChK
zOWg-2-L)NyBOam{%I@R1J@&s_+*CdwjXr@#n*E}@SmEIMM6NA~{XaX!baL5=Gs^N?
z=X(OO5zVxdk^Uz?Ma_(7lqK~4Q&7HQ++Zlrohv~uj#&%r*$QpCiJ?FFz6$~17XMlV
z4cDaea)95_<dIip^?lDKHr8xb(Bik~Ojg?$aiNUAeXW37*OI8NQxr?a+H02;qt?q(
zP!{B=<~HrB!~eZ!KoGB8-@2+mz4Kg}hsbu5G8BDw<v8qY>o=k0+XQ@0<#X3{%Ifti
zo-odDSJM9hv#RQmj*FpNolE}0_jo!MQ8y79fHGs=NdNHa%~D{k#c)Wi(j*`#-O<8P
zMYKMOH-1av6RuMO#?E~Y3S#%|*b&+|Wa$w4x_NU8f2N}%pmt*0jV;h_{(~f+2}xXK
z&;rN?GfJypTF#cD{5$rxZ*&FHtXtH;iK159E#XhAi%qvBRll25%vx$bc4eMfyIy#s
zFgnr-CiNRb@$T0JJfX&{pzv-3ZYx%`R3=tA=n6iv6t&UH_+@(A&Ay(ErzrtHfaiBa
z@Lwbx|AlsRXU>=KLj#-}%Arj+Rw65L+~LB-^3so%Z5+w*VKK3XJMfTTldv-ocvYSM
zM;|e<|5Nxcf6)n4fUmTfJm%^8QV8M@x`U(PtyPdqvF=ekle2+SOfFdCg4mU(X5|BZ
zo9R)DH6hE60DiacJGdei@4>*eoNVS2MZW@ahyijWamVm9+ZbIib{iQ3ojDGQx+kla
zwS5i-H=2cjm^OXKu&$ZQxiwky2f(c0=d(kgM$ckv32qT+qn-w{Cv9tB9T#-Bz99^*
z`<dgc<-fg&Jm2SmI2}(}2%(1yu1G0ikCh3O_rLskDC9yg5t$TL_*3OQbph1hfQFjJ
zTNSp25?nj{)Wxu4sTW!!tA<!DdKQS*eCq7U1&2iG>5bwh{T&R_sG|pj#E*NW+RZri
zB)Y&PE?0CQLELMqKdRc9XQ;H7bl-jzWi6q1=FO%EbmXH>e|VDbsn7FcPY%B}o_BZq
z+d*!B45%ySLghXCY=m6ug!5H*sGBk&dF$}Gf`)-j)Tl~WE;8>fIy3lk<eMgu)#yW6
zp6O4oD$JI*&Kd(q*+}kp<Th#bAwCM04;ozQvIEK3KDhuU71JJeZU*(?{6k5+V}ND)
z=c^OP%6f9_ODqj%GVv!j{`^YE+ofC6Y%+q6u6E$HttU-q?t!XRh<i4=@Pu_$H=OKh
zEki{!KO&gdd5G&^SLVph0nu#Dp9zBW;M#uLEG*3Dkl-C&+&2HWRMfUOm=qW5)}h%m
z3a@*f?MH|EgbFsfM`m|{HssQYN`v{^c)U4o&7@TyOqf-JteIrCZDiv2f%S5}W6an~
zQwrD5X;%gXZBO9yE!7WuXj=yEFIKrl%8L9nXlzHQ4kOR>Zh5hDpC<C2xkk;{OS9jO
zjm(;q2iyJVNGiy?^z12-(mkQ#^QPv@`k@gB+FfY6Qtg!iuh;0WUec+KIXFzFGEGy&
zmYgvFHv9^+=p|W|ngVlY<S8!7f%r+?3~Jj0ZG*WrpxBGVP9Y(pngO<DtMN<(h~BP?
zU5O3tFb{7SdrvXLw(nZ9Hf^#|jl^k!*=u$gZm<}gnOSnr?4<l^E5nCv2O1JB`eg<m
zU4Tb=e1jK$1l;Ay1Q2*8AAK3lt%;g$;XgQg#DA5J_qubdrTEjdbzhp#z>oY}cy~Sk
zD(w9NPy3x>RKfQdz#(9<g|rq<pp@5%a#F?nByC9I;vrP!>bN(=;#a54;}quS<@PzO
z#o?=rWh!qg8pahyc+ii9pKBcLGQw&aahy&pMpK&zTt%9fFP-Vcn$;~6XFn8%+Drx3
zpp&_1Mq#VPL3H5Fu&%39U9pp=B+?a|t;kOkkGrrKmdh6GslPsOK^vDlONfo}>)14J
z_q^g2Twq5v*kUBsD94>v&sViQ%8t1LQ%bk%Cpo+|r&J7;SuXytqw?<WD$5RZyqaD6
zdzW5M)}8DOXr`Leezkt3E2#J21Mevfi7TccbSU*0xb?pFZBn5h02X;>JHeyu613fv
zDyIqESYM{_%ueZ8)`E*xElD^su&3u^BsY7mDO~w3=T5`n8Rsd#_y--%D-o)Np-Eq>
z@fj39y});M8T>w&PYBMb4)<R9k2#FUY$giK;#2kdQOLFB7kH<WmN+*USE;s`uRPQu
zFrkDULV^V5<?enF+gxG5#yAQTUomCgnRBCGFhuJD<MQwGHI*LxjNpe+u@1N+SH}oG
zEX?&WY0utnDb$pH-?Xg(ArvvaF<#=G4>=(*&T;CBzK<LcayLswMLtW{d8d>&;uD=L
zS%7VJka?Fh>wcvhwRV`95xvxlD<{X@{h*;2NvO1C`6q~qR`JCr>|+A6k|^&_w@d>j
zY>!*olY{$tqfjF@BfZgn{pul<L>>T{vU`=Dpg0Rl^virRrHcbRQvYpNRXGZ&Ur-^1
zrM@3(L`;{=-e)?flH#<k=7v9~(I->#qb`S%0tW>qJ>Qk?Z^0Ds;-17<1S#)1G>ggO
zjy>Q>HLV+cYRE!KHN{bCJs8!E<)|s6vK4<Nu0e@Q<WLKs9HyW>GpA;yl3L(vI?}s(
zsJ1oS9Jt@#zR4lJuR2JV;`iEs<5ozJ!^t^qi;3SPcSCpDG0+$paaX*P-JP(WOe>os
z4>Jp@v?N9G>&UdZGX+0>b^Yzgo%^~0&3<<0lg&Dju|taSRxDWe!{y5QN9dSdQqlfX
zPy5)*idiu|{4Fw`Gyor{z4}g23`kQ4Iet~S7+S&2^(oAoFbH6jMARcH&=PEPd&+cG
zVCYbapvuRe+`7M?e<(9r3Dq}LvzP5pSXD2|l&hbr4WnV4)(s()YBX`=<T}AjO;z`k
zk)$4#^tOcUpY1<gFM9On9ea0nm}HfUzshd0o4jlw2p@jED}`TyS0tWDOo)wZJ*RQ<
z&4LbDxlg9EyG{5BhzK)y0$<f-K@Ad~J!y4v%@^!5`Ab8s?IkB^GS;a>lP^uQmQylL
zR41zKb&^Jk+&2C^$zTdb$UO%#kP+GQ$yTTJR#!nsRt-bGeu<$f<6S#U&#yqSZFsNw
zaGe<_Fi!KoTDt^~8CnKqGaXK>w67Et_Dn31b9o;g#T+(B{Hf5_<TKhHn`{gBnX<e?
zu$fygE~ApREJg`ie-=)hm!?y%DW?E%g_=#qqj}|nskFWq7DYS0nw@&v@YQO_y=c$S
zwc9FvDtn)T(^cpY?RoOWtOq3us@PD=T4U4CaixVx-M8&9m$$>RCq?!%HKkuO@iI0~
zmZ1Cr#@zXN=v_+yL9DU=vu_+*A)zLs<L&o56lv!1w+DRO?Q}+*ddWsqtW^4)cEz=g
zeh^R4rg~K=uZG}28D~CQi%&7%v+dV<4t)tWDw)%=l^5f68r`;#cCIvdJWTnL4Dq*A
zONnl?s^R0(%eL|^{IWMy-sZiJSt9?fKVhOP>gnarx75`+(?TxDG;~}zMp^5ULu+#6
zQoB|lAb;mVryWiEPQ8);0DCIqU2wYx2DeKQ1$59b>AR66vqX#jA8l2!%7=6pIn`mp
zO#rLlPiW#S&)OY*y7beOeT>i!wk~T=$A*u8^xfV;G2+SuWl=%ChqNNkqg#mk!>nMc
zVW&Bbids?c$oI0!S5OGsOl<FuS1+vRDxy~Gk0!PWV_@r}ac4Uo@;z^3$f)G;E+l-!
zqF4cPn)9*VcFr`D<{)L7Pc|IVvvnxpm!~)@^Q}-`pP&VnN<Ek-^nK04&>VARgWOlD
zdG#xWZ<TF*DnD$BTS=!xJHtfkV9M5Y7%|;mix8LsyU+B`osW<LRKEEe(8e?weE8@(
zHKiL7HJhemdLw9^)eDp`KB(RwmvGAtDT=)f60h?}%qf-A=QVZ!kF`?MIgY_;$Kec4
zbr8Cb@v9U6A!Qkd>^O>jC&@xFl5JrIi6;>q_eVBP#!FlKY>**f-0x0pp&OvrExuD-
zNY@by8KlX!qm9pu8TDm&zfYP`Wn~y&k2s=wi0|v+igC<(^irI68ZmPH14YMis!d`d
zM$_0T4v{hFK(beUm-j8H4gnb^Up-w1O_(-t*sq#yYT6g6dPINl0ePX5*LjT3h`j<+
zeZEHndQk1qUkyrWd;t2`k2n%{9Bi2Qs^H>!jp%e}_&iD6Lu$k=?Lqqb!@Rj{X?-?{
z+v0b@R7U^hMhiZ#iF(lQ!1_ynx?u7mgYz56H2->R|6J_f+631c2lJKw+h6!FACHUt
zujSYyejc?eA73$b0|9ySbZR!Wa!^0xcKqtnA8+KZRK1Ugey?J$DkIVXr<>9bXk)j}
zrcJ7I((gQDGWSW+n>UY;d+u-wH<R%sAPWB|CI3T|Jn!9X`nm{F)0D+m5xv;X?lPx5
z*PyJ;1FL>`-tZdTtCQ@P8KG)hEO#k(?DNb#v9tQ}s6~odMLO_CBWbphjp^hzK?`(7
zt@e2rxdPHFTw%`zPaQMG+3WNZ0uxL%z}UgclNyOA)el_7tp|AX%nisrE54SW-n_VY
zg7Z!rdF%>MoX5v%fw5=&_Nyj}DLpXho_oWw@u5=e_xn9Yz9&Y{$G$%UA=2}fdOyC!
zS%R^fPwiF;b0^NP)SKB6UYXG|IF`?23MWUy%`*!w_V5K{`~vv}m&Zx!ew3@`aLWBR
zr<K6XI{ap~rO}Ksn4~HGoD-Z+$@KPUc7&-GvUw>OswPvj)Q*7r<TSD5MC=044$};t
zdFD&PYzo_i!eCyOg1BSOF<Y62Q=e=bfy8B9$UT_T$LN?Pvdm`wX#Ir+sJ-um?Tu6E
zqK0RmI1oK}KF2v>Za_~|$Cx{9e}PYhR0=$k)$6h9Yoq^xE6<r0+}UpN+kS|#q{f@8
zZR^b-Hr^;@SayD$z_(CixEM{^>E{ZF1221()^;XYtm+rjU$5I;L`pRK1v5BxqGqSn
z5xj5sVTGw`m+6OX+Pa^F;rejVTfDpPdqz{`z3#Tq5Vwf!L~B+1J-qA3DV_&({jX}L
zU&xW|ooB^Xcv8RKfT>DRxXJz)HCoqmxtxrcNSNxMs>drQGZ)qkC_%$m-<Bhf6^p&P
z4nI(FfNK$~Rez&!Nr(DpEdmN0C}Uk=U8oG?61M@3uKbekk~S>YADcHXo8_Is4wmw2
z^Y0?7qhOs6Q3mW+UIL4iuftq+34sfw_WkLjLjU)hZRp9)<Hi9KY;T=Kj_=en11hQA
zehvP<C6g6_ah+n6$s|~hmWwSJ78gu^0=N)Wm-XXhXzlV>8ltRysBbc$Qx-sXtzqfW
z*RS{`yR3Wb)~(DF;P7kXPJ7cYGlopD3XXb;(M}0l!BXvc<U_VSj6~B(7J$BpFOhsI
zX~nP%?+o%U%5qN1MSPh3QEas@wR3vWqWNS6yH7N$^Ry-W>%25`-{JZrkVI=Ll8V2U
z32$W@Te~Z!l20YIOrw>$T4eYQN?$alw>@n$cylkKVK5EC(%%-9==HGCS+m}w>y4Sr
zFA)%uUogUP8*8~Dm&$)RFq<}!D!7=i<ary&qt?$M=hsF3n@=xcyo&R)iVpWZc4zY9
z<_w9bRTWz2W5986w%cKs>H4j3X0c_`*A2Wm$AYLWalh;Nmn+*ruuNR=Xz*#SW^(|V
z?DLn<53<%U((88;p$)I!sgnV&z$JCEQ5SFEI5~ywqa`Mm_s356-&K02T3X2Dx02mD
zO0bWa_xq+9c|ji2xSiN1c4g1b#$C<q$V@$iC15?Xzr;K@;CNTo*xg{~YccP+SlL}-
z4eiIkh;d+53T$z@GXO@%S<2{PDHF2!qIUofCimkjpdcfMG~IZiShK?e>Jt17?CQ2`
zI<mk@zpi%U?JxPQZ89NBvWMaS@PWQP{QtNAEhPXvnwv+OK7Tn`yb@Tiy;5^o`fjl&
zBdU1d3~o7hcj2f09Yz-r(((WZDRb#D$MnY9_E+WQW8P~^jAI`4k!A(rp5ek<d7IRo
zZN8Y;dj1Z1DsztFxuW`l?SnEyPq(f|2!_6WDTYl0N~5)cQJneN;-RJ}-2QKuuQ>ig
z$^fS6IPg0+oyU|{MfkHFl(oNq1NtlP1E5F21OdRFc%W_XMMV15rtsbCJSK}ho)LWp
z7s_`Ne+fguDj>7D_-323B(va9dJR_^a0&3Y?#aW<o{!Sj=HQt+y!}Co8s(~>0ZTw5
zZ{U}%=n?Kk#{%Sx(~g8n+m<6&a~Vo>w<l$@;@qEZ9?}hk=1~0pdaRkZnu?Er+<e=E
z;GxUoG-uudHRk}%E|i2qcX)sYSe9|-S0~qW2w}|Wy74(`_tvdH=oNOhI^Ekt!$2RU
z7!uxgHc58D7P1z~52vD08e)G~1H2oX6Q<zA)50PEo1KI_Zis*x)x}uM9Nu{IGyh(L
z9#|olg%S&DwjNHe#COkE?`m0xg0gtKU6Z%YuC;>*$<a12eRzgl#t=5WoWHIdIGdS>
zPCl_+*KZVKLzHo<V`t#Ic&fIWqoUd0rSEnNH-kE&=J!cF(fUcO1gHcw;aJc5)R`Zb
zFR@2ODtX!p9t=>J4eVDB<Fj`gOu)$YfeA>0p`N+^%qG6;M&Yg{BAV3`G#vLYoJ2fF
zaIF36mUo9OAcAG`*UO=6*#g+)ad6^QQC`d=ugSF~Xc7S2H*iB0g4g_fM4r5V5jb+q
z$wrTZF5FSGiO~QbSPM-aOS&^=SF!gW^IYjMvvuRII4Uu(6>}Yhc1CWZst+f?K;de~
zoH^B@Z5{Ja5cD;cM@rRDcu1nXaf2uwOqDbdYraav=qdDSvmENAY__h#{Hubsw)H_=
z)jw}C3<Yy|d(C-6<8pLXBH3ri5)#qC6Ii>0R@d-bxZa1sIh~1+aP0dQ_BfJtUhL*w
zpoQ3{rEqtRftmFo+zl$(c!^&ki&eQKm#Rhl+!v%yZCV9d=Lzm2VE+*kN)qr@cU@rz
zzwz?{xE0}Fc~om3!t|1ykPPB>yB&Uj7?tjp0i66-z%SSDE`2f&ajk3dLomFxwnLow
zXZjH%^rD#yQ+`RQ6F@Ch3JG`9w>Xq~l28oi*TyZ23#eQo+H<LGorkZ74YzO3lfST*
zzn<!75TJ`m(#WZFEx{PRMLHU;3gJWR*)@3odnYo7dU?X=j;dfq1C^`XhAcrr7E)Yt
z)nw6RZk2NQaIqo9NTxc|JJ*kHK+(<SLv3^BO&0WuDjJg&A9)S09}h1ab#lyH?i096
z39K9ug+Nc!VK>H9i`9e3;GTn;#zN^aoS4>Ry~5hb(;M~?#9_fj<>qjYIh{UJJ$pn1
zj)6N4;af_&wfYaSK6&+GQ%-cL(K%l2Vp<Ob>QbYJzIj<x4~c4iUY1BB|z;UQ=!{
zNFy;}K%kq|)~I@Qn2#TS!t{KZER;iLSwNEPuPfubQA{X_{o`gjSoL_bGi7$N%un2-
zx)x2%im~X4s`0P5vkA>*PM2{CZOZGjeY)5D*`H&k?SLs26C3^bW=NX6WbNlZ?r+n|
z*gkLxqTthu`<y3<c&maN`ZA8+5;$$sEBn`!YMXX}Aqu-#Hn+?xeWz%cDN{|4oayAC
zYa4ZV%n6pd<cnI&6usfR@h?W+?~Ishl?2%@{3C0zd6ZGki5G{}z2Cjew~?CoS`R{Q
zOcDv!!(FS|kX&X@!z{u$+Ni#<#a7543gVYd<cHIp=3VOo-_`0;N!3d6XhuXn@<Hq!
zcTaBa=_E$6GcBDbA_R<85;!W%D6XQDBLlS2+K*)OgO#T3QsxaZQRSM|L`6R1h_41W
zgI)M^@_iZV53(cgW%rLZT0{0mbAfWxGN}p2J^L>2Nvar99h@9>Vt#E9o=sON<Ho%%
zTCZ+gL1#KNZndD=tJqrU32-f5%VvI$&xa?^Ot&*@EsK_V{d~$*T-ZFvCTS$y%ks2r
z)(7SKEuC4`orRZ|M>o!cv&z})-%Ow2!8Z-f<vSLQx+R}>rBSl@-ND*aeVgFW&XOho
zy}pUMU&WB+;T|U%ZYBr1`oB=XcN&KOR&C%j!s`5cQ-;@w+b9*7SF&OYo`L+&&B8XU
zV+OERupAe#f=BOW^{r=16AwEdFErt_fZ$Aa?|rE_{Q0~*SC7TF-(YXAwf2)rr|Z%E
z4&j=`smyjy>tTWO@I1qv0~nsasov%FPPcc~b5!?JqaoX<7~{qw9)K&MPR=}J`#Vw2
zMo286sbL4<>pU|Xj|={Tf{UTlqpThbql6utw*2C`+DQ6*=s_pP6f6hbYu<l3#DCQ6
zA*0v|>Dqhsov!_CW?`*_?^Ge$J|cxM!MRu&I;KnQj?MqFNdy)d+tg?=Hqj%heI0vn
zcA(99iTXQlZDLCfK1>;*&3wbK7WHpTMs*tmY&a#19C+Fl!)<}jZRqt!)!2P=>n?vH
zU%8fXb$ETGEBd`zOzwXzs$L|Ys=7gl9V*%GlfGd5!G7EoaZI37VfDg5UShn?OK`c(
zBBXJF^cL9ahfMjq&W@_wL$r&0FE*aVXv#dz*DjKOpqEe}gH*OH2yJJ0UuqvUuCQmt
z1`zipxkiqA>rk;;Z&Z+$L4^q(7bxS!&ZFs+fMgd(DWj#(gyq2Vj#~A7OpW?5%IJh4
z%Uz*nsbSV6U;|FMLKNn7835iko0)Lr77JNSZe_wFC&L`lTL6dPlozFW4jrA)zvk&u
zYuPT*oH?Yzv>h@7mU?*po@GjjP_uX1C#LLf_!8hjQSi9j+bEB^_+D+oTaI;xcVh_`
zIfgQ3nK*<yB%Apyptqga#}k4*O_kai`jRi&DRuS{Qa^|(b^+z#*crxesr+aP_72q2
zvt+Jvjcd75ZymGK?bu!~269*u#DW3v+m>=N*OX92E6&*`t0bEXr&I)H*9zBKBe!df
z=}ks6K&+q5SXMxet!}!1OMOEE!l!(EOO_YIlLMrKT_MRP!+H5%iP&@{inRt`x6QMo
zYWalhDv>d_w>PZek#c;7&Eqaxi9p4?&Z=M~7gcnHgqh`gaAv`}Y#*1Hq+>^yZr18W
zW6r~jzy~SgYCInThGYvxw^(Own9tX(V1ly&vw8@sSrO(A*jU=AvvG3M$|e6PLG`}E
z)VGc{b1z%KGy69FJc2!n(*X0iVH<B`31wg!-2K<Vr4e&N(-xOF;Q)8s`U7T~nPldf
z@WAo8W$yIg!QclR&lGwM>Z=(LEU$C{bwX9vH>iGYujKypEoMvio?BZZ82M2XvZjl5
z%<td>nsnFu{7uDS^{SHC;G;daktGU3wvV;krIl(B{eyu$y-wW5XZM5&U<-d_WrE_>
z&;20f(iuY#SH0_FA3n&vjOOt;KWivAGEePzy4As#OEq0ri2VFcy<QOqLK<xFhYOw%
z0U{{Cs8rJr%%NTgDSbhEW*8_}7KqD4rBJVq@t}8{X_9Qxije*{wL61c&y8FBL>6{k
z7VgFu^OO^NmZ{#872i2OB<Au7Y<fgq9np1tLvQKC_zTThAw@T@PHOrIu#(1h8hlHJ
z8_J7c-o-r$GSS|CBpSh2@>Iop_dHW<(o%Rry0tI}DiXyFlC0J=Kr%3&-EZ5$;f3L2
zbqi1{oIZH_8G{z|u$Lj_KiOK*UShMoG)U@RE?96qdz8ao-@q^NR`c;*(6bB9mGvoW
zN^~9L=gz_0$0HN)F3qkhpgNw36zK&<uktk;n|=~|!017nvNg2Pgw-?RdZRJ;u2q+%
zcY5y?Ip$Y$8;|MPGE#_ZJ!*wohqHNB4PHx>)Uf2_?(G3sLP{SruzgYr@|bK~^4EKK
zWwr(TP#!$tin8zc+VqH7Bz@1wv|K#vvqR~bT{n-bcuBrO=wJJJtQtbff{E7R-hB?Y
zmg4bK@nMqoluXGVLhm;C!K8+=CNc!vUVM}I$xnw;xrkit+wcD{4DqBr-Fkt$?~he*
zxcc7P^iI&2;2eyo%mEljJx%p?{xx~DheXYOU^^KR-DP#Dwm+$^+q>%L35;SQ+!>lf
zn^qz3)d!uEXIl>86m0=*v1FqA-^h%Krr-;5nrn)Mv=TNpe`k(s2ENTPca!kR3RIMq
z#)~^!Dsh~+7JCp0sPz_5%lCP^@pGWYt=l=0fOU|X&*GP#L8`aCzn0!klngZKSBlCR
z?%qRiv@zGcdLZukb{EL>M=$@mO01Pn(Q5!H^5)Ct`FPTfC`<xq5Zc`lZ<qM#m}Kon
z;C7Hs{%8rHUU*RTEhZXdj4C`n@TAL2zB+_!K0!)BW`!*UP_zVAnT?L6Wol}4LfRh#
zg1<x#tvW||bBBi0^N_^wiQ(7+RI?Ehw*c*8>Mm^(JDS)D&h*<t`Z~3HGDT1E>kwqm
z5%3P-=%~Q79O$?|V9fn#@8-BPzR~udA*ETbQFzu5Vs5A3$loNFY>itcu2O7>PW<a<
z=vCZA5M4Iop6o`_s>7M9!%*1(&fV?=O3?MMt@toO(|_nmVdvtlKNY00mIb9E@0E5g
zHeIb7ff2`~PYH@0hPo#VmR!#UxI!ZQCGwo<73NZCDXsT46Yg0(4}*IG7j|H@zUrm2
z@9UG)8+RYc8G6C$X|fmhn9jM4whIk^Ca|L=cucT*K1rU)Ods#Wgv%{x)H{y^4&~NL
z7zN`x3ufc@N26j5duwP>!9(nXbZ~eqg~SrdJ<C^~_$s&Y4b8Iw$r)^I7@XFFI*bdj
z8DTt_62S$Q@v+PJV+?%VP<NmPHS*Zfwrk`mum|6XQ8FVZUoAk#Nr#VMv1H4~8rvtg
zn?Zy--3bLj*IivmX7%%Ya4vrrg{I$=c9MT6r9t;`H{y2=+o6sjuN&a$a)cl#v}rMU
zywv3l=`}J`cvo?^d(V>$K3y=6$82-sN_b>3&qKNcnqA@f?;(%q@xRkHHt~fk7I_{a
zlt)0<A{aD}xC3yQd`?d3H&Z+FP<d><p5W)x+LSeA@LH~%53p^8?%}pDRWd!+<2A8`
zmz*6l`F1(@=X^-wS)mZ0b~|S-?tSBNmL1P0_gRW-7P>c>j(Sx#>g>q8&Mm;Vc(NYe
zUip29ILpIJ_}Ks`eQc~RXZ*b9J-ua~$HeU>ulsF;?k5*RJQ33xgxPwYs*JC*K!3mp
zXOkHip5_E&eWY@QY7aQ+6PHuEs=W2fop!7`S&TVBL${W(a%FvpO#zu^=KR2M&c4Q&
zQ94d8wP*q;n5%-c&1T=91QqzYM{hpj3UF^QTi0MBwZ;w#c#nqU!}JapNbdGW;*AX}
ziqSl7v-)MRCifGb;7rv&P5A3_gaS|-kVRg&3n^|!E5nlet<jYWq70PZ$#5yluW#`r
zC`uym+5k~wW1rB9k|8_%K{u$-7n)f#XCw-Skq3enk5MU`d0ioCvj_<@sYb>^9l^~}
z7bx6<t{-wv0bqEPv{^k+P>k*rC@Ix_B<%_4;<)RA#Z%{z^!VNyps7AojR0Jdv4#2e
zU`_624c~G|0^B#J7*09fe_1?vqS%RdR=$3#45{N-aH;pon31#|#gyUxEX9t1PGRZv
z^Pp6c{~)UU_x2P@Rp8YOV5@(0-ETkOv2aiC^wH^m!IPP99}enwj^wo`zfjX-?02dR
z&0$hXB2a08SBJ5hY@z-d$9AN6X!K)0U*08a6lb|)CN8%A`%ZmQ+WSkD24`daHDm5g
zEP#@M)@W;vcN%^NE1dX-FHMS~U~}hUId`tTz{+Swc&hPepke@j;=>*3iAz6yUX6Kl
z82g8X1(?I<`vKX%(DE2Ku|2=0tGBExJY%vGcPK0jwqDYWvH*jzj5)GoHSKjvZ&>%?
zO^bZ(+mkEFiUGRy!eqRif)<m$uykZlv+H87u5w}yz)~Q*Hs_cQ9Nk&;D;F+1L*czO
z?gZ;t=vmC!QlKQR9tE-W92<`}OJ$FgA*)!zoOwlUN=8IOwy#1&_DtyA?1fD&UwGt>
z4d)vS>W^949)xzR{bR%&{;va!iXzOa@dK~dc_-ccq-UH?lZi+t8=C9wGPO1NYlwKX
z=jWKV7QgV=fpT^XeBMyE+Z;6lR4O$ne^IPls|81ai2RS(y8yoOF4r`ff-rxn=cqqJ
zS!atQKBTdw0h-o-D4AqPc*%(3%E}<9^qTsvF;$k`*jc(;rNq7b_7A@A0|2-tTIXu6
zvVC8*<_^&K4mf8O=_$otA7;$K`1QNT0*BM9$fPJA)re}^V#L}GI98Vg<e8;bKiH)9
z)!dKUV(a^PDnZ3dv0G_)Ma=q35JhPZaK!575r9E>b1GA^kqIzHTwvF_e=;Fr_Sd<5
z@+jc@A3|~bfJYha9JE`i6WEv^1!I-S#-3syOaKWH%J;6>HdN{Z5*<w6c}k>r2Nbfh
zf0Y=s>*j2U+AZXX{i}0$cp96lxer*czx_k^0uZo$-}%Ek+unazxPU0N-^q-?#cyyM
z@D99ohTfb%QNBHP<HGl2kz)TlF86;8{Ql|x)ieJehhDLfA;v9k0TcG=0omjWSD5t;
zNxZoiIG6ZJLJvL_QZM~>&Kon>YJ$MRqg1~=PzMAkmTlEM)mmkV?t4i4cJb!N9gvd4
z5O&Q@`s3IOB9)0x$s+INVG-o}O}oCS*ti{?KS9`GSAEBmm+W?+;cu`1Q%=|SyS|_P
zd<BLceS83DVPNO?9jmjP5rP^ewJ9=@9WMy%3Z*XpazPA21^J(<sIt-lKD^y%>?tRz
zO8<YGsw)S~jeu5-XL7e^v)@0QuI%9h;9j#h9Z%PrTR+9YnNQl@P?yyOgUODzb8!U+
zb^Czc^fcmt?kABSA=5lTIVn1<RqV`+yBpa9y1*S3Hf|+?dr5N2cT9}0f)+OH;T_Du
z2I^k`pGinG$YnQ$c0AOJ#&$eTnMKr=u_G&oU*{nJfOH>F5&|r@p&pQF&<ADfmfvTT
zKyM2mq?aNtNV-SHrh5<n7=+|SLUe)iWt>LQr#ogb>5$mK?Xdge)^l;OMmtB@e(Lf5
z{2i<8cycEsI}@NtMJ{J+KVHFE!?X(Td>rS)+(!xUer^t_k6L6+QTFjr)-90$*uo6F
zb==uyLrPOf6j_S#WpkI9>km85d}T`SmLLVM4;~kqXw1yuGwf!TJ{gu-87YiZzZU8O
z;Ms$F&m8YH9GN-S`b(VA)%Tqr10mr8N*vN$cjgR;zPQ<mw>C7`{ugMu|KLDl9cDIV
z!-$a-Mr^V?a7laFp)T&o<Kb%F8a*jIQ&2qcT|g^RRIq8}5v5;J3w1wz0`+eY@ZDTB
z%V~yOYR@a6pfq^A)n5;D|0NabmNP|S^^(tthXx3N&cU5;xc4|NXfqsGT<at^NGoIp
z4x)E^D&G0}6fb?FkSG6MKbPdBr=SU_Q4W$$QkUc<UBA>J1Xtjr(Xa~aSe-qq9b`F_
zmyh+RHP3WR>9(PJ_Q_na%MUEA5A2HB5xiW6r|KpSS&PoN?Gdq27FcZefeJ{zv}0Og
z<Hkm$TL-~5=IG<@+lzVNl>@7*F}HFw*-cYCv*-(Kaj`3Rb{0_Rl{cF}zy!*wSJ&uO
z(df|dCrh`|B{MA#ra~XdXe91-G}UA=!6e%Y>&YSp9zA#cWlncD3K?o5hUpv;9vRH#
z2}9@iNj7ekV$-LT9`HF1XM>2;3}l?ramq)$0ye#Bb<_j!uruF|QfmIWuBl+GxGciA
zsCkK|9<LnACXbQC&razFIgvozKVU|fh1g%%Ht*G}H3QT+*3Su?hhAC{q8)Z3a8wLv
z`hBN+DQeY~u_2(@6+?u%5tzfdyu<hUr2yc2Es)ijQzCcAb=9C&Tm}0$D89GWqRvv*
zX~oGc^U#eJzo{;C3R|aoiL1_Al{I?80vKPNhM;+JUYAKB6MBm6ol17;I+TP`=9d|V
zLp=$yH#h+5Hudl$`M=CB$=E!8WIs)h<vnR|txe6evlcZhL<%evbTu{dc^%W%_JZ9-
zXp0M(8|WJQ($nK3%-4FX!G%pO+9Pq!kY2{gEz~Hg5V%j97ar;tD%6wZ+AF${0C-s^
zQu57WB~A`FO+NxK$gp`s4%+RQ;9TLpbj=fsG3rdAKWIokO0)O@dt#V_iq{m7^Vn1-
z#iaO|srEr9nZEF?h=QceIB~HmA>u~o(*mo(!v0M8i_heQbTO=x-FqL8?0b-S`!!>S
zof<}Y<dsw2JTyV$-Z4~B&l9#baC+<AA=-8QkPRzV0?|owOpgBWqOrYXfTDtG;zXIk
zXzuDu(d@(6zz|!b+y_9{_L>{eWB0HrN;z4<42ATD&kNZJWo^l_wNs`o*Mxkzc7~<5
zlfZ=4kV3-8DxoYS$Fsskz7m+W<e1|MhDl*v4X}z(cW}ejlvxTkwnd!m>5(YW+<Jy=
z+=DK-*VjLrM~iATV#)fw2^fo$KG7Lp*0fT|_c|9Jsav6_YDo3#HK!Z|IE?n2t-NVu
zj$_`9oY98B7zXCX<mQ>Vms<dID?~|{ZD)B9&bl#=2a^3WH?i4v18?CK>CQB19!7sp
zY+^3TQ|)*qd|(1IRm9p#wwlPNv{_tzH=WhZ57u&P<CJjqy@{M6e-Ld!k5%N9DP+u!
zdS1<pI6A*s4AowEzRwpbLWgM1VhzP!t;=T&_bpO=&qy8}N)`nHyS5N}+0j!bT6Pl`
zjsLtC5M9>){<G;RvSquxgH7;IiONaxy>|HqU%T_e6Qk~-Q{L4p&i5%kl}Y~c<EKxC
z8Oe^2y-3WHQ#uAE`D|p~y(>A<$0$aWkTKxsf(CEX=1`(}vFPXw6Z}jg3sca)+>J&#
zrde}lRe<ic^%B>B%R(7p&&+eZ1Uiedw2Az#in6G8we@yg<g$o&yb1d~hV``-{Dcu`
zzc+CYuQw!@FDre%GU7KB4jTF{OkEXdZS`?;c6eTzg+C3ogvMI<`dnpx!6RjFh>nw#
z4%Lf*qrd1QEJQ-lz&4UrBsb(vpj_sZ@%TEtI*Def`bTlub0;p62fS$Php~q)B9hEP
z#mk#5WY$hxG)0(JqRSt!b&BGs8HK6yv3W?Bw$N(DgB!61$7ja-t|{@YV=5JV=^SW?
zkeh$OKa=~I@ywt77o312yVJJ~IQ#&yo6Cmk&vX9m<|8FGT}_XGTu|a9`D?o-*y?^S
z27ne-?v>7m>aEFa9qFw@CYwniH>{78Q8rcmvkxIJ&U(K1he6j{oq;A7%=Q{gbNN6f
z_A{m3wc-@AP*M{qHTc;4^@f4KwAhf4p%Qe_z!au&LAQ)4c_Oa8f(*37pIWsxLV25~
z4W^@9XVAmCjzg+t#n8^8<jcO3k<#aT1Kt^(b|n}u3r2hH?9XXH+#s+QHrf^eXWf{J
z*|uT=G!9&Dy;8wC#(fk6z_L6{v1y_0p&=)MZl4(|kt97UigcYSOju?&ti50wXB}5H
zbxbJO*KQ%6NJc2E8@I0;Iso7<GA&TwdyqG%PKo1$%`;-Q{Rf_-2a@0{z;Le{mdmC?
z->r)U&g}HGUp;O)2IlO|=AYZ4=k*{I>Gz}DI>dqV>1(05CIA36+2QX#*yzDyVL;>T
z@0dqvZN1y)VJl34wyLWHx2~veP=8F<_kxr7cD-1`f%MxZYGD3h&c2QR8&?lFEb{x)
zm!DQaPghIgs4ZF17h0LpFwGf74L*?!0H)!l-vQHuF9}0RG_(VWT57bEtIZ7K5}WfP
zCnKVVtoXM5d*1(-vd2lpA-2QLO;w))P!gxe=PT1m7K*D5AjjlxYTcJAe2Z@;Pp&o@
zQ)9F^*~8vw1L|;^jYLx&Wy7)@%^tKy5NeU-QS!~_EHh>eS56F9HCZ~4Dk<iniH9n`
z9V|yrF8*iMLV4&OsNI5+kn_b=tGB6@<`6K!dh;(^x-akpu1M{al&>ED4|3{~Zea2&
zJvz4y<Jc~LK2rm`XbwQhn9%K)myB0{a~CG|AvmMH|1toBKkqOAJ$_rgg%5)DCltW3
z)CV6GEEQ7k{V}9YAp*rl13?mhex(-Z6J^^gF5JHvno=^o4M3`6Phx`rCX?|4<r{Ol
zUoE+R_f~^RApi>3)+6)m3m<gk(WQx+S&K3)^J8Fz{%R>r6y>BKwQ~eos<R|ZARR+(
zBQo#ta~go{jOZ62o9;0EYT=bkyNtP!vP;f{gVpYYblExva(EiBll|tQ04Kq?0SA5J
zbU{m5dQ*CwMP@Me3M_8=ke}1EZJyCGb_yJH)x(5})}`=t*}B;`PTZ1&>=c%o=Qkc;
z0p6g^llQW`QjjOxK`KKs&Gc8K(Pf|bxgv;-P^g#j<a{sUU4KF<0B0h^G_ZIHg~uWP
zhBN&WX!+Jgr!0=l{Dlun)*>^*@nkE$(5<eChCT#e(NM_FCAoBh2mV$Rt#d(iH-RZA
zo`D0gc%!-j%2`NfK|YIU7)t1l(5%c;%+<vPK0~f2m;Re8c#j7s7+_}}G^r`Lh~HDK
z>bd01_w$I14Go{c9?xp?)7v?&JF;f(P#rUHy!T<CsW**1h=N}dl9zey2Gft?z5~Kr
zy658x?Pu3fhir~$%kZX3T|4=M<j>@g|Ha#Thc%sUYr~9n+zL!|1QCQ06m&#-hmdhV
z5NToqqzi~hlTHXlWrR2=5kZg=qM(2jX+r3rpp*zmkq`ojln@|5XdysI@;-^q?6dbd
z-}jyGz25iaFE2xW<tgh~>t6S|*Jsv^q8tUYiY(EUft@jHFjt4RzZ~7fU4n68A%>hM
zQTBjE7K3wim^geawfNZ6z8GmE^@IU{4)!ITq_E`eH1;>1%w9N?O5}DTudF@CmHVve
z_<fX-PH#DiI^6Al@t>?h@pm-lX_)jgd=aa9U*RxSnyeV-4XU7J^_L}1;mu?{S2hoa
zkXztBeg1xf_g^uC#1;ZBSms<2+}*gZuz*z|DLx=xdQ!aE^!=<V+gLWWt!CA^OC|C*
zzkpY30nl0nTtQPPFz>6nm&NYOQ5)2pe$3#wzU&hZetb59dF$Hbx0g<ueSdv$<@#W)
z^8wTkr91e0McRG8EWIry*W|sMik$d*`47j$L0x4!?c7zo!LmN|`h}Qt51iu9xx@0F
zLW`UFGVE*S&ZM@`pT^dEtEIo(J^I!FHh9A!B#Y_&EA_fY1XOW6ccJ0~;SaDM*1&#X
zBF-8L$=N%pq8?Q1UGLRcs@6QvM?R3qAdVsDK0KSvGD`bw!@T`<#m0RUW>WKXM{iL}
zPhi{?sTOy~)QW4t-`}k`>Ja0|uSsPTEy}r_aLZAXwO|JoICmao|E0?$$R;H%@HodT
z2J2?ooL3;4(JMQ6e<e$_e{d&A>YD3?H1FWWdw0&Bg2u_2zX$7F3Y7V~Xxwu1iw{<x
zhRS4nw^chT#l%)-7<nq0(`J9LVJ>I2Ix(awmgN2>GA1(*SIsF+kW|I(D*PkUUB1wa
z2*de~MoBjRWHQdi((9X8^03KgA95v`wFxJfbtU3%v@9dSXMM|Uue1P*@mX<8!7Qh?
z9)kgu9Psrgq)9Vo{HwfUkXb3)qW4Akm0ydpB^^@T@#H!iSTgfgxV2%{sTx7DXxSc^
zQ74HMv;W<zP_F`P2`54#Na9k|pv1I!h~H7Ag|;0z`*_&zTY7MOX*W~y284Q+P-4PA
zYJP3B-fqe-Xp+AMmAEc+H0{wN!payMVotcdUI!rr%84JVNOX1Rm>o$85C&}jcLN={
zDsvOZcon2pJlwM4#6}daKvdJYF-!2Z_QUa)dma=@cSf3%MG{$7AbS2zhkTUwuXIR>
zU)6B8`X-#h?bQ&AfY}ylD0E*3<vur2J)zTG&aAo0l*dK(c-8i!0u%=%HdU#oRm934
zv_ZcF8+z6kgRyfQ(#|$WSHHW=gb?!Wge%hAw|u}kXme%Mi6lp<Bo`fsf@L3b-Zd($
z`i=i64ft_q2{v3T1dbV>V%WTSdQr1pHd`S>;vUw%zkl_Q%mFaUD#hjBMwxgKrbekU
zV;w1YLnQwzC~X|o{nV#(!k&Aff)nwH<;M*$Y~aGl)0>Us6p?K?p919D2*CG`FMSX^
zCfyzH$sg)^GCd#?!A5N3ff~*G9RPNFiIrsTv;f=ItWY2hjrUeNG;qR(IoIy=E?PfZ
z?Xkv0A*9PFrth|+G(z@<j7fNCYrcC%`B7DTs_e|pOPc3WT%<C(ALffks+KD&B<-j{
zYqwG=q(7=Z>*kfS`ZbTFSQ@*v1cE?d>J{M3y;Ro1t-VJ+IfP@MU_BK?+r|QslVhX5
za22vg3Y8F;seqc(V%liIbA;xt5}7}(=xOvMK?}k!dL{<9XBf$}jTNMxNZRF3u1oij
zu=r>AY^MK{?;J!TXNu&8#om4clj&)w7riC)2;FvEyqjj!4F~jg%itr4>3i;84tEi4
zneFtSJ!Y;UP1Sc3^KhtYpU~?Bt3Cn~EpF9WXDgS?m@3Vo&zOUHoQEpXMnVVEACwO3
zVNS+x>D@ZKYI{*SX?sPAzs5LK+XfWco;pbzoC1}vIoxWq3i}@1+u86ShZNgrKLfVX
zCwcM<kt<lm-&5QZen}=`V85NjBcdeTk33ybePB&Dh?KbGm=p}PS`dfM`$UK^BHs@2
zrmWoauPFokn|9_i25YpMzNdLV0Keo@c?qm5p1++Z{KQ9uPnVDB(O;xKk|!+onf)qw
z>F}`(pmE~YW>s!{MhKh!sQguEd%0Uq7r)fM^<#{5`It72as)y+q3W6A12U3)sF-jh
zz#;8&N1Kbv-P8`YL>9ee<M(!-j&_&cK!U2Yqlgubxs{8ImnmO*-M*_#F9A+(F*$6y
z!mc<2a$2YlGScVaZY3CaR|Tp<5^r_2tfx{h4<nw~;JzHbNoM)?00z=6$<&dAkvtW7
znzZ+@zvAr6vkAK%GfazD-&<%aA~0S5WQ6{jbPOCTrtsA8;(S`=4ck0Di&wi5Bs*1H
zlLm<7e%)Ua|D;Xvz#L4x*K9i3^%>r@rev<5D;%nbE8v~8A?$`3M$0c1QE>%`Fokb`
zcvnwtxv*`PRFCN4PXAA>t`%AB)QuRNo`<ZtlHw-o=Cu!%HQk#x(RApizc)(}nf@`-
zQXu6-lea+WF&-fKHlz+Zjq5TmsauF>>LXs~<v_OFr}kvqM^<eE(;}pr`3S*G2n%X`
zzH1SXh9q)T5kqb<Dy2utskUMl35+_s0dl-`!WE9dFQ}PcqxE(O*ctO>s+=Lq`u9Qs
zlUpl}ajxSJUQw7Go+)*A{&B;UiY+?lhsoOob0GD4ogyrdHJp(-?MM(PrG8Usi%(|m
z+L}jBK`m4Xi3WFXy!Sc~dv2#}D&9|MI~wo15{2deauS+D7gg@^xdAX-fEWXqQsgLt
z((qc7pra`*J_|dtREDg#4qO{SL>kKK0`}t2<BH~~lMRI{j%9B~M{_uinPv-AWixBT
zbs&sp2I%<U5;n<B$gtb3eufbz$ItNihlZ^l(oy>3cOi77KvkQu%rK}z*LM|t-6T5E
z2tobXKFL9>>=ba;<73dyi7I7F?~w>2w|j+vmF;-Y|I!YD3}ZI5X|6b{jjjNpcZdqy
z`K~?3yW77**F!qIX&L#B6DKn-^J^OR^2f!F0ytf0vM*LKzN2k5JJP>?t$e|Zht5{<
zD0~8?SBB@;?z1~wcQx<^qj?GpicPleydQc5<6T?H{ZYV-KEOkYF&o_mfOE)`Br846
z1H-1*WQtt`shS|EaA&7>1o5-fpFDWF^u@mcx8VQ&dl)<a5C0GSX$tKv$F<-5yISEL
zs40J*z#aadA^!i9xBqsn?-Mm7&ypcY^*-R$G)vRMovsxdpPlsQ-fC@FzJHRJG#*^a
z!d1ny-r@$5<cI5nn-B2tyI?o_1O9kquvOmt>5LH04`$||@}tIr29wO|pV{)^6NK-~
zKmE@HUS~OgPl9>4`ELYXdp{IBW#^Dcd;zW8+QGIBC`>i~?q0H4e-$ROG!D{Z)TWmp
zhSdY)ZV!eOOj6kw{;Un~Q|<Wt)Zd;L1@=Mq1?YB0sqxFDx9dU~Y>@1ncU;geQ5fCM
zpLLt$B#=fLfc+VErWGpvr>?P^D-MvppD3z<v{7@F&1HvBG7@0p)(#Cq&SlHjSYt?I
zBR!eyH2KbBoijjZ4Z%WH`r}&kKBA+EH^lX2dg4#e^p21FH*_R6R&ur4W26I`^u?DP
zHnwelZORnj?;nwq-wC>PQgS@Tez^;E9w?KDP1%Y#+kLXbL;vFG;#N@*DS%&dl*YXP
znPK68KNczR=Y}7ksrut8nDWb<aX>utdE0?WOuG<$tMxP_Zu$OV1tf_>o*YJwaLS#;
zEFZ34G?bkLnrfn4jB1Dai+{fPU%>T+%YOmaf{i64J=`|$csQ+g?*3Cj2BD6|!>`+8
zeVw(J(<1{6WmBS*ou2j8<60ccXE2G{Bb)yvs9k5~P`mrvtp0=WIhi?_0qSGHbxpu^
z;VFW{0Do?%7jVY^*Yw&koT(yh_vGGEQNBg^oZN8WqaZGY`3Z;kL?Jt*D0=6QbW_jT
zA4w#pFU=q&6qLGZOBi0Tp=i#ZNX0cLR(PwO^sEaz>nAI}He+eo8o5wHT={}YYNWRz
zsjfsmDUp>$48GR-_Tv$pCuNgoGc|+`?=ORo;GqA&l`F>IJRBskyp9%EF1D?zc~&XW
zuZPREO?X#N*Y1>WQ%)5N?s9i4DX=19+as@8L(`xi%c+DsP%f1tA<A%5?1blMWj1Dc
zH-_pv8Kw1H^L-VOt8F<OS%7k=Q3yCRHP@B)AQwh&N=am4Vo#IbdNNj(D9|?aMA96y
z)a4GY9Nstyjf|V8K6h2^DX;YGDGQGVG($?}Mn5vyz<|nrh9lb_PzI&wK7A`CHXk_;
z%S{1Oj4dN%=(b;vUgjY88S}HUC!zS7B=rVVcRIbe3(+{aV$zsn9%w#5@73eph~%o>
zYWQ%>AjQ%yj>@vQn7Xa`I*$>M+A#glRJW@g510V}SboC;mgTJmu$me)^27BA^edON
zcjaetw{|xXzFzqoXJ%m7E;nqeeDAck8{Mh3`lTs}Hy^cuHnp{9UMBqiqu7$RF9M`_
zxdsXZv``aHMrye55&sf#y$qr~DTl-ItU?U>Sk%nr2gnoEr?&tN^LsM8ytC2pO$prm
z)meIf?m$vSy|K>|FFPnaZZq;rlHB3{9-9_dlo#jeOUn<87>UXQsYH0wS1T65o(ZPk
zK8jT9G8CL@(juQ4?iktargHmYe`Yjp7-JPeQ7ymvq>#~QS<5M(E0sF_&~y=H7xfHK
zT`D9}gBz3E-D$Ov>0a4UC>MNWcg7UEKc69zvwx7WOo0XvL6wsgF+;gPVdotnqe^m!
z=~T~wiQhT?=jPS-;TyADLt(GShw-I-mo@8AUX!=dg9)|%FF0o-7o0e=`b|k+SUTo1
zea}BWSdL%MWZpxnlo)E%CNn&?n|qpx4cH9WGkeY^yH})|YD>l(c0BLuXvEGs8O?Sg
zVQoVgERCiT3m&u9ebXAkh&mWN;S|^~k$`XzPM!dWGG$4T_U!EsfndlPRRd7Da=qEu
z)-(A2>=g7AkKH&SUp_|*eps8^m(6@l$_gG#TG9B(V-Wn`Al0%im3Ta1o4Gsfj*Lwx
z=!v(&p;U^pPC%zJs9n-Msh(hY2k@7-#Y#PwH^5aT`X7@9$NC3e36vh%KUn8=q2|V_
z$M#Y~EAn}pl8^aDFHJbEWVKn?%EYHyZ$g~^qie5voj-5J&3myot10O?k(Pv|>fk(=
zM|MeS;exCTD~ircnSJubDL%PYT|XsKBDi}W0%a>`N3K?R&3p}4kiSmw<gqJieoRVc
zmqs2AF+c5v(Rh(n#Pmt|7{-`%=5url<WlXXR+5R;?zH*C!EXew0S2u^C6BSv!};Wz
z4wj$4Y0Hz}80Gzr<P~|{h3Z@rlBVnBNWEV-)QiiZF>Aqq4)m$iOWQmkO09T8wfqE4
z?5!doexHa=S}ObXF9h6k|KP+L6!#S%-DUOz8!AN+Mo01h^Vo{|NskWesL}FfbuJwW
zwh-b|Yc~nC9@TL?8RVGyQ{!~$WykA1J)*yE6mD&ySU!+{FYA^ASp$OVX`m00k2pB>
zlXp!CoZGZT9Oe;h>;4OZt$wt^cn&x)XV`5)7KS9861{|JxA9&;(@0|0+#;f<!30Ig
zPDTim6t54h9eEgL2~@5qS|guPXU@y?PJr^21asFR4b-V5J?mX)DNhK_0>LA9pMCan
zF3v%eq6a-GEe?rypXcrVl?#?axmj~*d0F;BJd!Ex4L&xla^GMjtWdwLpT|~n2yV%z
z`92+NfYmLE*>1TRO5*rJ442FEmQ$qzU{a}y50~114M2Z53+Zy~TFJ-3=ryCsj8<V#
znaATP+iWCn=4>4C>Akwap*%CUiz~@d82tyIeXuXUtCba=B&T-HY#Fm6Exu#fng$r!
zt4b^K8E^hv|IJX=ro8MIIzv0(${9eem!IsxUC`Xbcl@bRG~yvs{?7ARxsUL}_FYx}
zvQSX-;V7ux`ZYsF|LO-X=_#v(mNW>V8bq*$+Spw6c@4{JvN3-xI_*d9$!xN*TaJH!
z`I_f}*$((?xwuP~-Hw(CD*qj_twAD{{kfm#(QNMi$&v;WN+M#~svSOH4eOPp-Ft|k
zYqC@gA86SeplOG`y<+>3mFlYDbEMHEc*cmY2<`lAOE?Iqm|fJ!1ll#gVdR&PZ@(nu
zkSchQ@Y~8X(kj|1!-F6#ehkAa3)i@yCoj!B8Rz3gI<Tq<N~&ffyLJ`K%|HG)Pt$pL
zYqdx6bo>a(GlBAQxzQJcYIl_Cd2ncO!p)HpPCG$Q6bwwd_C-VSS_3-xQ@}nE7dzC%
zFSH^#(KG+s99s(=d-xIx(gay@9rb7c$L}J=yg!BeX0QyWk<aNqbUDuNnnSHJFPgaN
zRLAaOcTBuxR{Aag`iXb>8H47mBZ&>imQ$HAv8ks5@|#eGpZc0X>8Ls3m&IHVFHI^%
z){sUp)<%HOd7h5UWcEcIP2N=i_?!w9Yj%tok$7`I;8vTswUFYZ%O8><iGojp{gV1_
zYL@6lr$cSimypP*#e|rmTN$E0iQgH<6>q`=5(kpvEAX_AD>g<o#YBy#@1%Q{;E!q>
z+iZhJOD$+47>D(TdL`Uvl0Pf|deW9oecSnY{3O}KD)%(kp~yAWCq*YBX}0eXP_QaB
zM3YLoEI(@x9}xj8f__S9O1+=MuLc`WjWQ(Gylq-Mqjq<`G|~u*P#J#!NzzaPGBa8L
zI5+qy#fVcYYN&A-NXzJjpvrh~YI$fUh=@_NJwV_t^3@BC=y{A_Z2#e4c{!^DAB{!5
zZ@_Nas+O7Spih>E-C)_AN`iwA!5!AF0+>{cNtM@f^ygV4^1xQ+F3^gIl_y!MbZvcb
zjxvu8I%pPk%FglV(o-SHW6c;z<r5IwalG_JLXW4rC9PytR`mLC_;q|Y_X_ai>$_`%
za`-N(()o?bXrYNz=#0^5%V|&?2awzCmo%<VDt?qRzYgLXwOPr1m7tEd#a%-y!l|`7
z(&7Hx2d(^f*{c>Nj5YQZt}=6M1Mp|v6dSfTbS$v@r#uBDLgeMtixO+pV!H~wttI31
zP$HhgK5uYtpQ|R}++|QL090}Kv_=b8TkwA6OwXr`$%1@vSZ3VF3b4D<piyjIk|(FW
zAR(39jC;9c3K%%NdH{_EUjXsqvGRo^j7#T6*&|G15Ap<2+4twfGc6d)oQF0~^!uYT
z+p5E*OWyt0#9CcTUNCvbYWAq}ilh(W@soxI5Z)zoUvNi}ZH8_w78+V+!@PAGGr1Y1
zm<^XIjesV}YX^}5!$aoJxt+^Ab)RBL4mfs}tGS-7@iLuJdc1nSiXVWSjQ4Nn;g|-e
zgnTB99(Rqrckv5kf(K)sgzis={uqD;!q+nk6$DBZu=7<U5vzr#HlVUyV*pyg@~*eV
z)+i*6q-?Uw<+(?tdst`w9;NHNRRBim3S%;xQpFo5&vc#Z4uFB|j{l(oxZUWMu2AXa
zsMj(3Yh%n133wTmOtGcC5C$Wy!OOHSEZMV!J}f&R7_$X%kyin%<xLB?L{qV5jZ)z-
z#=mMqwcI`_>7n#R;b=$S4CMzAnEF`U(xDLVT=jyi_OZc0%<Q~9b)&q%>-uB(Y=Q0U
z;z)^Slj7vEtp|0fr`PAtB3#p<#9(+NdBxs3%b@0y9SF+G8=D^H<5P{g>~#42CgR9Q
zw6{(%xBdDT?gPbgW!KM&voy(58>TBg#*@)uyFXaV%yM`10HLPP2kopo?lJg08GTUs
zC_wm(iPFm^tX~c{{<&o>_@Hlmd)AQ}cF7EEisLbHOXJ4mmiM{W%%eCO*=REIb6->Z
z37mT}>xG4G{W#z?C-<4Eisy3lr15Ejku}|Xy}<eaGLHSV+&+*SSf?7GBfeVkVgaZZ
zULW-7R;f|SR%>m5W+xk`xM$G;v>UA1RpFgZIrNE@HBpp3myyz7S;M(9ko5Xk`uQz$
zL&yG^QAke85ku(fEPFR-jTzXV<+MBSqMR&F0%ziFF0?JM?)<hQg_4nv^>q5NHa%LU
zy>B0_PGi0tS0J8#ew(KIbtOEyO&KA9-%|uHz7_qkV}9skPrKRd)Mj5P=_@#a98jo3
zZ#ibjLHEar?il>}lkpyB<+J6Dz8rv$Qc&BL<2;UQH+HYMU%|I3tqfe*U;-L0@b%In
z6rgJ>Weaz+g5fD9jiF$IHb+1@Chd?S4fo;`jqxZQyPuz?s_OVCaJ5h^J<t0q%h@j=
z$2|wfhg+R;sL|sU?~Y=*M<5%RC8nX;mxjF>E>2vX@l_-!!l#{NJ$?j^BGJRkhdW}+
zz0GdPsx8Ac{&0`^`tm`NEsAoY<`Lm`pueHUONx#ZBoXSjOo}DYoE<4zlRrgDb&LM)
z8)_s}`HYzLNJ}yPS*so~v*B1e<=J`dsR`gC=z`QL720NdmLOdIDe`<_ft-iAnqYR;
zWy|hs#zi<#Gc>2;e6=FwT>%s5U)0NA>K?nE?(@mz&!&^t(`j|j+zDs#f-%VxeIswS
zBy5BF!Va=jANXuN{Dk<2hl*0tpJ~;Zh8t(B18>si&gh+gjjxdJoIV`?m|^I+y~|4}
zse2;|-ZQzGX@}n^@BFs-LB*vft#6AZ4r&Y9<ol&*)gvT(4RT>S>qV-RfW4@dR7-dy
zTxvNyGH0sTkhD+o&T)9JSn5>iFH~mM=!^atiR#JC?Wt>+^84A|E7k#WVWYuRmiJDZ
z^8A<;l7-y9z(bf4YAe-i<E+-FiAsp}z%8Lr<}~t(TC#xm0X_xy43h;R#H%A3{+-$D
z^^)(hj*Lwr^8gE=>G07#+i&ZE-;~T9N$VJyQ%`Rh+16~tr^(*Yoou+k15+2H1rQGa
zWB1aUew2x!kd*E$sYjqRfbYG1PWpf0_1b9j0i5X?zt$Y-a%wLR6ivv<`eUBaiwyaL
zrL(=%pARcNr+x&-{mAig=*BjIHY>|*xR7Jx9YAOfPqh`f^R01#Getp<{ket*MxR3X
zPg7ZnJg5=@gyx$A(0|bjoqK9PK|d8soX1^XR&KcIBvCT6lmB<1B&IolhmWYeCC=|<
zCAIAf=jjv-Me^UGX!yAl1HGwAhEJ2&p5XtTQT!ormXT;I4i4ahaYHiy1Hk7UicTdU
z=|+a$8Xd;)<r@TsDtP{r7oa9i&lhK~lEBk}4<dz!?(#qX(=AwV!2#SQExFN}U@tH*
z&Wg1>S~2g5>tK7p1I<CekmA%!dl~A{{?5)arv;iNl^5F(#(q*;u7LX};2io5(FgzJ
zHHs;<{|;p8u-AD=)WTuq^9rmNNZS-1$K2V}K9cbTRl6|kr7MQF59|_NbGp&eu%#Kz
zwQAb>h|?A0jtO#2YUTypYbzuVS`809J>~t&9xof9MYPC<ME?v@Xb8ifLlf44vb?e0
zF{hhDQdTLEzqgjU)1NAW8wiw+E)3j4r=rjn)ReW;ShtLy?4DPYL>7&3HWne2s_n@$
z-LolP)yIFYdB5ukP(p`aXw{YOM&4vjyac(TzLO9?)XlY&?mrmEaGVf7p02|kpJ2*U
z>#UJmSWJ39`YP>D)h1S5(`WQMqq^yA139c+v2V@U4hR3G`<9n{?JUw3istMG74d@y
zA)N;`z(SR8D~6CEOGQZ9%Kb(eNYl?tcNZ$I(q6Fh&1>f-WOzMgz{<TIT1Oo1zJ(<+
zaWfW$4AMfpcRicA`LSR`r_*QmiGFnORYR<OOu&7?<1T`QsSl_)MAD_g)GkRppr!Ko
z-=&WV#+m{1k=R<cv*1NeNdp_h$p|3+&>7_Ml=sDjH$1H0<!!xFklc77q$ON66=6Z;
z0g6TKd@?JzhJbelXlAf@x1w8z=Ee5Z81gqT4!tI&6}DK@>4Q7jkG>+jJ4+_@S8JXe
zs?sp8pGG^v2nHHOd<BUT5XtJjY&Y0M^~<7Hwy$NAUb1|^(jGKtx8;Jq1OftfFZS&w
z%>?;+9f5*Y9ysr6m7t&$Vj@xZ+Z$eq-QIMXH!n%0vAz1DOTEDH@&vyPlwN@No3=_(
z_PP!<&8ExFWIk9TKWY~qiYTEP@}9c%{+s&cPIl<6O2@wF@Rs@cH9_9PpA~g@+f+$U
z2=HPv6;EJK=b43GO?Vh?T(h=~|NPe?d#2lrfkIl3a6=#{ngvp#Pa-%^k;{c|;#ck;
z4bWOfF(%?6(V|N)f!5|$2~aTR86WT0e(SM*)qaJe3OQ?HLJ_8O+*IyLq#Zh{MocGM
z4ouY*)Wcg^w})<nF(TDT1HGY^doWO^n=IU568CclUmZ@<mc2xp{7u_?U51qit&aCF
zn$%esj^rnG2DN8iCUtv)rJy9S9x^kB3LxT7X&M;QcY#gO*`6P}J?=E19!!k#l_#-r
zf#?$oQsGJaP6lw1OkNJosJ>QHdmvtB_)3H+=q$35HB11)oRBU_GH<lJt0jaYobvjQ
zRF?GLid-BcZ=iE@)Ju;BOi!<s^@`|9>8-!d`#xMym0PlfU<+#P@p|^jo!u6<Ci6uq
zya{CW&n22&bvlsPF>iJirgMGx=s6(aE-IZh5;;g)ViOxoG<xsnN$C{B2;ZwPhLkbY
zo3nVw(D_RAeoXaLy4Rb=&DjZ~&ce9u^tadO^NsD-*m0WnYr{@a6BP7*#po7yynRq_
z2I>?xe0tiVFlR&~Q)Se)<Pox9#m21;5vbM|u63L84?B?5lMM_u6xtg_Mbf)O-xDB&
ziOm!wtWz|$UAPTIp-Dw=96-7B$T_&;+*F)0Zw$n<M}0A3Ryw}~U1_;X?rWr^Sk0d=
zGD#t#8#khL-i&|^<hMa>F-|6?Smo-{J|QPM)sqvg+A2r4i-Y<T*`$_xQgh~#;v!r1
zyuA#7_f=WT+X1b{GAd<T9Y9JC+>|$$&N0xfq6)`|k7*uNehu?gyMut(aitxJHt#u_
z_Ci@={h&t_GgR2i`uhE-8M$u6lA_~n-@3qcn((y5Y09AfRo_QoD_4UFIf)hIt>2ob
zQ`@C$DTfoDdgU7%9r+R>?EMp%n~|rGjUS#F+`}AP8bGRf^CpMuVkld<l4_5Nw`q-Y
zg=U(pZKeqDRlSrvFp+DIiY=%YEP(N9b7DDha18;&x4&l0lbaSjHA9B^77faIm0FLr
z^Nmd)IRYaxdZ-r|wFwz;^L@L8?0{mPEVBkVcN*Ptk85Hmn;nRxGmX_KCDl=y<I0e<
zqHctZWqJ{(7uobMED8v2wxb@MV7_lWJwDDCsMh#AUp(l(G31N2^uz?5uVNe$irD)W
z6_W6j=)fe`@Ner2ZX1kK&)na88t)7?)bC(J)yRNuSF)%Mgo*I%nEiqudbwqJ1%f^b
z>Q$C$CF3rTCb7h({e@f&n6qj5W}A$@Brna`ylb=5x-sAib-LFZcHXjfwxDpqF?PFf
z37CQ#I?MNIC9^@BltEwUOtvY>pXtgodnL+2t82X3FF?8md$*3tBz!3YCcO%@V=$vS
zK(zOTcD`KNjY!(d7zA1>>|x}bxdY3r6&y=S<41t!13nounStHf0f<7zY^E=k=s}{q
zOmaMQ05uz=gRM-EbTF#hzxVdy*LO0-Iibl!PLc)OMd%2c5;D1#5J3DLyuG_zg>Ox)
zstoAy&~jI?a9+~pNxd>u!1VUmOq@l$Vl%<CaV8h<5E><PphpD3b35~Q%OT4dgnyK}
zS?#7qc6QUDbq0C;+P6CxyA|eP?%THpT7;oX)4dDR25k@M=V1GVyNKkMra2PaDp=$v
zlThD=8F0{lodS*5hJLkngi|{5NbCj7qn@U@=bsN^Lh&ThjNWEZMfYK~04edW+3`Z?
zdTxd#^u<WRRzFoLa`9{6Q=-U4Fl>{c_Y2_SJ;p9a1pN_k|MUyKuWIVQdXqL4QWt5Z
zdVTOY&{c|VCPz29sS6P2z*yBVZWacYa^J4sabw)j4O?t3Mqj0_2hiT?|58@d0M71O
z_3=M+G{7Mt@(pu4sZhgd4qmzqrxUz&&_+4Z0^ToL;)I>VFrH4GGhu2H>nS&-9t7wD
zs}GDb3mjp=?!0q!tl4F+Sk=$YCj0g@68G&9D%FqOK~`8oPd{9@Sbf0h_jC}&6bsY&
z33T0ae7Zx*jV#B(BMpUMJvo14F_d0bmCY)t1MYpGpd3$B(OaKan5E6DJqrZoG0v9H
zUZqx&zJPB^H39lkL@EX;?mJM|&K+;R{jc%2G+|KxYWZLtkDIV2#P_kUUMHh<6d%R;
z=X6aw@H|@tlJ6j%TJpk6teff8zh-(XeR~wYZv)}Jh8cig(c4D@)L)(}Ij8*175?PK
z8E@`dQY!uHL73ledC?%;miFz^y!vthfm7=FfBlIj-=ixHc;q<9Kk}L-fzJUpQRkP2
zBE}nF*X}g%GxT>Vr<vuA=jpe=&xh*x{<KZ}ocriZ5C_F;+$Z2H3<R-ko<y>)0WN|D
zWieGi;1AO`utf^z82GkyH@J^{M{iDX65w-T*QEIMasq2={2>5;@c&(x7tfRZ+FAEK
zN-$vORhl}5j&AJU?Du}ntDMY(!81?TVTuJzMS^cfHD-|!A#wk#qwL%!$Qs#x=tHP0
zc+~woiD>@kU-})sNW1eid{`V7Im}@jTi|A7jMvDkq`i_6msS{W_pVq(;<<=H6Q@n8
z(>x41H9bmSei0qMDJJUO8Iq75x_Xm36$)cUZ<twd+h2$;L{6Tgo*5N$Dd}NNX*#lI
zIQ<h_{X0GM!;-23Q{ZR|7LFd?ZBT-nk<wm$aA`t9J4aV$?iIX&Ha1;$G)@=YZLVCR
zJFut#gm77CrvY@Lj&ajgH&t&gSlg47(6yHyv1Z^ni1B8Brbo;U1r2|M_e77g^!@v~
zoS-4|BK`?d>wd5ehc^ATJ{@e}=*k+sX+GFbcX3gXx+=_GlXDulMc1v=l`+$$J2o*y
zMT0$JRa)uww7Kpdr@A!hVoi|-q}bd|Uo5?Ed9=HJ6wO@KjGi3<6Z|FgSvO@#jLa6Q
zcnF8B>S}I9L|BIZ9%?j%U#OquhOZgGlKZ>h<ki@wFtg}##(gS=(PxsX>bF>LL7mPZ
zvq$0@<nmTBZ&nV&`p~^!;C36n9wgr`FRZRq6tzNXvon)3My?rqV5~cjp@GMlU#P!Y
z7d%>*#eR(G?9h_S)(0H}nG82lcefYTPO~j^Gmjcn)gz%l5)~S1q~r%(+4MQ78R8KX
z?E5Eejl-5}egfpciw4jOK`d7tbfl>JWDu7!;YfZ8k89d^)3*OkWeay%y!%-&x-bjt
zM{W#cRb0bLd7@oN4_49#Yfi_fi<0|L>d6xN5q+MBU`Ko5ybzsY`bo%&1-)mRz|f@x
zNoY8{E^^iR0U+oEA0WKeF5w2?)o^L}{>;$TX-?rLxF9(y4>ho|43AEM>uM<W#BhqP
zW=T!WKZ1kv*@tagoMZveX*j~&k@bXAbh|UP%Z^*0WU0R~wQi_SwFz1S*KH1%0njy%
z>~#WRC@~8wcG#??b;CYmZkX~j+5o3dcJvUCw;Xu4u@N3cCmbn{V;sjzO|M<PUN+kU
z#RNB>m<9E=&e#5R3*Z9h^kW1^S%TXHDrSOoqKO!9osbf1-X5un;ne}|GqVZRL34Zm
z3T^xU2SVy^TlC*8gUyqWI`$)8LV{Rq_jgd8&U?tr;cp^+HT-ZESe?9oiC4d0?|Z?l
z_{?l!Nz_0v-jUeB^HHNpxhi(!l%@?9YLlYrNa|P@_FK2?^uwJ1p|A55{`(G{1A#|3
zt_|1~Q&x%{C!Zy4m%|G=xA-9^);*bgHSDrso>bK3($Qo}$VJ~e<Fsu>ppp{YTb$Z?
zszk5mi!V$)Pq%oG?+!-4(2Xg+-Fr69!WVmObUA*1{rLD^OfD$+S|%PIl{)x`o%B||
zbKObY{L7ILl*&$<1AK$2(LK5lf~^1qF?K+0|E&frcn()?Lu1*mO$p0Hl}^7?nG(HR
zt24Aee}w%y<k%UYdlEaEWA?)!aOG0yWe@;WUpExHI7Nf_J*f6-J;ATRwktd=v^9)m
zhB~=<U2ZXPLa<<LO$z^=v`%HSa_)UK-U;+~eB~%rh~$&ahfQo+mVqA8GHgvd;?rIb
zxLJWF1I9A?QcI74j$W2Kkew_9J*?t{GlK_hooJR7I}^k$YWp{v-^KJ?iLx`?@Oh--
zJzoj=*RcXZ)bjWkX%bJ@Zwu2T&L<98JOIVM&q;mXpYvZs%Lgi%K`CTjXbI-3srZ7h
z9qaDDy|x*6ZR#MBjtsln%GmgwoQMR`tRid%1m<15s(J6?Qzi8QT=cc#)Kv`WHli-l
zAUgdtm>1D`zxleHt!rHN-cG-GaCNKL;)c`jQU9pQC?B!t^kJU3bVnV;u#EQB6`(Zo
z`rFms{uMPwUFN<XYsP-!L~p>z*e^4wn6Z~YK!^AbmdwJs)fvShi@&Ev<IW%I4^x@(
z3x*NuJQZ4jjSsSPpC{3wwv7Tu;|}P&Ym6L~Ton-5ce~F-4&4vTvWwQ@`~X3Uf-SL*
zicKQvgIOQZ93Mi3^Og+a?<F$?+HNS3{zLr!oO6AJ#F@|%o`FdwRF`RLm%&rqgBM1l
z^}50N%*UxN2WTxuqPuIfM?EX`*B9wq-KbEH(t`n7Ga-dvPB)!+CoCB;b}5-GVijdK
zkqKa$L!ismzul24tv{UM4GM*ZE;kO1>|Zhla!!Pmwbvjueh}g`ZEgYNMZgj{t?$qH
zQqsUfiH__0cd6rfs8Q&A&q4tR7R@*Y7p{JV)sL0qIm_aT^yIJ&afQ06?08ay{l=FT
zlhA8NDA+T}<nH#~(M{N|)3R6o_0@j@ukQF)=wH$_R4;q|$&Td`AbTJkT@&ixsB|9?
z?|@MMXJ2g3!>|CZr?tAo`ssN)-?YiN=O9KTe~S^f3dUZ3ixJ7YE=7sYR6^hxW&81l
z63m=?O1Np?qO89c!)Z_-oCLQ9#j}&8UwO0wN_^MLmXDKUPLN-D#_g=BR0Mw!d#x;1
zbcZjAd1Mh98C68d&oe>GDZ0fW>i4W%wH8lxZ!OR@o_yZ5@d~3w|Iifnu4cY_tOWKv
z@NKda%VtA8HmUKOq0j(^;=G3u|4HVjeZ6<N*PxS$3s$)T;$YFs2J$oX#h?vn)5lXz
z-oDdqCzzmFEKmXUX$6(e2hu!jfj{@-e~uDOLwCru-1Uo#=DyDx9rDE1QbdohuctaL
zlYBSCEW&k9G(B1yN9K#z2_vUD`M(Tcj~5z;1|mwka^<l}0Yqr$&(87ZY+5E2XfPmO
z`Wh^`@AXo9s1o6_NyN0<Q9-TwO(kH%D8=#ku_ea#gom8D1N%0K$WW%oc7d@qPAClo
z59e9Qmuh7W0rUmIj~U7Qn31>QB|H5lK&Z;KdgqTluIfF%W|cw&c%mc_EZ7O)uK;Uf
zFb4oXBt;y}Jdy}~(<UgM<a9@Wzq09j(1`J_cCZc__8_ipAJWBQooLgBX_FE7%zRoB
zK`8(mL24Qn*>}a-A(#pDi^Cp`HER+R744(1<n=2yh0shy9hMX7usC73ZZZClMR>(Y
zV0eJs0XgMUX(iF*ZzCBCY(od7^ec!R+hTXZN?6{jrOI-~5o+U65X{4{>#Z;MP{9%u
z;j4W}3ga1BU6WInEPWbY!vqEX00AYneV%PM0+4T2dO)9v;-|4@;2HW9=NQ9=*fJV}
z?_zeN-X@T&qF9zXl>6F$8^RXwJh!j3mPs)TN1%V(gcE-qBYeCHEwM|B4uA4Dk}&kl
zOpAvkWUk=DQSr}K_kIRc#=QeGL7k{8xgxm6*{>LhcBy&+0p1XuvTmcgB{(OWRHPca
z38#wt`eFUr@k8Fmjf=2?Wg>G~!a_&Z57bU8o@hG#d`p9rh+IJZ<9!-L@nO4*mtXv1
z&b|5Uzn4l2I7@=`MH8V}fi{a0vRX+U+Gb3aO!_isut_}8V4KH#ZMp%}lRs@cGjjOB
zi&CpLH8YUbt81k6tN!glTMsd-n|men!oCh#6fWg7TF|!d7S^}k@cDs)J=p2f(T9@>
z2Nq%{@$BD|!vh~@CQFu7{IB4~+;*|^8b`=F6t<PVRIk{6r$^P+hCtb}dH<(APziiX
z;L!ZySc~;XsyldVOh)W>vdX-SRg^LP1gyoxPn3dPnk7Ct*fK0N<o!>ZZd1qo(M%z3
zmg5bYA$PmDN*-BSLgq?bow7tExvC?qdMf0_$APM6x<R{#;>H9(i~@N<uJNWNLE0)8
zgN#!R@A+7}A>LFL7!AVB54`9{(F;oyWnd%^V9It+sIIxQ^W-l??3xljUgrqHA&%6Z
zeXkmA^0%5e>JY--+m~6}Gc5mp<Ji8I!?~#)dC><Gxl4_vt)+_U+CgEQu47Az(zAc8
zfcZ9k^58LN(q;+Ej|Nw&mB+In8vwDN7xvcwr)dE~Ue0j9c-9ylh<3x7`73%cd9fmh
z7WaUVv-xM)neU;gfPk41$jRp_ol*4S$FRVa0?>|t(krX)@g?|82k_FGla2Usv9r4G
zEFe;k`f>~Tfun#OPXp$L(q93qfPm5w-~v5+cjtS+3S!mdV@0n=HOD)F`Z8Xo@zl0E
zM`!=}<tq3Rsz0s_jvcg&<8a0f=f?%#gI{p17;{q_(2EzrOnLS)mEHk5T;-=0%Ha9h
z$-4+rOT)-zCf;yF{d)2yU5@ZE0u?T<7%$+w+wGsPqmhwZ+TFS$%m7VX-=@+5?dwdy
z5m2RPGVanb?MH`M%wA%*CDXVh(&Pp0fwi3QBnMDAp=l0U&%OevcNH=oktDDWv|Hnc
z+h8~d-WT6*79m;Sf=Mlx$%UHQSbz%kWn`M7hs=Te_m@*FaO8qswhz7vugPk5KY}nH
z-ZTa|xCw~B!xwx+0JOQdacTdcwp$4LTvVu`$&Z-*OVvOQNXW?Ty*kK#h#C!iJf14I
zRKx29RVO)&Iuk&i9c%ZS=7EOb)0|axat|%MeQxe<ra@uUD^?K|p!1$b9)!8J?yl_#
zmpqhcvIw6HSEQSVJUT2=uPqVgivt{}HoX($1`vuQUJR44UV}W-!QiTo$3rMlLuo%p
zJ5E$eQ=v!4XMvbO%mOWvi!4v5*JLu!?t8*o-9#>9a|UfERieccqf-qkJW4jov%-(~
zp~BAXn7ap33T@hqfx#}e3IF--Kz+dLF&+~LEUlzMI*<vp1*RVJsQQ(rc53L^EeFv1
z#-?Z9s^zYS%Em!RT4wjlfYV7sB@f%PrpQkfFPBrpY(0c>I2n+%XC+QJAMl(}hO#YE
z5Gx!jyh$_Lk%=`*4tCqmaRRM*PZBd&?0ew(&emA3^N6~qQXA}mZgS3DIityj`ao!<
zhmpN_?}*2cSS*Ua3k0SKfBX?Ft3q?Ed9^!W)HH0^<z2iz6Lb<ZQveN92`zAuwA%$q
zvoWzd{TA#3wHB?<*1y%%zAT%-)w*u;UZ0L*j{qV~)<G*e;?f(|51?m+S$V^Z3ncBB
zl*Vd!ipSbZU)lQjkvAr!?-Mq3J>zkyuh4GC@LxO>X|=moK(p}x`b@lZ=VBjFhi;!q
zrgiUFK=7#B76rnGMf45~=7FAzvt!MH>+^uy^9F0=CXF45)<yhc87M`U$kyX3Ae^`u
zNKI34dA(ya$NUW}ZYt|?AR>@WJ)1n=bu8GyG3{J*fL}m9=<x9bmfFKU#mW#N6iV_Z
zNMPu<zaPl5F9v=%?LzmlX=yrY#H>IwZYNLTi)x|0Z^MrP9)jI{v^4PMTrHIuCnE&@
z!)UZkH<kMWX|GQ&iM(Xk4`rcFa%J%tqNeAzChfNyIY13R%%cV>n{A<kB2HDvfKF9|
zw6nbK<ZsLhJf28=e*`O<g^jSdQ2il%4E!2AdJ=}Rx!g1AZ^D?nYtK^wAh}tiwR%YH
zX`dCKEv8#RhE@xv(Dvsp-Kr<qPKYE2c&2Jvrh&g@##<@PxA5<q#mFbHqNpE~w|+yg
z?m(SK_t55C>=w%Gg?BqXq`lK?e}_~t3pSE9SJl2YSvu4-nqiN0F<~UhWC4|aP(QEi
zzHOa|NHS&6rS<!OZvQ;)Wu$Bd=si&<eIwDm&rBol#$T`CoF6P#M3Zb!GwUMt1Pf0@
zC!8?2m1i5fv&XTCtC4LHftuaWEe02#y8Sg8$hQ(eN2@2TIc~y>61%{X4bNXlW@dIz
zuwzDfQeC{ysTAmA{BguD(cstTETs__nu;_DU+gUdqdFzKnxw4413k8B(@~iQL7G%b
z!_yVcr~YGcPU2RvbOmN;H%l`WzwFo(91ulfMqF;i0OOG*soc?!FZJ_MEm96B0u9ii
z-VI5<^!z5=M5MW4jquv@fC6%pL>np;Q+6q9?V}~8Dw><vAF$b;$kuFN-rF5YuXAk5
zs){5vMA6but(48|?TI!d9f)$RQhoGaqH1!qCkiVep^ka|oaMT#mCsvcmp`#uU;8Iy
z_M#zcef1>9?bWS?hrpxu{S%%#wfdhBH2J1e&#+!A?{zWk$FAgtsOeDs;1Oo2Pev$!
z+PnR*#JE~TNqQf^+BtX43EgiO;4CJ2trmMEY&|t_GcJmiq<E{6i5^Bna??UfH0@Vt
zqhe#G0FyzTE$Xk&I{OLn(U1HXZ^#(+CBLH7q2YD9^cg?SKn-iHvVg9H#!aX;vX0yk
z1B$PqzUv$7SwTfm7pJ<F0xpdp%)VfOUum2Xj9z*cVP++Vm2*%?(+qRW3ta7@M=i9}
z&g;W1-d>xRn(3aNtuKzis`_E4Dv4;}rWJy4q`L)okv_86nuA{J3HI2*8>Pb0+69N6
zsCG+=Tjl{PDJ&{-x2j(#wUStzF`*k-^`VVR@vUZ#wakZxf*%a3mXK2dTtkhlskzm@
zs%<~WcjzE&@X*J3ORA85QjE@l*F3wS@(yi6JL@U5motfUX}Oy>^_d|2u>D16>glBY
zpb)2Qp(4wKR9Dl`5?)KzluqjY@SJtofMyX|%T`8rH^!?3)N+^oOX%<GIi_-lup_x$
zM1Z>000<0doOLVt`F)D_n`t|IUPwxR=52$H;+wcjy68b>G7`>U?g60s#B^^?Dwb$0
zoKoG-SvYswfdMYmW>%XCZ=oK6)^ZCtMT7HL@{A0lK7xdC2WTm7<vFbIDK!p`aCHRz
zJnf>d(q1MBk=;m+Y52yMuv3}K?kj^Z`ec%2qqcrxkG2Rpa@0_xR1fTI2j=-V2tDD8
zT%XR)7po^E@hzMBkTk7uUm#!@b@Bu>OSHEo{IcJY3u|z$3nT>Vn^qa49#BI6yqt8@
zFXJqk^L`U1;Wnx4PNxy-(kWUOK3X+f0u|0ODh~a;Hnn9LjH$c^h!jqSGiOHUjuywM
zBf8_etJVB`Zr_`8{|o>(qiW#xd4L$eauw5E)IXCM3Nkzi>ibk0HW82LDt{q*v{@JQ
z9#ZY@<%5uxCtqGKyvq!zo_A5vc38NJUJ`}`m+KsOA<C2$m8Bm<cMPA2owwuYXdr$O
zg<H;Ssrxj{V-k3^VKTW_!k>EVuXm)6AvePw>aKlwKHqROyGWDX>+q!aB&y5}X!@!0
z5+NJV5R&0a@DpCnLm<P;VGj*hp{}D98E+ES>qu_+37Om@B6_DQ@V=_A9rqEj5;s_6
zBMI?Mk9Ps=V88h@<*BQI;YqIM6%BrmZmwnR61ejkL}UkIQ_&ymaXfs!)Sh<UCUva(
zlF&PJL!fbo<R{SiJOur-75Qt_<?9+r1JU~v>1ohAyN?&rSgJh*gzehyzlRMpQliiY
zma37$#t|p8*E}FFsYGVl8>}wEB(<rF&AR2ctd-^*6$OeyCohat2IzGs<Rh%Pk*)a*
z4J<Y#nwx;E#|B5D=A^OIQ{Y5tT7V7CvhNkIed8E)<Yur?0?j+7=DxeQ&9Q-sNADcx
z%~4(@abBEd>807QP`g5t8UZa@K85pqV@C<*%bl{X8o^8DQeZ*?Ku(|mJ^NApj>%1=
z6FNU<MMRIfmSsL!djb>3%u7q!@w3uN1|ZQW6$cCM0of*{GEe&1d43*AfOv*Savci0
zM8XY!bE&KTc#$TM(){=<v19Rm^Ra=0G}W1gqS3nWrA%h?%K^~6q5179ucEf<_oh?^
zExaeTkDq|zMd)Ch0di=9w*Jsk+~6S>z)$epndhsVc|XE$eYl%3F=h_F`TXrCxNkpk
zxB1Hfu0rv^&C)Bef2FmYzn8b>5%^&faA@cGs(jvPqrZQX%;R(cwDy~{+_Ou6g$vDV
zyd)AYFJ@EDPcA8Zzb%3f<l52rqL_RPns*y|0?Wu=toH$6ABCR%S<ZON$FLh0cB56z
zsF=k8f`lhl8Wpn-RetveO51q9XahY?K`~`ut0_3)nGdRgCr5K)O3A6;8YLh0N0Hn@
zNlic$;?Xr;k`H`_CP4GTZo`=sX?+o^aORDOW8)+ItTy;=E#F<XC|V83H;@i3wMV>I
z2F*^+@`^(v?j}0^Wc7p%|Fm$hH`%~18dNdYf^;}gMgCN~;Eyvecj8}MpgG~%)%Tl~
z5WgAogrFO(fD8Tb_BF0Dv=pQjd4ie4kkAL~VXfK!=~qVYbr4s%L7;hw(*v#J(g#w-
zQ1R_9@4h)D0z)|>=u&;{@odthvLy-#)|Lcg7+e0W6Eac7tT{vGw+1quk9);(79!C&
zFm7UwI&p^;X(w^fP-#R|s!Kc5coJGgV|FmZ&KdSUXHcQFP7vjeWzQjKLtq_Ave#Y#
zQ9#|=+%1r{&W$j%U8s}pW6N}gOw!1r%AION{rTE>7o2&m6XWq%w3zbE_C@&?rg3r;
zi}HQ&ocGG>xI_Z2KoAe58rSD(!tcpI6cfU1fmxN6Qhl%)3Wx>u#)icstndrVDtwn)
zw|@0L#2w00blkmu1%g6WOG|??z$VMawo1rxh`kz~+lsV2Td#Kg^D=BGG9=E7G({G!
z%?tI#tIGHyZz^cM_Iq@Km`G!ZHgw!=lmT>7%#G1X+tgFOGqBt^(9mitckAdoUWf*F
z{6Qe|>%V6MDx#?TWWUJG-aWLHP1FZaLy{DJ#ZcC&tz(iBCw<;>CU>ql;{n|zV53k*
zvOh8;+w9Ga>+pe1?j=L?Trp*ttDJ37fbSq$rQ=Q;tO0FbBA(KTR0Ljc)?joI`he9M
zr3wkcx*wdQ_Gi|;swf#{zZnLe43DA0OE>VAP3hHmGaC=#Nk0&Mxi2sJg(n^8kZcEn
z*bX%j8o#En96g?BBg{lSjPz<iP_ZTwb<<FR>k9h+G`3Yp2SlyyE5T!M?hltckma1I
z<+OpQD6j|UGQJ66#;sxy_a=)!^W%(qsBrm<Yrpk0y{3AUO!)oTj;HSj;DxPO#eBTR
zOw{Nh{}b8mE|WCF@T@?_PfG*?=ym~--|q#60wT|gqzTiHmk>QV*7gx&{f(b9_)*cI
zCm6JTu<PPvRK~Pk->@VY8&5tSKPxL;VZ2%`i9IV@7<fc`HMkRD82wxm{t0F1woN-y
zVH{AFrE549GM$@&8&b38l9!}VdLlqc#O$#XMsbz$(_oXB)!He97VNoZjat7iONa8)
z;#t`~8W80L@!!TE^L$(|;ARd5d2th=<NuoUS^YdJ4aV6W<W81~Lh42vg3a^3q=o$a
z3vZmAO+=u_){A84Pi8#$!V8qkd;Nz>N4qB7Wn;Nse&wf<s_1lc$rWpMo0@{Z-U;pW
zmWO!g)7?H~SD65<6-A?<%|vLgoR~s{)s4}$;G4eIY?+-ccPlz#j~4~L+`0Bxm*JP1
z2n`})N3!D<L=wM_oWA0>;pNB8;|^{K@%KqlhxCELw4dG}gzyG-mZuIR$AoukBz0%x
z!uNp~yy4y@SxSY!TGByFq>cN%@clVek?3;GSx}Va;BIrf<?eQI`uChuC}KL~S}`A2
z9yADzKX-c}VImy&o$zPm#<LrD1j{~24i+974)2cyVw7E(jt7@sk|wX_V<Sg*3GVgP
zy6rqMI`11=7(QL~ecWEHrqBVu<w#9~t6@sR(I&Z+-iIRzF1&md$YEtHgVW{e^<HQZ
z4PlXI#e62|K6HD4P`C!7AMw|4Df;*%k^@z$Q-N%KMuxyt%ACj<h$FE+EZIY9^3`L_
zcOO{`?vC>LL$cP+@Y}*V%bF0Un{Hb2fyG0K?tasQO-GwT>yEU9TuLFDZ}@0ZKtNp7
znvJk=60?$X3~Zk=iMmgenAJ_H5i`#iC<G#cnn|-BXM+%Z3_<bXEp-TH*8I%}Jz}Gf
zw*2}#5D{is16v&8E<HBUU~;GHg79uhKDapZqVD)Htx(+eOpYhB5jK0e$$9`L(jfhQ
z{!mqF`G{|)U)@PwuFxGA-$v!}3&1#I&u-ZpqJL?yY;~1UdB-k~I6BaxPrW_PRUSVI
zoWu<PE_t6X8CI2f^XCcTGbYV<ElXTu%L}dKyMWiyKDKrMsF6-+<;!hqt$4G_9}-KZ
zP)BdY5_W1>hzaQ-rmGSpzkHj`lx_~-5z@WL27VjRSDd4{^as3L9mwB}CQAX6p#u6a
zGn(S_<NUfOS5C<U{P;S6o)e91AA2czz$*4d_l{d@&lDtqT55Wy$_6v|B6A!W%q%Fz
zF8N8%&ev&VAG=!-x5!&pX-W!hIW<l(107qdYe@UGT;v(B#+7HP6#La!3_hTQa=oeI
zJYf%~JY~@bYIEWi=I5)v&!L;&Mu?wV%IA%*??m-KAmtGTV(&w)kUecB-9gnMIpk6l
zk9#67<AUxW%spLAT&P%jLx)Y3e&uRhe?lQ0whF&dg2SBMM_bA;8Spqe04<MFy*@<t
z@LRHCmim%f;F9jdSh2VBIF+s_!CK>R6*aL^|Hd)##<9lYkfyjdAQ|`P4(b|6?N-lK
z@>$5o9XYYMH#O7Qno^p-n~vHw_RmfqsnQ%U7<#j^cIFenPX8Xmi@^EGn-fG>w8qFp
zfkKOiY_rxnnPt(sl5WvW6#^OuRTu}tG{=)_K@*8AEG$J)&F%M<zkJAY>cWXQna-*y
zRwQVpdqMVmMQGT9QzFhyg`EL({`0%X`ZA(1oMX}md(F6uB~PV(P)qHK#rJv9=`yiC
zh)8dJ@ue3?iQz*1k6@b*)Ow&(WeFw2&t2YOUo2_IMMm>_!j<oaf5giy>yPyBo8kNc
z05+2aLE)kfkTDFhs2iod@X}iBI{L1LIG(P%VnN=5L2I~g3ujM9<6u(y*TEKkvCk!g
z4E@sdFKg!_?!1j?+OsGZo<nlQO`0VJA-INoRt)B9Nz*;-6$4X|a~njnT#>Y6Tx$r0
zt_r5|L9YrI0_kFMR*>bz8&TZQ))O09HPP<q8Z+7cI8b)H!4%V$4`1kyFT|4$^s3z?
z)XD|Pw#M;ZG@z-%IH2*W)PxmL|JG<l_o<Kb)~C5uk|GiPaoc;GUQs`L);v-~=Epho
zyjTYA*9wHuFuc3Q&rp`O61Qt27PWg&K5jc1KNG|^dcyi!x=ojaM+J|+s~P=I)XVvx
z=+A?`m6$=a*x?31X$1UBu|#*~JW4A>H#Vvi4YK%`qJU`u7V;E!4kV{9%KGAds`C_&
z-5!0xkP|lk5c%XW?4&gPU6#dzc0{N!;4yzfN@&2MhXjfUa)9|-iFZKw<h9DZshLVI
zENR@VreP(8>MM@q{5B^B&Yqd?L`?UW>1#mhs{=uQdMO-7H(`XHH5l^M{Ez#>#HPYL
zN+o;2rGDm7kM`t7sG%EbDWGI6+2Ab@v*b`=6eXsZIbpkQ5}~F?+n&`Too<l5-76Uy
zB$GgUVF!#UkF@}7_iy07d691eZJA}G`jj^;`qozFpk0(@r`=u(Qx@D7_HVa69S}AT
z%1-JuBDEI%RXouK<ETqyER>mcx|f89dRTs5U_ZWln(^;B&1A{iVpnL8c*|WU|EbVu
ztYoeD`Fnv_iH-LR>tG#Y1Z;;FHY=$2IK7%xMglz+y;f7m2LgU1a(>)1x+9*b?5+48
zp5xj%8_NYLnMQz_ZPGY0h5i_19q^k#ch-WB)VZ0F2kW;Cv7#h_4JlT=Q!vLtb5FSd
zp)h<Pl1Oky=M*EDa;U(!{Tq3lV|H_M6+Mk*Uw8|o>N_pg6zdBp*lUl$A-r2yQm;>E
zI6T;b+P<5gSR(+lFf+pAYLZ-t^h(Iak-}DWNEh+-|KjdFqngaRzhN9l919|&0s_Ja
z3JM6K5I_Q`G*JOj=~a5KQbGvmpmd0c6s3tMD4oz-(4mALkxqa}389A&NPv*!J?Px`
zfA0Hz*SprUo=?wneQ+(qb*^*HKIiPS_is1ACh|k&|H=GqwT<<p;2O*VaHFA!5(U0{
zVtgk#)TAZf{B(lyMM_9NvWtg3m7q%fVSsOE?5_egKA}82&I#(pJTM|YVCe++bEb&Z
zmXMLwHp4=<8b`OxY$VvOhYtaJEXOvV5&^g=dti=t9<Rcw9L8GaNLHt<<2*V+0m%`N
z0pRlF5951-?L0o*nPqFm=^di+EHk*&9=)DaVm_!iM#{#yC@<s}6xYGQJ<`wy{DJlM
z*5#-5%oliN=Kb{3V3@K_<Ca>vw!>Nvsz3I`9A6~aZ62>e8@WUNu`+ESx`Mn&*HKk@
zrzjc75Ta{Gh_jL+=^EqQ-XO2LTgp#EW|30R{NHY|6JTTHXl;I-_~pk=oLx)G$Z4c^
zEnR9#^ihAay25*tk<kq%r}Yo^3nGplT_HucTLywo0_O3zLD?|TEHP(Z?EkpgpDT6W
zo8v<uR7qo}J_Cy3aPG>FtDpT!^w_3J2am<EnA^k&3XilxHG>p7-B2$892tV!Rx|&n
zPhh?MH_O-G>l4OFp@AIbdeJpN=quR8sGN8>^YY=p?C9#%0ivZDz*3uThO9Eh(7w7?
z{zGDTv}AtDd)faeR{%2A|2;n>>>qLhh|+DI?JW5!G=_Z0d5~EEY6CrdU5FMH2Ild?
zkjKy7bGy~<Ik=84jQ6PxdwplSlT__;QFTk{>hyE2{@BOObNBv7=y3xD8dJvyxfC}@
z86s$TtKS^8W~IMjIMR}v!$Z&AMDh7|j{-CQOtTO<5}xTj_j)A|B)4gzC~bZ{U|hVO
zA=wDE`Ig`@0`2svzt{xCHy{x1#Yrc2J^Ir1wdwWy-bY`sODe9F#gV)ax3LZ`p|~2W
z7BP`b-o3i9m(zBXOcK;^J0X>be{f8GwZHgseeKncG1C4vGjB{XZ6Amhut0t6!O<gh
zZ?4sY>};2_K>t7x_@~?m`E$?r+7~21y%uqRl^A(fop>BiLOONQ%~&2Swjk6ZgI1ah
zJnvKS|1yC6w;Uz{==(TceJ@ViVWiZVz#Lk51wx4)hlAoTtEcqg57im+p(he(r|yvA
zc`Jb%(B9()Qh%j_P!49XeC@!aesm5<wDsdr(gt%0aBXP<1$f;|kDbX3y{f$%6FKKK
z<r&AjQQgB^A?Z5gQB4szJ)8S?@rKWtVtdZCd(UUUz5c&^PJeiY?Y;275)0M`@*$R9
z|54$F?6v3QcP*p$DR8#+a%NrR7ws(S&vM&Cpb(lM7n8(D%pyh5`o4-7NaCrzh1_52
zqX(bK)5A5f+MN{<v+c~ZN%)sAPZxA%6>PF87{j4jO=8h^PSl6q+umWUf{NrMZ9FqO
zee&4~a1G6wJEJ^SyFGzrSg9oTtNP4`!-W*T!R`U)&q|n|?JiFyE7BKuOfF>(FH|(`
z789Wxb?M9mwJZg>-h``lSO2nnGh8niFGweA=k?cKobwBPj5<OIqK6QH1M2(UKItZk
zA}8EJ_l<n;q%yFD-<cS0<_VeRQmyQDe;sHU2;4*0QB1p~h!ElfOl~4fxn9AO#l%_C
zs`hiTd~{h~Jww+uim2!Z^;iFrrL1m-yM{C>?YBPsx|Ep>TuO;E^uZH5(v{RorIMYd
zOvcw$R#bPI+rQ!U$;5u_)(ZbL+=8xEceU=Crv!eb8&?Ed@h3L#Iow*k?pNJal82I=
z@)*7or@sKV*kPCIV)!r^)&s@UMD<<ON(3h0GR5EDvVNjY5#6t<oL7#RTbrWR*I)e;
zw)A?~^G<D<Qci*I1M_McNp;^IZuhx!VnMfBHMf5>KwndrW~H*a>7Es^Wfh_pOosNk
zvzk12nfhJs(+vnZH3OG5SjV`v6l;-bl;%)tk^cYdnQHxu7E-XWVL~OSv?YUDU8j2V
z?wk&UG~dTC+H|b+XkzFJeIgPlOi7Brn`P!ffWSjlyK(#!(~Mpma9IUn<CZ9e+Sx!o
z&oG&KC7o8ygo$MZulJCnf_uy6mzP9QPYHdX8u%gLTc2^1a^&t!iyshFOilzq_yqXx
z*H?L3?Nv3|+v32i49-L8O)X^A{}eKT{qSAi`)lEI`0BpysXQU?`5VCbcA}oOkqTTr
zm&jh4KexDEYqMXYBc^s0pn$=(sy%ONq%U1Q<hK`W$R-&N@__r+0K(i@@uzn>v=p9r
zIe2l75k&D6nM5pp3P1eQQc4vBeiVx4p!{>!%%gkAg(v_y8=cur-2eA5K4*eV&zJLn
zgJ-F#VvNAyUB4eurGObKgYw7Lx|9u6=GD=khq`nSIE8Y*omg$iS5=H#w#ex4*c%FX
z6|f5HeXAifL(xpnRpqqUJ>+(c<w^4R?4Y>x*LSSo?~%c&*Y6xMeZG-@+;UUcIOeNO
z$H9~LX4CiIxKY5nLce*l^2R5=C5etf@wdib5385hyt{BHizx8+&jWYA9l8Df!i}lv
zduEBZi}p!AOJO_s8y3xCm0z{%bqL5}q-JDf5NY+rtkKuX*zS<sfVs?CQY{Iri*-}K
zHOJQ{5*{`4EnYSE{sqUfJwfP&$z50LRmiEiA~?7BZ{Lk4?5<DRB)|TR<ARC3{lfS5
z1|Y%D00!_ub<i$sZvNjbu4{uInjJP#g2y6wlKzf(rDuZgg!jzbBQ}gw1N(FcKg*YY
zM(2Iuv*r-pMEU<fox%4LodNtCcIe-}5Ny0T;1wO+M!2^-G`-#VZx<4_)!SimpWigh
zHd($>`;++f`_E`S;A{d0XdS1k=(lC_uUAFEluI@Lg++rtI%NONh`IPcX5$~v$Zs6~
zs;A@jj2)D0`SCz*h5>B*$HSlfWiMs}JiuGf`d>_4)9FW|O7N?-$p6}BKRZB0ZH(Kl
zt{)20il^TYtWMkA8_IPIpuk9MGhv1(JSq3)z2j_+(5!cT2T<YJ*dC_?I#M&Zy{DW|
z0ICzyO5BJPpoKv_<hS>(mM53M?1xCZkybHV57p%)7kETK#!75t<LtU8@?&yOjmq@B
zUvUdK5wy3gNkdied#itb+-)DnP^rpj`BD!kEx3Tx7?*$NLyHuO+EQ<kC!Tl!ic(9l
zo=wy$DMO}jop#G|i5vq7&hfu7L@<m$kT@OY-T<wSI00AKYf96DTtxMwoeFY5PFb+z
zYC0kFDbEk$WCeL68FE&b<sZDgY>(J>dU;7-=pFSESC){@HxZw4&h?s!Ss=jzI|Wz*
z<E#^0oKaiZk(_fUJ}7~lox&1d`>S8+r<_`m=GEagQU}0_!Key)mm;#N`o`|aL$!gB
zi_<j9f70`FGFYs%gt%TUt;7?EP<6u!ke!LJ-I8j~OwmlNS1!i#>3s9fQ{NqMLT%kU
zy%F;t^nCY)kD9FJ%PHc4OdU#8l)zA!soMcM<3@EL?VA>;^BMG#rnPIYKFLY1Uq5t>
z#4jv(^^L5$yIzO4YWs|TA^?McKrYNz_{eiuBlwyy=`7-gip-5hpWR9jeUPuYAgwhH
zeKVdEhsVpl8FznS*f%Le1eE{lcqQZfKj|6+=-P;ooSw+(n5K7PybTC})bJ1^m9mHi
zt<C4)$1_}60y?Xm>`M_yQN23aW{m)M!wzeav{2l=l-)P$vA>h9`QN(*94)G06Mkbx
zq%Gn=iBWvhxp$LW&>m^^S>^n4+VBy|p{lVG^;xeq17JB&7FkSQc&-=F{*Rf>pXC_3
zoHeXIyEYEMGha&7n|z<($!sl(XQ+*AEzY=zX2c3MYk8oVXjSPX9bBjL&yCIrrnH?N
z8Q2U6pBWj1EYA1YRkQE*F>FcGN^H$i-~9&m2Q85+EV79wfu^LlIW@D?xQ2;%;;8ih
z8bgx^VW#?3!eY`)lK^?UZ&rQOIHNhmVa6X~80y&azR7BCa-e+GPY3Q1U*jSR`?f|7
z|G2`bxOxya?y>&kUQh;0Jx60k3UKePa^aKoo@s$9=u3^^f<3o4mPDv7g}S5G3}Q~l
z1l=5o&nkZzM=%JZlGhV@3Z&dVate7yj)f%5guKA4h=%U1`PUO*%|ifDPUUozfN0gg
z01xWdoIH&NC1L(7mLy0hR32oL$C-lUjBD@Q(nc$T1SYgzW=!y(y1?V^66gotF|7Hp
zan0)YN0DPIV*l6+Z^dMdn{A3?+UX8+FTRgj2R~AO1KDtb7&cuJ=1(ri3?Ja4JbY(V
z@_?jhn9^7f>LRN++*t0cM}1MUs>5{WQWZ>48<=~3ph@TpHSxcPEF%=tKKTudn}cig
zc!pYVlcJ3|M7&9AdOXr?FUk{(n<)s&cx=+b87~?@SOXRE+6C>MOW=<EXqT<64Nqa8
zNK+o}|2##4-~`5W2wH#>XqBY)QrG2jF$m>E<!y)whd-F6M$)w&uKS_og-P&?xS6IC
z`V4o&UZbU+vz;YpbiSPh%P!F$BIYLcAFBlz9XIVwWWLt$wFtqU=q-$vxd0int$xvp
z0yH4WZNm1i84o#@GLP!x^!|gf01j$w4mqcK!REL(GLoio_oZ485#N1+7N06>9!RY~
zUlAsySTNktbDSE%3V*KFDFR1bgWg<*?ug}oe`s%^!Sla=Nkrb*{*8@3`GIJzEGf2c
zZXJ~tGD7}wM;2cH5svs@-TdhG|I&Z1r*L#?{hZr~rDw?EXlrk8c50AQOISeGov`U4
z8^^7_)N7^!D*pZR+HWQPb?KHi#tSr5@E*-P^S}|AnYxg?g>#L`$l&3LKu#E{9CUl}
zuRFa8Th@#e4{t88amfr{^seS^{SIr*78y>ad=CHnGItvF8mRHPAgCu;RdvDR%*oc(
z(n_+^$XI+LCM0q<gNxHZP^gkVn{g#nC=Ti%<#|R~Lrg$15GKRh@HXA7b+9!%hbO83
zGkXj3)ZRp%K$cIoeju+ce-UWh7oq6pu5ZpCN}v9%dB{H!N>!UBN0DE^y5WMPi#{*}
zbhqtkJ8}LrWNtn?rLC#<420n1y|aLo?4_@qAV9yq8{zz+MV9I+<yfW}_4I#M^iWtF
z+=j;4rHLUK$7?~d>pMjEkAgg1le}G)0@<`(QCYmPg`%mzwV;}GcGHvey<klcZfS3P
zG6mDz**w%1RIGCEWO&iUaoBRi7Sng0<jSt$ZMr@1%|@9=OR7ZD(YZMxjL04K#gAZ4
z32!Ezia<6NrtdceEOYNmRjmnO^)!=9#{>eMWCU~@urA)6pR{7cnkEs3N;ZxIG38~M
zPn!D>kF!oc8X4p6uvJwT%sPSJmgpMQRF|@P{Pu#BgriAbwF;wqI$k>TkKA4X<uY!u
zcfp9q-dm}Y6fkG`A9K#|UtT*AGqpA<BS%`R9Oh~LE+4I?byT_GhK{_CA*zgWU7xqA
z>iqf$X9DJq2vYIY;~C);=@@gBl5~5EA^DSA3Tk2fuE@}x^Td{BMN%L`qF7M59QIQN
zd|MDN5bPzxYj=QgpJu`#&)q!L8WdgWX?&8{);tzo6t_uie8FK>?*Klplmkav$vau|
zuAM%H*+C}v%eoKL@64r|Ivyj*3aH#1)wZ%&s~~17@g6OAtGB;=p?Lmgb0v`7bb=@k
zuu}g1sBcr*T%3~#DUN8fn2aec+r^Ljb5>&#yC%+VPs>mgyl<hQkD@N0v%+W`!^TI5
zrv!K;Ce*j%S0Sg7RUKd-X5Q1J-gJxTY|kw9TmiS(O|4^s;_P9rHu+JV+c7a}S`&_y
z=gTrOPLrRG<PJgH+zVOig{;0R+-3sU!=$a)E67b$i8q<Ay44jb`B&M>wTj{Tx_2pZ
zjRvlz#cmB#*Qgn66|3!E?!<igBf_Waa8Bd+@1bz_vU1Aw^jaCNlQ{n*<6WQ)6smN%
zzc%PvOH|h;d5!;I^j|N6&wzvpt!i^VSobUIfp?%kH#^QQqTN>Sf(ZP^=}{qS5fB;8
z7U#2xElqKgj&hwnJf{vKYp^ayZ0gOrj(?l6Nt1iO!3TCqMH1#*itFQzd(>jfC5?jt
zG6IvswW0{w>@RgFoEPH*JY;o@KZIVb_X=AnOzmf@m>ZeWYJJ*kAS0)Dez(TVwweK@
z3E>(PwgbJ?GnXah&RI#qZ6)E}&kFM7C3Q{9tMLX2G0&?z+HEamGSsR^Mdj}b!f&1)
zJ<{3E{7G7yoFg?uK3RinsOhizG!u)Qkr0&8YERip<yCN$p*KfIq2Szb%*u4w_iKuH
zA>OCEV!Z-u6oR`ilQw!|*_RTg)`n~@2a{7so6jY1E;aF#V{E40v)>72Iia3+FFoMO
z>WzIl{bqVTyRh3)ZTg&8#J$~H>4B04Tv;}wg$lc$b@lC#hal(H17Wk%ja+x!6U)Y5
za}BEtx+n6VLn>#-E+4SB3mvJZ0!5&#A=347;3LKAV=0=W^&99njjdE#e;K1<NP)YA
z_NUKF_C2Cz>=AV>sbuyy-4*_$f@Zp3m{)kyO-=&u;l&kN!?rh%3h^5T_*A>%k!@(c
zCzY;DK?$vxe!Wl!Z_D~-pXC4(nBW1&9JTdU?e3UkC!e3)Hs!8$d#q5}ZaYaLzvimy
z0LT14S9`hr%hl*F-}lQ@uAbHNl+ab&eAJp~m8HDkZ0r^`npQsX$TemfO@0$z<o*Im
zZ5CbUB{)o`Kz%=ryLFbS4Y%;nV||Ks<hdi_RqE1n_pvrUv8I<xj)prXmrq>f%``k3
z0XysG$idzkHjPiBx;9U(#eT8&(p9Xl<_<Cki0zX~7ql?iNA_6i_=&vp-hcW!pM2wb
z`lYn4ou&WV3r9PKU*<}9Nth(?!zy&D)c<P__$*2Ai|Jby-*A&q%)dbgJd=7FhkoSB
zFN|cAeu2)ps$T5usZJ@+c%jCYl9(YY5KQR);&s{EE|n&@(^9SynpIMP*r)B?$-_BL
z9T`^sZqzmEc8?x!bJ?j9!xZNBc=k+#Bci&Exgx(>GqWmPKQ&SU33_VaX2n+619)%S
zjqVNjjY1cNU`y|<k*-d~Av;G|=z^cVQ!(YqfVT6I=_lfPy+c(xZ95bdz#-&*a)>{#
zpj4O5_LR$Uv#byYZ%m43{3ojTUlGk(>aYcs0uEE+bMx{R>Vj|yySq5DdYp*%+#<3z
z=vJ1!RSx%-)6X<zAEO0IuF70sKSB%Ko#JdHk(i&!G#10B%D{6cH-e`mcuQ4^Xgmnd
zTJM<lwW1Szij@!gv$t-Sz^oMyjynh9yo`5U5;=zTSAQwx&gl#n=ycCGxGNStwbm!j
zJ0zw%(i^#T{bZ_ZWAs(anX16J2o2tso;vHZ_1%QAst)7OWAx4W-*B^UesYhqRj2Ij
z^jZSk@MHVeNA|B!i;FkvQZ`SZNu!+TnfM2$D+#nWc-9JwMhc#yQo2Tm#)Gi*-I7r1
z`%r4WPnBZWm;E+rR*RNfMW?xu;_N+^^W^f-jri#XBm<^6oQWJK?Q2D5u_}Dx4FE;7
zrP4ODq;~_!ANtYvdn=c`o9KRolb7dqro6mb>nFF?m%6r-TFC5du6>sFVUNa#oGjGD
zb+5#l3tsM?9;-;Y12dO~Eg!PClfEEAZ$?)N)g@yvs*IPu{{HNaugWv73aqBf@Se=z
zyFv*kF}9i-W(1XrLL31Bl~}WWj%fNTC3f-&)4RLex-oe3JQBI)kaX>oF?mgoX|Ok1
z{E|wmD*PK~vTxySLHeTyu)98^g){+XAo)4mximc9sUo}XcT5tSZ1GC=_|cgv?q+@I
zq_mt;H_uz>cY*2+ndT6;{k5+!axZ?|D1Wh6VeHb<n{Bf;3;T6z8I!*9e<-gHrIBc!
ztS(%!;$?x!#$cDC;+;7y27Y+g3h$5U?JeDeAD6PZ%nPkbUw;2c2I_ln9EQZuGq!@s
zi`9*FHFTq~Pwj}_8FclQkmK%(PrPXG>V{G@Ed(ppS?jMn&B-z(S2JnKor;E=W}#j>
zWR5JxV=y^s;A+wuAQa-Sh{>NV6N2pB1U)-57*^JFKj)SWc%x){d?GqDcy=E_VA&HK
z_}r<|nrTk_MrY+XN}`vx&&iywR=aScirt+mP;hh;Y8q4Jdl9*G7OC7fb7j3uuW7dX
zf&Z%8^~HBzzDp2~MikHjgQx28JI_M5$jHjEg4JAlC<DJ+fDzoir&zDpJT*$zDE3vL
z8(Qpsp`FxYPypSRXqhES1vR@u%AeA6F0R)wnCr{97#4MfWu-V=><JAaO`j#P#=v%J
zq!t;GEY(qmkWFhZeWDEjBX+y8x5Fr3Fd_L{v-0T|h-1E4dReByn9zn$MXeLzt!rCv
zc^~ii>w(FI`=An!3Gj}i%CFXZNVm8>#L@Bw72=?`<ngf>$gRc^IT0wLaebp^cr%@Q
zPw4IKdy=7#Ei<Gj2u$#-p}AIsp3_D|eNJBi{DIwF7|ELcGY^3!U+ryZOt+Y{4m4fP
zQ&P5@v%Uwtz_g&#=vi6*OF8HbIVE>$=cQi6#Ov0JR@d(J_C3QRa0M|3gICK5=1QoP
z&Z)dz%7oUN@+fmbm93oCmAu8C(Q~lzkWgwy^(mpyuMT_$x~#N%9@g4QP`AAO37oCQ
zjIr<DioV7SRGJ@)nQqe^mgk5ui)dZTA&hIQ1Tyoax6(>f$*De7a<uJ%PZ*(XAr{rO
zYgC~~+Jo=tvBy=#j-`~O6C|h)A&ww<*Y8VzCVbWUazk-?#>b-Y#oNnSY)%7dSwa3l
zTuA^7sd;J&2rY((k&Hi%O_xWQ(_u%+V|v%_o%ED;RFdZO>g>?HMw3Q{;;@Y20LE=a
z+#4fAmxvX-M~(G`?Bk#N#d>WmaDGi$Fv76~K0dE3=*wFz&|Bm_3YUZ@^u_9M1njKw
zuNsZBY>TQAXk)omn2mBI%!h1+-E9LaKe~4{Q=4<cujzy<xhKPgXwA9nTcNu3#3)DT
zc7&Oy8y|NoS%FLsRK)2Y<VG7U^5fU@Q!<$!PVg(H#E~|V_iJRob?f{ogm<@9R`MQ`
z7C$CbUzZ*>Ig;~bCpjZyiZbT%V<R|?C4{B4?He#}c9vbp%u*z9AVVnoEiHTf|9U&3
zsC4XWy^`JB+ZxPU-n+}>t!AMhJf87%1{iz@ZUxN#Fu|RskZmE$HGJ3T^2&%m_Pb@r
zl#Fj`cU9v;nxJLF`V^}s-PQoez-fG|1NZWJ#s;XVZizjpJt=fKf~G=9Y$c+pT?F&`
zW6MXQDOQxvKE(k(HQh<!-R1+*c&}x%-tAed%1L{Mr*8dl5q4`;XeZ96N+wJg4i!vY
zx;d^>3)r|v-QEQ_HshUlA-=^?^&z;8j8Dp+xe%@Os=UzY2b(Kh%JefOzsBzR-K~U|
zvAhBey|;`D2)|W4x*fK8DeQ)KtCA%vI?nt(z->q5rE1n*inZyIMr9O<PkYwVM|7lu
zk5_!!V~RtCS&W>3lSITawyV}3F&n~pA!K(qFUhyi<pE%g4&a;u=KZ(W`gIUGcl^(=
zFK_FswN4+?I~p^M?5vHeXTGjx&Kl{So~%!IpEyv$+@W-KJa1TvleVhC*y2}CwjMdC
zLB2|GbDgFIZ(P?Gka9Fh*xM27slqA4no?ONz2BbMMifU$Q%@7j`8~vso$>42*C;z=
z-Pjkt7|rf#ZKc5aiwK>$?)^5&dmSS8D(%tuo61j9O)s;@-`L+m1K=qHXar$bd?dOD
zpKPoLG&0j8f_D;&@GqyU;OYMK=X&Umn>as7yUR6YyvMG=k(hdpgVoGDdB%<h5|i<+
z#^c_}BK$jhW8TFIeJ#!j0hWh{-3K+TK`zL-s<?4MoDfijrJ~RARaKn@i_hO}`?4$d
zjAEd!Kn?=vysFh!?=b*;EU5c>&%*oi;pHPd=7Ak>+X#7(?lafj<wA8P=gd%D1E=*Q
zr`HMe<LdfTc!a{i{cjJYb$)<C?g-MeY4)>Tfvz&VkAvE~0Uy7}WZYG(e>L2+nNOUg
z#fU+^*&`e;Z+%lmES}f9;3z*GzrT4#zJYnFI=S4;J@10b@0fPSlyY#Zw?^&PAj=UX
zC0=4O6@o7CKQwJxd;9o~_x#5gPQKr9ZO&YwXK?m7o?kOgl{k-#g|`Dfd>Og$lw8-S
z;IRBECnCP3$^9rL#1*LBj;DwWe@AKSX9iH}_a<o00?xTmocfxL5%A?XPpI9!qo$u)
zwy``R4O`!wtu7MBHf!*VI>6iacs&nIsgGG&nS`+l)K+ZGhrGT6bL$PKjI`fN`A>Fk
z>`EB%Q(bGq*Z5ioHu+Rp1jWlaI6hnma8Sh61-jR|wTpslt0w`I=~VJd$nR>I5Xq`8
zSDP6p_mQz|IHCAC01<Goy9r(VA}L5IN9S)JQ#RqYixqt*T(1aNJ`_;Njeu6PM;$1Y
zHusu3UlMy}(zkQ8G6*|`<guzrVPA5m)qBGI$WO3{s&rR8MDFoil(CytM&tB`pJ(l2
z0*x`vvue^?wdk!^zZLD{le<h-;1!r(Ru(*{WTi;%6)~DL?)7NV)gKpVDvvvwc)DaM
zwf16~!0N*=KU)Y0(!<&L{t!ap(<4(nZ8xSgSIhZ;`9UFY0Aa{~l6Z^Qq6Tjs;%=Q9
z;pH&HUH)5{B3uL8>u*RUqF`p1{LY8ZDeA-#YpcNJ@=U+6Pt|zJErD)mS+I*c#4ohB
zK6L7HROou!Rr#^EvQoN-<9V8=FkPeZ*DjY4XaCJlVfXv%q$gKAszYem#AQR?44=+3
z)w^F@Lz}&<B}TlhFqS|P8X5P>3%}<674UJpG`D1gt11va0l?#tUg9yQQV^6-q~VsP
zWrH(^mu!!4d&UiSJbDk10d{aO+zB!oQk$<ln|FFIA$jt$n7^q#RioRea+HS7HlEg1
zB*fS9oh8-Gyh=;1LpINa`&^AR3mb#JK24m>HTnB2{D=Cv9Q6-A7n(KrKo?<6o*k~o
zx-{GHD~e4wmLZA*M#kRNT(E3DaL^U#BgYk?O4Iu<&B|o(l=Qc|e@LDBsd>4}Y<}!J
z0U~61k6|T_3H5vb3i+e=5*roi_9SGO-ztOwEqiF|$9FV7<$=5J<-tN7f{gomCA}Ro
z%DFyru(KTv7)*paN@%u>6D2nMHW02joXam~Sb}+b!6Q1*Td|(I`CW~v2r&pS%oZ9P
zIz8><pHUn++-E!y-i+~(&{btLq`KrUH+<qfCX`*q2hjqS!hU~$=G%0aVXwf#^mqHb
z|15hA4c9^Y3%M`SV}~nsAh&)hVri0jknJ&q9exaqMMSB&5)!Q=JO0^qRirG#T*dss
zj6q>_5qk|nPmq2VO)Q{HKk=#Z-sbPX-&VZCziv~~ucs>TAVot`^%>3D%YL*xsJ8dc
z^*w`~>`Ls_br6K9@Y(x-W4T9cVw}#Ij~?$FHL~2rQ>r=u^A`flAK(E$F$)CHzI4c>
zzP`H>9?`>Yluhb~8+MSYJ}A4(-9l2bF)2}a+_rKLfg)R23E=6K1UFlo3}B4l$g8--
zU&MI^p)Vt#B00x>soz7AJxq(|Akw@LkYdmJ+7G$<%FIxf7nOU1aL9HkX&Uy6K#BV;
zG?3L&5-<XgUdZSlKXnN=8~EMM_PL*8MfcPF1)IN{wRmkc_(aP^yMyQficP(<ozkW1
zCan>6+<CAg@U%Ui*uBNHGE}KV84p*MO*X7u4<m{a6!F<P5ixt4s+;ifKy6XHrxi@f
zo8hr7FOYdEG%Ic}&NA{BNoi^hn=Al>eD@T1!uSqHM1;j@e1pP*f|%e~*G&7=g>(Yy
z5r6v!Xz^1h_1P5^&#MH+qaTZPRqGhj>s69blJAm?fF&0pC)?HRByPc(?Bxta|525d
zZfPtDx0i&QJPR-5NWd^hE=D`HG~W^7bd1k5xjQRE321%F&fm0;dR4h_-%$2M_PEC-
zV5S}vJVn<?uNXfm_D7LHORFp4)Q~uH=xvXA+tTc*dY}ORRA~OK0K@0j${H|>V}SJ(
z4;xRQM|Abn8!k*git#8R(-b#YIh-e3Tgw_WdpTZWn2}IB?0mVfm~?cik`TFCOnCiW
zScG#SyHh-X@ccR?hH$){?g|ZV0g(nn!n)U`x)Kw7{EMb5k9JgRk{OnLno<<`6OYo&
zPyAf2S^syj%LnO0;F8W%WNgIDbb0Km6r7E?>6bOg5xqi8-ru^~=pn?5bDPwot#FBP
znDJApHO++*2zV1RZL}R+QpW&l*S;?4<73i-u!WEwg!r{s_?*NNo<ADagkDXq_1r~R
zK7PUDweRbTk&<6ZAzH^QEswHGUmy-mElz*!i|KS8I;VRpL_u&!>713mIoETP<uO<8
z+HO*yJI=0<wW#1-sZ;#=zm`r-i?dOd+4xwt{GA&cYph?|M+QWaS?n3)C*lR`q?pP9
zvi%=tS{L$EZ`!LV`A1LZUyv=okM;(9fO%a%-KZDkkkGXxoD%1mn1Gono77St5jFec
zKM^bH*S=<K<Walk%{|n$dL`b&XDc<^bsyXOdj;A->mtqOB01&R{!3+(VxJ;CvDUIJ
zMl)*b_8P&3xEH~fI!Q`snr3x}G5*I3pWTr^=U|hjuStFOgK=}7x#r{k<0=pP5@X|$
zK%dNvv@e$2!=Ly-ddGQ`!Z4flN-=!Mxz$m{_gpTJ8EnqWpK6|hwNBZ_Jbd`|1;9C-
zPiDp)?U0v@cD1$O`LyL~6->+15Qrj$5Ztc}`93HlHvERO>zXYSdzH`#@VLJhTghkb
zo(!)*?5A=O!AkmghKD$NE7?Xt!7!a!sb%r`0NZCsB!z>`R3`rg_V#!~-ner;_L6u>
zJbtwHuEwqd5F4*fpOqk_@V)d@$X~9XzOFw{D7^t7D$bO)nIpm}dr07NQrWV-xR<=v
z4@Sw~*&gx(wFLRct2EMm*}mqB927D?$_`_dspW{x`PfQQ8m7M|<VNV&Yn$v#O$wFS
zPQP#$%d4zpwu_*8)LdhN!g6L>Sd?=@%0Vr=na**X-7dl-2CMXS_g<T%_D^<P=6+EG
zF1&8D2f_g7xW^XhC2XZ!o`{{-t!J!yZ|4Ac?;x@NL*Ykd%cBvPX|fYwo0sab-?W?G
zDV!`z!(1@d<9jKc*>q7bUP~Z`rNsB>7CJuKJmYn3Y=?M?)r9EGb{_=h2msDA=YMP$
zAc2a!N-&f1vH!cvJ-fsIozLfnUgTCK2x2?hwPn*%AiR27rc1nQ_53h{8tKw_dRR*!
zI@Lad<UA2_?p;t5vRiEK-X&jn3xg7MKWp6-(Pl!%a9OmM2<QgfZt)eK6w<e%ig2m&
z_C4PnT!}C@%sj4N#5%p*@{1uz&##zqN`J+z{CJ?xuWX;M7%F#|SKYUkrhh<X7fYPt
zbhdyMeGE)>8*DIm!;DX6D9dib-d;lB*sE*rnj0ATn&=<v^*3!v6UiUfiH1_Q=OVoe
zB$bd>v8ffk;_>RSW;D2qTBFqZdaw;=E$fqt?`FovrIH0BVNnJ`x$c|wrHDiVs0Z|J
zr(RM7{~!%JfBTpO0&Bk$9vNhLW$qGXZ-fVwMgKa+!)B~;LhN;y;o)U_RRhR?d);(4
zs<=V)6lbEG!9saF$>}v-jX%lQVj!=rchM%*q>sQ@c(+r<k?SsfB4v<ZE=u?wd<Dt6
zG#wx;n5p+_eIyU9b9aZM1hs6rNN)?}*T1BSUM0`e_@VQv29QcbSk|Dodve&|^HfBa
zYd&8gQPCeStJvV0xrpz0k2;I&$h}mrjp()Sj9Fa7^c!?Pm@7g_ewC>$A6@#aqURNI
zZTbsW|3JW-L}9&GJ`s?GU$Z~aE~7J{QE3%ng(USQrIp5Rib}!lC-(l`<p8{?n#ey@
z*G0WD+J#zj&;K(Bw{a0kFS1c73VK<4yR}!6UIbQzc^aZmele6Z{C@gTodG3zh<xS}
z9qu$uaZ7~ta#R<U`KATM%4K`vteLE;l&G#n7pX*_($kxGn($_zbCNx!dQ(-zXH*;4
zv*m9jwC#{=F!=zNkN5^9mG9tSUD576OAt9@F-53i#O9IkycN9yfq<2oJY?<NL3mGl
z-3QO85ecm%gX9C(KBB|2#BvR84F^!+t${bb4ZwM=u<y^T2R4(04`78!6g`nS<$D<s
z?5@-t4fvbcec#?skx+duC7pl4ioVh9lkBTaR+S3&4?C9`JzqtgDuJ6gH0lIrx(`%C
za~Xy*O(~7KSC%k0bPPv|P?PS(ZXa=Wt<81oZyrJ=Jxdxs_yub%?I?FsU{yLHMhKVX
zyEwyrO}e-m4(f-yZezJVuMJH|b(8&&0(7E%XM%l7QF5BWoWZ7FS26rGwm)|VZEv`J
z4?o0xQL9R(1ZQa3In}XLj7FN2m=ra+66}S1U;B(gbBh-f=M-~46`~|dpY12u8qMl{
zZ;OOUh>5~(wB#RTbTvdkx9N?2v&GV&vMG6cqSgW7s!rV)lNNjW^?iPxwuc(6M{MrJ
z!E;%Wf%Yw`kqUwd2a^}dxI%v~>kM&xO?zl!KZ;iPZ&xYU6Gi8&*Yw>k&BRvVh=Y%p
z&T4exQ8qERZ!7>58mfjG7Ds+m!3NGU?ap8hSv=Ni=>z^Qcp-8eS5{0QOhraCh4rCt
zc9oVzC%|OOCZW0Xn7&zy;F{oNl%$Bt8E;x7sZ%VsJvzc=;7PT5Hw8ET&{Con+kTrW
z6xTj%+#=rPQQx?3=)9w3I!t;ri7nUBRfnFNju2K!1xE+ea##1rxkyQrk?Jauyh<5a
zs}^40m7(n~Xs1j-hjmS=80PAgS-hcESfnu_9Zjl&@sCrllsTt$+S{upht-C5h5u>7
zp}pDm%3j*kU2TwUy)V*PvHi##>xxauOpUvrZ;Ju&XMzjoX3RJ=YC(?Asm#+TQO9te
zu%zZh->qS;thm%<oPBTY*f)%ME|k01cF{im0bmyMZ+A{LBpy2{#G7DbV`P+jmtGIm
z&u8f{SiWaKA+nSbtL%Zjt*Xb6TRhloAUPITIZQ94Wp*3SMZa^-FZe6p>-lj+<z2nR
zovPP3gx~i!i#R3<_a4CdjwQPTzDl3Il%aPE4<~ppnR=dRs&=m^f{o8*+AZb(Ryd?|
z5lPwT9Fq`BEJRIegzUG#QlNM>zlO$nru_+6#`_R-sgq(&Cu*rGYQ_7dC_2ZFDag%$
z;yg;QX)OF>&lj+enEoLC58^Bfyj6S}(U%nAUdm~2ljWn>A--fW3^$E{O)g!@${SO0
zV5sDpIE=zVUAvNFz^16`5myo)^oO{z<T@a8l|d85t9Qf_niCC5l$fEls|G51{5!9@
zoT+zUhs>P&l?y5PIazV-f|{EWXv*RDuwxq!if6t>HspVLX%ywqofAcSVKg?Iq?Ob#
zy*t)tXUqIUxa48UY?5iHr!0(T^p5+kWg0At)SyQbKG0EPKa|Gv>+I!DQq)K3lmk|c
zzuTL0PmkVGb>1Q@-mcJ2h9SGq+_>|cACniH?{BKNOC0nac(!BU2h(8CG=b#5j3Qku
zLM?}s86hM5HaEV76`CNrW*{GfW*w00N7kbb<rn1lyLMq3AKEihMAjEqZYMYI`N~aX
zLHv@o^Ot%A<2`wGR;l6wC2oP<>VhmCdfa1egV8HXyxR?tk`GiKwl$3PE`{_{sWnnd
zOyvp&Y|g-7z9-+ji{#M9j%qB-k-xQhhE$#Tr0Pq8%Bv_u2O)%378!sPjUh=0VGJ;`
z+wkB$M?1H`IxpZhDKedy(@`r~@8%&tcu8nc#5uOhH-3?|R2`XWpM)(x1W!7%4a?_J
z;a*6nTY>c?pixql(ZlL?X9(h(bxXn#VqMqfUDT^~$6>BmPzZv)6&0x(vlSmH<&lQy
zJYV{}?#j79!2zUvo1vOlZ%)X1MWH=jEwg)8IaFLx*301OL`4H~RMO<5gD2VTZ4diM
zom)FXUp@0}5AL{?iGQOBEGy}pV(hwVr<lt+y=OAp<m|N&jr-`yG-1?~qV&0sT~jHP
z7Vq84WCXISfg9)g`Vlt&aHPQ893rP9ae$^;9^U=hLG{=Sq(fBv@vs58&Pw8O;vT|u
z>gjfxQ@%}Janz=Vuf5uYsJI}wj2OADrzl;fL{fHFk};oBb(Z^@F3wVvdOn42!PG2>
zF%c4y3F%D$Fy73UJHm4v+LJqZb+;OK;;^ns_QM}yNY2TPix1Jt?{=UuM^zgD#b&65
zcZ!5*)i7o{=9Ed+*m<8o)VQ8Ku|`bO4vh-aw)FDyP`7-z^)a&en9QY=&zgP<f<v`4
zy@~kStC4f=Tc5J7m)P{t*5tT0>4rpY9d6B~YfrKCR0AI|<q*=xhJ09wCKdX9yE#%L
z9Bb^~XeE>&28VBRa%~dY;DyMpVLp-tdKqp$S2>byz}F1+?G+OGgDFu>1a;HLvS1^`
zL<{>`=Ua}iio4N0X<&BCc0Khpt)eH1!_T5H&|0Mm)9ANNaawLDfI4<%+88{p#TcVF
zCY^Cz>b1+r{8cHeVrH7&z(XS6>qJ<*^SUSH&zM&rabv=cY=LCGltr)}<>R6UsGV>T
zxDH-3S8-4w0zQ#GoE6*d+jrxVk9vs<K5g@r;+W-dxJYtyA^|<`DBQNxT5d^ci_DMD
ziJAR<Bvnezcv#P)UzkI6^Td3Gn+Ee8Df^nDHtce6NXs?xqw5rgv3jEzZObceG;RAG
zxWu53x@neY#vdMpI2)?{RbwzjH{t>*!KA<X3g_FeUthv&JX}=!G^F?Y1IA|l{)z@I
z%?T9A`}MF*`OqM5Rldtc*pBDx*7?5LYWMjsXORJw?x!e8#See={{k_B4ETMTe%d-B
zn76b;?C`m{f3ySj4q+A)Yv2|ZFh;$iFGYVe^gM8|whdaX(BVsK7*W1-&2(D4tN0EN
zzP};A@5y}z6cr++;0dA?)_3q-nA?JSvqi#9Qa0ac0|$Hp^8~i9E*;80QitmPG^|HF
zV6{D27sP2-`zs{Zm18JW<&F8R%!dt9?_3mgy9{Tg*9B21(r8YHMa;vs9uWyCRD_cz
z<TJ}u{k-1Wb<4U<L94+Vu+EG?h}SSnmD8=5PiOTF?XyGD=A6cU80XUR0n?H-ENOdK
zBwusZq%oCbq>{VHb_u?3ki<1P;Fv|M94Yhz)e-OG#h@xS4KEYsW&?_D22{CkLDvI~
zaK<uOb@#YN4&W?D?+qtKALMp~r@NV{AlXp~?kh6MZeb#b3nvID)%+vmb#9Z>pF)bs
zU0mgw+MNOp2cLJt=GK%%92kfR@h-#l2IrC^R+z!1!8SQ^CgfsukA!BTScaBi4qJ~|
zQ-5_sd;zI>5&wXved-oE!c}-(d6t$?y;d-(HguJh5!rCyfgmjwsW5g5(Xq#P2CN$+
zgKRxAcZpG&ABqmIG^qA(0wQf^N(t!{8&C~<+t$gQQ9DW9+bom7hj{%{1_aH{Uvfiy
zAUY?OdNPSH<h-87!>(DcIg|O=uKi&<8vC~!=Q=zME$`Nw!bfQ}-L7ROugi1!o2mCR
zesg@@?Wq~dLYKcI<6}il5)Mx+R4q{2XWAh~-9}^iznL4g#&?D%R$7otiyKyr(h5^O
zZ}vAuxaYUz8%dF9>?rqT8N*Q$&Kk`^zly?T-qGA6m^{s5TfP0!g0E`*u#8S-q8(!L
zilVvwXx+GI!OlmMP2X)?Vb$miVWuW`vOiacPrrokfKz6?Hlk}u)&jlxrvP_rQ{>0V
z=J&Rh<}6U+5S>T9mRLNmCiV2vzz6oF@jfG3jJcuV6QZu$=;0NONsz&#H?yIZQaoFl
zg%r>!)`7f_=M0!^36SpqcN3xf{&~HV>n_WZPIKfE5tMcQ?4euKn_ailRpUN}VhYe0
z4a*o0CzOnv;Al>L-EX22GV8N6W&7FI<J*>~lzjycvXw@w5Q4CVen6GHD)C>24a#y*
zQ^jM2fyPFbv;rN2Zut)#_-2TqP3?E<s-3cCP_t!Zq!wTOMJ4G6Q5Jco4Tu61f&SU{
zE3o)1+zo&Aju>rX&y;wV?tM9EZ{3+kevcjvdD~%--a1WrI($O<o(cL-c9Mrp@{d7)
zM2Ij<S*8g%oHiZQcT~K(qP*+Z;;5CFF_>chi`m}k+Dyc29G>JCUjHns$7i^JsGNtl
z|FW<RHV*7BN@t|ASZGYU<$D9q!XPAh`Nl3hrz2_nWLf-~5JaPNeEW$A4MLQb;ltTd
z;^vWG?jO5K)#57HxK+@E@oVDlam|-;<5J~MI7J`*`YJygQU1FaX)2i|&!j(6q&I32
zYjgkyz9Z;}3y<C&aM1W(C+`0cl3utH9pPs*vLfd0fn565LV9E-QD*K{TD?f?GoZux
zm(5;}M3`T>ZxNY7<ln|kETxM2ZdFBgKU#%TpdqGc3=l_TH;)_=vzSf5P3pyoiVb-e
z&M3RzPnO%{+C()Js^AKJP@b@i^Vmdbz#8IR`YI7G1ST)yu@y5P8W3{`&BTmG>uTxR
zUlp`hotTaw62sTvsd5KW{b6W|zL2HTk=Y%uZ{<T*i+^Z^g}1POiXHj1ybt1D3+oBW
znkgZ-Mv^VtHd!T;xCWG#OQoCU;Pv{>AdD(@g=_Os6~^~kf&SKVq`>`#9bBw4p7kyZ
zF<v<QPAMPnhA{ic;-0H*f6L@I-+cFMkuggOP72Qhs$OE2W8xw`-o@$~iI#S%ZV*Lr
zt8V(eY#2X;$2Q<yA1qmHP##A?SBtLA8T&Prc?_b@)2dnLY=C@<*UtPdy)(K<FV)8A
zZz6Vuam}cu4BBRuH{Oi3qFf14mN2sDS~^!SYh$)<+=%t_L?+6T&lK&zoz^LCH!D{R
zvWmnV9Wt_px?T$~oJX<a>P%4GRP;;^)-}^7pr5(~8Npof#`*4vx)ZtMgM*=I26yRk
zLoEM3fnt=~PHUvOf3T{hl_!(ix1#BG=7~X7T%e^PLeIj0b;(f1In-dm%(A?&XOr-J
zaue@ed{wKs5S=DPG(Se#Z8Fk`B_S8P0u+&-IBV1GRFhM#e+v0CLuSP;IQU@d^Bas`
z19n-%eb$Y`KXijX0!!?PWk4~|W#%zL{r=1<n~z5XPi&r9MJ=6SL1zw(Z3RHEx@gP!
zVSJZ(7UGR_iH7_@sMVmS>ttK$ZH9c!;*B=wsd$PpL0N7a1}Pe2<};R|cu$7B40fMO
zzNO=kr^D;$VbP)bZsGzTEWKz)#lc|L{|2L_EZWcL*Ew44tq<4lk6vyf+m^;c#0B4U
zFP)=jA1w;aGAOi#r`sivoX+5E`?DNRfK-C2^pJ+{*2#@S6Q_P|?V1%Z$E20uBzyTH
zCaycRit|%Eju=IT0<dGtUp}Pw9)4IaMmx(ni8f$AgPVI#DlGPdona~KMUM*&DE;c!
z6M}t|Pn|FTX|9yH216jWC8Q5Wc3yVDhLzEi6`iQ-2L~lGueM2i^8GEAs6G%dlE^F$
zS%WUH9Gw@HV@n)&?HqRB+VmxEG)@vNpSNrsljXE-i;(G=Y;NLCQ1k0CtqS=QwH)M#
zup(;3hT;cZ4YMc#!iyz|Sdg%VfoS~0n5ryAF@3C6*<;XDq`@DTcI{#bYsg5s7GIKd
zbS5ABl3vRg$SFrH6BphvGBGAp(}3X$eXi8Srl$Z;kP^*!D^))^kkT1mw?mV|(n&6s
zM1)o1WIxbrP0esKD12XC6~;hSk0;g>e!9OCxr2w0VT*)bOH9;~%X0UoyDt(b=o9P@
z@SQF0SORKEr$7=Xx1Cnpd)0x#H0+qkZ+hA&1#LHXY_ywHd1tB@l3ZBu1{AAWgfF6Y
z9C1s~0#BHZiQ^j))gfBZX&C$k*{7m_3P1M1fKxdWLsQ><+ya@E**u#vLts>-R!shY
zR6@Qhc-|e`9@HznYUo^=DJI$&LEel7%B$VGXlF5KVr)t1H^+3pn{o@Ad97PFb_28T
zl~^0wHyZ4phUz!YHEidHs&=6<_LecLAE`%P_rv&_+b-Srme*lY-z=Z6?S#8bQy#az
zcMlq)nXzyhoX3~C=13nuWrh|M1|(to<>-n?6HDUG&OMo|MS^_@&(g==3TFc<?tXaE
z8Qxl9K)z?9Wl3{wRRa_h$M;>U!ia`MSX^ETF7_D2Dv{P_x*R+f?eSYAiO|q4S^r)N
zyc6GLNuQA#DZyEP(6gVeBKjn47Nhn`EqvHH^cgDS?DT2CA#Af^4;a%<g3pkAhqSRZ
z<W0Su*qP9)A?q(#O-2W9wr$b*RQK_h1X}qw&fqay_#swFP*2eLj*8UBsH$$bIkj>=
zTzlK>eP7@t`=ndyPHm)iBeeiOUg(Y}Ih<2xz?WfUQWeuCp+97ht%OC}ese7;xEV0G
zH)A;@<>4=LDv$WZ=k_`2CQA%%j~C8hiXksPNGhc`FE6Vh_sYjrJ<NQ8bvfRB0tdY@
zv2=Uf?Q*{S023DpliL(*8&2ftOk{>M_`IDIaZL^)XtY&ejSc>=yIU}uYIu4rGjcbb
zGyY?N{A0qkECHF0cS}02>l=QlXVSJ2j`o_32mKqL7MbQ+9=d<2+Q`1`9mzXs*oHFa
ze)JB-a5v6qe5G={V&(D*5U`iww}EoNF_!;Tr^~RRIOk?{6t@37n1XgI^al=p9ovKB
zO-^2MzVfL|d`6iIZZZGTK`#6!-1c{j<Nw)!`IE|^-F|6Bv7Gb+Y;6}L_U5#+aDDr?
z&+Hr#tw*kOny^{NqV{TdzRhAMh_La$Y-|m0p5Cvh%zF|Uh*`4o?q3C=S@o}y*e;vN
z9ge#4J1F)EO6q<NAB4v&D6+E!d?@_&$tWuLq2@1ar9~iJ;>g<|WLb2kEt{8^{dtg2
z`yBXqRR)MM=9gVK9B9<K|8vHVk2RYAXcD*A<b}3!n|N{bK{n80x%C!^dfs@t*W$`j
zX8>sN1PqXR<rDNr1NXfKPfUI`uvUFw_{(QEP2qh{w(V<QBRjR`MS;Z=Vx1{5X>p44
zb4c#q=p2&5I&ZNlm?X!6ze|C?n?m%!7z?KNjUe~Nl9q6b%{pY$=fU=G6JT!FySQs#
zdL}d-c!IvS_jSsvP|%$BQ%cm|pEs?zwbz`<Hlp`G%@aFy2qKS>VAz*T=MJOollMJw
z|3Mq>i?mk>Y|8<Ooj3pWf=MpMxYZz(ZTVp5ZMM`NFyYHhCWPE|t~w1)*C)w^K7>i}
zu3tV^{ph~yRRHLoT3}-T_U{+C3%}VGPUow#?c6mv#8!Sf>Q}ZodA-_C!OvE|HuJqb
zpkrP(+;WZWZNc8?aN>mYE5AQsIAo&q!2CdoxzFs%tAE`IC(LS3ZigF<sP4ZG`gyWP
z1Fo1?X<&}*fJA#cqWR7IzFeNuZ1h%P`%KXHjgnuVaC~z;%I2^~^~assYzm`Sa$e5c
zbEM_{s9XmcnR&hz><=<&o&y27;J+ZawZNXV7^ooo#;iTt$HZFg-=Rl?h6@*qQsmr{
z{~cpAbhxoV0<>iZZLI}~TDzt#dBWlL!7|eZAXody9Vr$t=8VQJi5MzI0#h9uo74M<
z3d^BV#jtg*Cmey}&r|H0S1cMR93}Ie%<YOu5Il-q9k<Kc1By6Fy%U4dn&yx<>GMu4
zQ+?;y(%p^#>W}ae&arjp7Cu{f3#I^9+(VWX&*=)b+o$u9|B8HE5{ZvW9q8ZD63%a1
zw2EwRI%A~*H?dNA^J4&P)Ds*-vGnlBCE@k_9D!9@0@r7zpYZfqF4dFdFTw8s&WzP+
zun#D{fc_9$x13?Yk>jh;g4GBP30N)eQdOO1=4i9cryF?2>BDZSSH?Tep4gigJ%in~
z4P@UR@3RBIvq|&TFLX1=HKNyKe{g7vyyonSDQT=&+w6K1fb>?p5kpSDvt1LBMhE$}
z)C-45PNTV!poYB(>MC61p6!;SgBsZ&khx?p7|0sGWE18$xm2-Od2iEvq#(u9AmO1^
zv0!cdl8Dd9sJ3p@R#N0538Ygm$^3n@DT}B)=Q@C&N%-wamEH58Rrrs9-dc2)i;a+h
z!DOhcFn^I4EVRW+)9wQ+fN+BCt4zT-p9sTRJ^ucEi<5`h5U9=z=XBz%e_{Kaykr{Y
z2Z6FaJFOSYtq|yI>)tQTF}5P8eGaC!8}dm=%8u7|>YTGvhvg~cX`R)4&LZ6TQOZm|
zfVJ;%8E6QUPUO2QNeoK2i-~?D<yo4ktbTBL#q??E0ZP;8mc@!u`Hech!vbto9pcq(
zw_sn4Ar6iui}>iRp~yvJezvMn@tha3a({w~&B}e1>`R`=_9<Ho;qsu)1a1Q-9O5Ed
z`!cB6V<$7x#3(eANB_(dN3}H)QdCKdKYbOH*w9+#BBOt^rvD?HcrZj9uIAYK#S-;$
z>aRXKa(tv7i0`xy^Yc5~S9s(x>aV69mI;sGzU#xwey*{%slB~z%R+If5K!LJEyqCf
ztcl;RV7Y_d-{BY<t<eY0<)ZIgj%N;vc)gSmUOZT@DyUy=G_wv}j8k<sEs1`WRiq-Y
z9ns0=bf@t?;+w|xO~$|-;l(&?|L#q7OFKJRcD6c?6JYA21=rw?!&;8NJW(;WKl#!?
z>+(Lfw~hd)un>xd@T(TbZY{5lw^s;0g?EHa7->E{Sig|XHToMmnk;y#Jp6!=&@m{!
zYiq3rW)Fhi^WjuGAk^lGh{_d*YN`7nvkLEZ9j+qADjRjM;5XGLC92R&ZQ7=a#5+^1
zN%{E>uEbeKz9#I~3O_@i*R0b&s}k5hs?6NEwtjfbxmo4Xy2+EoMp2N-zL$aDi+*-d
zl_ULje7jmz1_W~#LiqNxSqxE&WZ0}#`Ae>v0m}-wSw8$2{uL#-kYm^^zdup)4G1~T
z;nF#ZF_L3G%-tN%loJSqd}i9HLv$V0+$<nAO&5eEh6P{i4cKAHE*YcE<Y5uiBQ(<-
zonp=csIJVekoo`t(O7<^7`<A8{5X^lw?-4x2{G6Vq4*;4ju#AQhgtoR+R6%~t8c~|
zoxVqL!OpuE=dB(TVV<+uy(+x}W<fAm9t`md4mDWGn>)QvSlX>a(CK3JpcS9aFV`I_
z5r9*UaHsJ-|CNmzLlJ3HV%a->G16F;``t?_ZRKp2<PozE5*$OY?`EVkW}DZGg?^R3
zCK-5s7~!oB>3tZYXwwiuxw5zT-|lJ5WVtuU%(rr-#EIs0hR>+Rrut96pb<j;pW4nn
zD(P&C<CJBsO1*SV%V&XUWof2jmX9(-ZHbQNBWqGeBhfTU^MNmFobi$kn&w;6lExBD
zElo!Rozi@#S^2;RR6Z~?5eQL0yoXxNU-#ep4{QC_V*P$+?X%DR{l5F`^Vv&Sfk|ZC
zpOO&$yg50@5;XP#P*wwKD*gJXo4K&SwxUCuZoJam9`NXM|9Yv%^MEJ3YOb_`v6KPI
ztKbGx^+lvX_Aax}JRxP-k%zvZzeN>N=*4?7J+u>Y^^r-sLlAIL$!^ogzOBuIUuXG!
z1ZHlgCMgLP^cGu-Pd|64NLTmKp)oHb_mRJ<EY1$ACC=I2sm+UKY!bGl<#Q;Rnn*9-
zqNx`V{*0{13Fa*R%fip*ToSjDyi;GpCibAHy-;%X19LHI02}Gcnly3$7e$H#j8?~m
z$$L3hS7Jd|=1}MgevULP5^>m5lG=}<{*yebVZu?cjn8zx(C>?QXpgb59xQddNcJ-+
zEv=T&EBp(NvKCvoAvaekqt!!V-;A`*+b5T}9=TGkPBNjootv$q1k#%_E>R8S?lU`j
zoFHd6HMB!+G;v4w8K4fYyYC*UKH=*G!0*q=7ZXs?15qWVmpYpHw0D)}_d=L#9rVkk
zIZ!aYeD#C3+7v$iuqjL%`wV)0bTu>2jP=;gu6Loq>0&UtTS=OiVHi7TAB&Jrc_T#a
zrS?Q{lD_fsP!b(LS&5y~wGU=<iPyx&`eziaBG_!Y%CbmYl!IrU^=_&DY39o~F`g{+
z=UHoa+tH6(aUJC{kaZqkxddM^peAdKR7bYNzXAbON~JFvyryjHJe^Y6*178EkImA&
z<$AfY%4;l!wSKqSeZz{S%trT>+llKTLv3JfeW;=#w%mfcLS}E<Y)W!Q#*|fN94x@_
z`kgLv+AS(3F0(L3ITJNGv^Rmatf+dHlJ%Ft6ugI`8$=p549=o@Fv2#Uo+O<Vr3&Xs
z#VVuT{fh@ECEEMdMjnYgi!J`Z0lJG``u-M9xNDzrj7CJAe*e3=Zn`9=F?1+FuRmIX
z1KC7F`>pe_gLiO^)!oXIT&^heR1+pxGzN=I=v(zlPd)6u*D(p($S(=sFE+NfEG(wp
z1Yp#`Aw_H`x{gub4$p_w1iUxhdEP6D>32^aNWBhDIBKOK(8rc1sGEca%!C>(*dDmH
zAH~?s^WCYvu9v;4)q=CRq?3MTOK5V}bICB&`ZwHOb^u~Dr1ecH%H~MWu7bNRIZ=};
zJ?L}8z=O(ZH3kh2v|*9@1vv}exTmynUxo?H$-T3kYXd2ab*`6eJcxIfjQbn=edd=F
z!+`!$S_iRs@Z@|jC{+6V^1DA#1<3=<#>FLKsP3LxRm+~IvZnZi0owGTYdAgh%a}Qb
z6tH3<eRME9PQ2QrDe_|QX$*an(B|2|R05o<!mf>%n}nSsOMt@($2lRnMKFyg>OFVU
zbu>PGB1(Rm*w3@R#}=E%aA?);r5h2UZ+9f#McFuuZl?)?O5Ks~#RH<9_Sl}`t|Xk#
z0!MD^gnTSVhzkbeg)n&Qiqe}@o09N3u;gpd2<t6fif>Qapi^dVrjbzv%E09v>Zy(|
zNv8=*2J4_X(85}1qfy^ou3g!Yu)Ckk(hh<$uX^gw5r%BNH7z5~F8a0J%;-@PB-)Eg
zK2Gw<(4NlzWI+26ulxeVw{U2L9Z#{7t98n2(PEu|4q)TQT7pBbkJpVI;c6CM+=$$(
zpb<1y(``y2H*)Ku6;ad>MA5d*hM;CDNxkda1|vMHckJ3z7NXvn9F<#St#v6nS+a&a
zVH!?*L1QW5p<o~VtKNI&9myhCFd3*R<%w+mFE2{Bn?)R5%vPxuJ*X`TKT$dzhg{lR
zVU^r39G&Dfgk|gdwq)%?&pl`UhOyszILtZuC#3(js0?DLSh*+VDgSmz`6~w8Q1@qV
z&B7_W7RgxRYyGt4s~aG772oHZ*|yJsdT=WwqKo!Atgd@L-<*Z;*3X>U3l$cd%N>xp
z1ZD;2_mwtvYn_#ZZUt^83=^|KlD?Vq=xaMLs3Lf-pXDLnYuj`AJlFx|ZtIBI-(B=}
z@}CjoC*!V+p!GInsBJ9~pRe-Aje1*(Pzy5G6)O=+bUmi6+7V=#%f#3|U5pQX`Y>H2
zni2Ml^YC;JxjoXW)#srQPT5oJd6$7Gl7o!Gyt}A7wfoo=Ql`M6Pr{ZtBBLu4(QTEO
zb@wA8z6bt#7t3>)5=}d+(#I9Y_^-$GvIx?D1Ik7_7xXmptZ$K#k5HnzzNSDMRmrd4
zY8QpqRH@<bENLwT%21+rss7H^$pl8wEWyd<c-gaIZ{r>i^yybdLz>KB#vKRn`ZIL*
z=v4;6;8O=Ia`U_+Cin_sVWyAT%%U4PtV;7^J8!x0#HXNx@wG1)&F{u9hcpq(?Liw>
zsqguQ_BdyNe;B*}#KRd>F+dOxSn@L@q-RB{Qh$BcAAZkzsojGIg9)MF-QbRTe>^7n
z0<ykoYU3rQ^q46nhHuWO=~ABKfBaiagRaJ@&A6O0?S&gZ6eJ%-TRt2jSy6n;lj`Ru
z_zWn%x+J<&G|U^bby01!;3cVQ;9FXD3``}3em7%4#Do?+<Cf9tmpIp^C&~(607KG!
zu>p)JY^{QAVrlx4lB%>eP4pv6E(J$hIz%9RLXHoXks%ysI>^&EJVg7Y%V)6L>&Fa%
z3$)(kv^NfuW@|k^iverLjnt_^KSDJ#*RBwDze#*OzRQ(zhr%w!`%T*3IfE$clZ3A{
z3bqvMNM_Ywg6@3U+IG-$1dp4MNc&1&4YFrAJxa^h@MW2;XJk_1ICRLH?Nnm$-nc`4
zAZRGQgN?f&PhuV46FVf5p7s^2CdODa&JnURqDB4Q4ZR@>9T=D#L2_6R87t>{6~HsW
zu0E7<yQX|Xqdu!OHN@#}Z!PK^H^PHy_WsR>a0b0l|Aq=4NU^CMld=3hEBZ_MF37Jy
zz?YwK16o=oFUNYDm<pqyI0S$Y%<6-l=H9cI6D<7&Hce8e=Rf4%Tj%>FF3+}!t)5Am
zMqAuyZI5{NLtKWZ%)ugm8_~faa0v+j;1Y9wB)k%|L<ROVoj25?o>>}6?!`tn3^%lg
z)dPB#LQ6p)`Co#eymUbM7yA4O@GQ2)Y#lR`gp)t;Ea%7j#?IaYL|+RS?ylrRV%dz0
z$n_t=OVc$`e-8l+*7v*APCzqT3B%Kjheo2S3KYx24*o5oPD!HYxZbrjA`M4c77>f!
zTtbn!yGo0?hHsO?0_T3H;2R9Z7w<<mCRQdTccE5&kV`Ci4sjlRA=c(o2ux3l&rMt}
zIq`#=HU9<KmI70Tii!C{t=9ih?QbbCTF^DK`#Db4UM?fjePdQ}`WpOC#PhKGRzDOq
zC?*9^1_bu(wy4c{cU5Am-zt5xJW%p%@SpIZ0Ri7F<>T{db;$ing?yd01oSTL>m3X`
z!mFB{&5!q`HcZQE_EpIQc|19NVr>INkw8cRnE}Xb3jO49g&}G%;855nP1Up){n@pD
zejMZ_wi=rr2!R9!B!EfZ*3)%Yp}*hQDeIPfpSAEk@JvAV*;KV#;Y(mFv0rOazlrT{
zmyrKO2s^>scKahgQsCD532D9Y`afhb$t-py?$L|I(a0^c|B%PN_UVcxvK_5K%VlLd
xS-C<`M=j_HXqF0N*1n19?~nPv{bJ8}$ku8%nDIM&wvQ>E;^OFrta1qa^}l17L(c#J
literal 0
HcmV?d00001
diff --git a/doc/guides/contributing/stable.rst b/doc/guides/contributing/stable.rst
index 6a5eee9..2b563d4 100644
--- a/doc/guides/contributing/stable.rst
+++ b/doc/guides/contributing/stable.rst
@@ -1,7 +1,7 @@
.. SPDX-License-Identifier: BSD-3-Clause
Copyright 2018 The DPDK contributors
-.. stable_lts_releases:
+.. _stable_lts_releases:
DPDK Stable Releases and Long Term Support
==========================================
@@ -53,6 +53,9 @@ year's November (X.11) release will be maintained as an LTS for 2 years.
After the X.11 release, an LTS branch will be created for it at
http://git.dpdk.org/dpdk-stable where bugfixes will be backported to.
+A LTS release may align with the declaration of a new major ABI version,
+please read the :ref:`abi_policy` for more information.
+
It is anticipated that there will be at least 4 releases per year of the LTS
or approximately 1 every 3 months. However, the cadence can be shorter or
longer depending on the number and criticality of the backported
@@ -119,10 +122,3 @@ A Stable Release will be released by:
list.
Stable releases are available on the `dpdk.org download page <http://core.dpdk.org/download/>`_.
-
-
-ABI
----
-
-The Stable Release should not be seen as a way of breaking or circumventing
-the DPDK ABI policy.
--
2.7.4
^ permalink raw reply [relevance 19%]
* [dpdk-dev] [PATCH v5 4/4] doc: add maintainer for abi policy
2019-09-27 16:30 10% [dpdk-dev] [PATCH v5 0/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
` (2 preceding siblings ...)
2019-09-27 16:30 30% ` [dpdk-dev] [PATCH v5 3/4] doc: updates to versioning guide for " Ray Kinsella
@ 2019-09-27 16:30 13% ` Ray Kinsella
3 siblings, 0 replies; 200+ results
From: Ray Kinsella @ 2019-09-27 16:30 UTC (permalink / raw)
To: dev
Cc: mdr, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, nhorman,
maxime.coquelin, john.mcnamara, marko.kovacevic, hemant.agrawal,
ktraynor
Add an entry to the maintainer file for the abi policy.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
MAINTAINERS | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/MAINTAINERS b/MAINTAINERS
index b3d9aad..d231f03 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -80,6 +80,10 @@ M: Marko Kovacevic <marko.kovacevic@intel.com>
F: README
F: doc/
+ABI Policy
+M: Ray Kinsella <mdr@ashroe.eu>
+F: doc/guides/contributing/abi_*.rst
+
Developers and Maintainers Tools
M: Thomas Monjalon <thomas@monjalon.net>
F: MAINTAINERS
--
2.7.4
^ permalink raw reply [relevance 13%]
* [dpdk-dev] [PATCH v5 3/4] doc: updates to versioning guide for abi versions
2019-09-27 16:30 10% [dpdk-dev] [PATCH v5 0/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
2019-09-27 16:30 13% ` [dpdk-dev] [PATCH v5 1/4] doc: separate versioning.rst into version and policy Ray Kinsella
2019-09-27 16:30 19% ` [dpdk-dev] [PATCH v5 2/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
@ 2019-09-27 16:30 30% ` Ray Kinsella
2019-09-27 16:30 13% ` [dpdk-dev] [PATCH v5 4/4] doc: add maintainer for abi policy Ray Kinsella
3 siblings, 0 replies; 200+ results
From: Ray Kinsella @ 2019-09-27 16:30 UTC (permalink / raw)
To: dev
Cc: mdr, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, nhorman,
maxime.coquelin, john.mcnamara, marko.kovacevic, hemant.agrawal,
ktraynor
Updates to the ABI versioning guide, to account for the changes to the DPDK
ABI/API policy. Fixes for references to abi versioning and policy guides.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
doc/guides/contributing/abi_versioning.rst | 248 +++++++++++++++++++----------
doc/guides/contributing/patches.rst | 6 +-
doc/guides/rel_notes/deprecation.rst | 2 +-
3 files changed, 172 insertions(+), 84 deletions(-)
diff --git a/doc/guides/contributing/abi_versioning.rst b/doc/guides/contributing/abi_versioning.rst
index 53e6ac0..76c63c8 100644
--- a/doc/guides/contributing/abi_versioning.rst
+++ b/doc/guides/contributing/abi_versioning.rst
@@ -1,44 +1,134 @@
.. SPDX-License-Identifier: BSD-3-Clause
Copyright 2018 The DPDK contributors
-.. library_versioning:
+.. _abi_versioning:
-Library versioning
+ABI Versioning
+==============
+
+This document details the mechanics of ABI version management in DPDK.
+
+.. _what_is_soname:
+
+What is a library's soname?
+---------------------------
+
+System libraries usually adopt the familiar major and minor version naming
+convention, where major versions (e.g. ``librte_eal 20.x, 21.x``) are presumed
+to be ABI incompatible with each other and minor versions (e.g. ``librte_eal
+20.1, 20.2``) are presumed to be ABI compatible. A library's `soname
+<https://en.wikipedia.org/wiki/Soname>`_. is typically used to provide backward
+compatibility information about a given library, describing the lowest common
+denominator ABI supported by the library. The soname or logical name for the
+library, is typically comprised of the library's name and major version e.g.
+``librte_eal.so.20``.
+
+During an application's build process, a library's soname is noted as a runtime
+dependency of the application. This information is then used by the `dynamic
+linker <https://en.wikipedia.org/wiki/Dynamic_linker>`_ when resolving the
+applications dependencies at runtime, to load a library supporting the correct
+ABI version. The library loaded at runtime therefore, may be a minor revision
+supporting the same major ABI version (e.g. ``librte_eal.20.2``), as the library
+used to link the application (e.g ``librte_eal.20.0``).
+
+.. _major_abi_versions:
+
+Major ABI versions
------------------
-Downstreams might want to provide different DPDK releases at the same time to
-support multiple consumers of DPDK linked against older and newer sonames.
+An ABI version change to a given library, especially in core libraries such as
+``librte_mbuf``, may cause an implicit ripple effect on the ABI of it's
+consuming libraries, causing ABI breakages. There may however be no explicit
+reason to bump a dependent library's ABI version, as there may have been no
+obvious change to the dependent library's API, even though the library's ABI
+compatibility will have been broken.
+
+This interdependence of DPDK libraries, means that ABI versioning of libraries
+is more manageable at a project level, with all project libraries sharing a
+**single ABI version**. In addition, the need to maintain a stable ABI for some
+number of releases as described in the section :ref:`abi_policy`, means
+that ABI version increments need to carefully planned and managed at a project
+level.
+
+Major ABI versions are therefore declared typically aligned with an LTS release
+and is then supported some number of subsequent releases, shared across all
+libraries. This means that a single project level ABI version, reflected in all
+individual library's soname, library filenames and associated version maps
+persists over multiple releases.
+
+.. code-block:: none
+
+ $ head ./lib/librte_acl/rte_acl_version.map
+ DPDK_20 {
+ global:
+ ...
-Also due to the interdependencies that DPDK libraries can have applications
-might end up with an executable space in which multiple versions of a library
-are mapped by ld.so.
+ $ head ./lib/librte_eal/rte_eal_version.map
+ DPDK_20 {
+ global:
+ ...
-Think of LibA that got an ABI bump and LibB that did not get an ABI bump but is
-depending on LibA.
+When an ABI change is made between major ABI versions to a given library, a new
+section is added to that library's version map describing the impending new ABI
+version, as described in the section :ref:`example_abi_macro_usage`. The
+library's soname and filename however do not change, e.g. ``libacl.so.20``, as
+ABI compatibility with the last major ABI version continues to be preserved for
+that library.
-.. note::
+.. code-block:: none
- Application
- \-> LibA.old
- \-> LibB.new -> LibA.new
+ $ head ./lib/librte_acl/rte_acl_version.map
+ DPDK_20 {
+ global:
+ ...
-That is a conflict which can be avoided by setting ``CONFIG_RTE_MAJOR_ABI``.
-If set, the value of ``CONFIG_RTE_MAJOR_ABI`` overwrites all - otherwise per
-library - versions defined in the libraries ``LIBABIVER``.
-An example might be ``CONFIG_RTE_MAJOR_ABI=16.11`` which will make all libraries
-``librte<?>.so.16.11`` instead of ``librte<?>.so.<LIBABIVER>``.
+ DPDK_21 {
+ global:
+
+ } DPDK_20;
+ ...
+ $ head ./lib/librte_eal/rte_eal_version.map
+ DPDK_20 {
+ global:
+ ...
+
+However when a new ABI version is declared, for example DPDK ``21``, old
+depreciated functions may be safely removed at this point and the entire old
+major ABI version removed, see the section :ref:`deprecating_entire_abi` on
+how this may be done.
+
+.. code-block:: none
+
+ $ head ./lib/librte_acl/rte_acl_version.map
+ DPDK_21 {
+ global:
+ ...
+
+ $ head ./lib/librte_eal/rte_eal_version.map
+ DPDK_21 {
+ global:
+ ...
+
+At the same time, the major ABI version is changed atomically across all
+libraries by incrementing the major version in individual library's soname, e.g.
+``libacl.so.21``. This is done by bumping the LIBABIVER number in the libraries
+Makefile to indicate to dynamic linking applications that this is a later, and
+possibly incompatible library version:
+
+.. code-block:: c
+
+ -LIBABIVER := 20
+ +LIBABIVER := 21
-ABI versioning
---------------
Versioning Macros
-~~~~~~~~~~~~~~~~~
+-----------------
When a symbol is exported from a library to provide an API, it also provides a
calling convention (ABI) that is embodied in its name, return type and
arguments. Occasionally that function may need to change to accommodate new
-functionality or behavior. When that occurs, it is desirable to allow for
+functionality or behavior. When that occurs, it is may be required to allow for
backward compatibility for a time with older binaries that are dynamically
linked to the DPDK.
@@ -61,8 +151,10 @@ The macros exported are:
fully qualified function ``p``, so that if a symbol becomes versioned, it
can still be mapped back to the public symbol name.
+.. _example_abi_macro_usage:
+
Examples of ABI Macro use
-^^^^^^^^^^^^^^^^^^^^^^^^^
+~~~~~~~~~~~~~~~~~~~~~~~~~
Updating a public API
_____________________
@@ -106,16 +198,16 @@ maintain previous ABI versions that are accessible only to previously compiled
binaries
The addition of a parameter to the function is ABI breaking as the function is
-public, and existing application may use it in its current form. However, the
+public, and existing application may use it in its current form. However, the
compatibility macros in DPDK allow a developer to use symbol versioning so that
multiple functions can be mapped to the same public symbol based on when an
-application was linked to it. To see how this is done, we start with the
-requisite libraries version map file. Initially the version map file for the
-acl library looks like this
+application was linked to it. To see how this is done, we start with the
+requisite libraries version map file. Initially the version map file for the acl
+library looks like this
.. code-block:: none
- DPDK_2.0 {
+ DPDK_20 {
global:
rte_acl_add_rules;
@@ -141,7 +233,7 @@ This file needs to be modified as follows
.. code-block:: none
- DPDK_2.0 {
+ DPDK_20 {
global:
rte_acl_add_rules;
@@ -163,16 +255,16 @@ This file needs to be modified as follows
local: *;
};
- DPDK_2.1 {
+ DPDK_21 {
global:
rte_acl_create;
- } DPDK_2.0;
+ } DPDK_20;
The addition of the new block tells the linker that a new version node is
-available (DPDK_2.1), which contains the symbol rte_acl_create, and inherits the
-symbols from the DPDK_2.0 node. This list is directly translated into a list of
-exported symbols when DPDK is compiled as a shared library
+available (DPDK_21), which contains the symbol rte_acl_create, and inherits
+the symbols from the DPDK_20 node. This list is directly translated into a
+list of exported symbols when DPDK is compiled as a shared library
Next, we need to specify in the code which function map to the rte_acl_create
symbol at which versions. First, at the site of the initial symbol definition,
@@ -191,22 +283,22 @@ with the public symbol name
Note that the base name of the symbol was kept intact, as this is conducive to
the macros used for versioning symbols. That is our next step, mapping this new
-symbol name to the initial symbol name at version node 2.0. Immediately after
+symbol name to the initial symbol name at version node 20. Immediately after
the function, we add this line of code
.. code-block:: c
- VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
+ VERSION_SYMBOL(rte_acl_create, _v20, 20);
Remembering to also add the rte_compat.h header to the requisite c file where
-these changes are being made. The above macro instructs the linker to create a
-new symbol ``rte_acl_create@DPDK_2.0``, which matches the symbol created in older
-builds, but now points to the above newly named function. We have now mapped
-the original rte_acl_create symbol to the original function (but with a new
-name)
+these changes are being made. The above macro instructs the linker to create a
+new symbol ``rte_acl_create@DPDK_20``, which matches the symbol created in
+older builds, but now points to the above newly named function. We have now
+mapped the original rte_acl_create symbol to the original function (but with a
+new name)
-Next, we need to create the 2.1 version of the symbol. We create a new function
-name, with a different suffix, and implement it appropriately
+Next, we need to create the 21 version of the symbol. We create a new function
+name, with a different suffix, and implement it appropriately
.. code-block:: c
@@ -220,12 +312,12 @@ name, with a different suffix, and implement it appropriately
return ctx;
}
-This code serves as our new API call. Its the same as our old call, but adds
-the new parameter in place. Next we need to map this function to the symbol
-``rte_acl_create@DPDK_2.1``. To do this, we modify the public prototype of the call
-in the header file, adding the macro there to inform all including applications,
-that on re-link, the default rte_acl_create symbol should point to this
-function. Note that we could do this by simply naming the function above
+This code serves as our new API call. Its the same as our old call, but adds the
+new parameter in place. Next we need to map this function to the symbol
+``rte_acl_create@DPDK_21``. To do this, we modify the public prototype of the
+call in the header file, adding the macro there to inform all including
+applications, that on re-link, the default rte_acl_create symbol should point to
+this function. Note that we could do this by simply naming the function above
rte_acl_create, and the linker would chose the most recent version tag to apply
in the version script, but we can also do this in the header file
@@ -233,11 +325,11 @@ in the version script, but we can also do this in the header file
struct rte_acl_ctx *
-rte_acl_create(const struct rte_acl_param *param);
- +rte_acl_create(const struct rte_acl_param *param, int debug);
- +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
+ +rte_acl_create_v21(const struct rte_acl_param *param, int debug);
+ +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 21);
The BIND_DEFAULT_SYMBOL macro explicitly tells applications that include this
-header, to link to the rte_acl_create_v21 function and apply the DPDK_2.1
+header, to link to the rte_acl_create_v21 function and apply the DPDK_21
version node to it. This method is more explicit and flexible than just
re-implementing the exact symbol name, and allows for other features (such as
linking to the old symbol version by default, when the new ABI is to be opt-in
@@ -257,6 +349,7 @@ assumption is that the most recent version of the symbol is the one you want to
map. So, back in the C file where, immediately after ``rte_acl_create_v21`` is
defined, we add this
+
.. code-block:: c
struct rte_acl_ctx *
@@ -270,21 +363,22 @@ That tells the compiler that, when building a static library, any calls to the
symbol ``rte_acl_create`` should be linked to ``rte_acl_create_v21``
That's it, on the next shared library rebuild, there will be two versions of
-rte_acl_create, an old DPDK_2.0 version, used by previously built applications,
-and a new DPDK_2.1 version, used by future built applications.
+rte_acl_create, an old DPDK_20 version, used by previously built applications,
+and a new DPDK_21 version, used by future built applications.
Deprecating part of a public API
________________________________
-Lets assume that you've done the above update, and after a few releases have
-passed you decide you would like to retire the old version of the function.
-After having gone through the ABI deprecation announcement process, removal is
-easy. Start by removing the symbol from the requisite version map file:
+Lets assume that you've done the above update, and in preparation for the next
+major ABI version you decide you would like to retire the old version of the
+function. After having gone through the ABI deprecation announcement process,
+removal is easy. Start by removing the symbol from the requisite version map
+file:
.. code-block:: none
- DPDK_2.0 {
+ DPDK_20 {
global:
rte_acl_add_rules;
@@ -306,48 +400,42 @@ easy. Start by removing the symbol from the requisite version map file:
local: *;
};
- DPDK_2.1 {
+ DPDK_21 {
global:
rte_acl_create;
- } DPDK_2.0;
+ } DPDK_20;
Next remove the corresponding versioned export.
.. code-block:: c
- -VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
+ -VERSION_SYMBOL(rte_acl_create, _v20, 20);
Note that the internal function definition could also be removed, but its used
-in our example by the newer version _v21, so we leave it in place. This is a
-coding style choice.
-
-Lastly, we need to bump the LIBABIVER number for this library in the Makefile to
-indicate to applications doing dynamic linking that this is a later, and
-possibly incompatible library version:
-
-.. code-block:: c
+in our example by the newer version v21, so we leave it in place and declare it
+as static. This is a coding style choice.
- -LIBABIVER := 1
- +LIBABIVER := 2
+.. _deprecating_entire_abi:
Deprecating an entire ABI version
_________________________________
-While removing a symbol from and ABI may be useful, it is often more practical
-to remove an entire version node at once. If a version node completely
-specifies an API, then removing part of it, typically makes it incomplete. In
-those cases it is better to remove the entire node
+While removing a symbol from an ABI may be useful, it is more practical to
+remove an entire version node at once, as is typically done at the declaration
+of a major ABI version. If a version node completely specifies an API, then
+removing part of it, typically makes it incomplete. In those cases it is better
+to remove the entire node.
To do this, start by modifying the version map file, such that all symbols from
-the node to be removed are merged into the next node in the map
+the node to be removed are merged into the next node in the map.
In the case of our map above, it would transform to look as follows
.. code-block:: none
- DPDK_2.1 {
+ DPDK_21 {
global:
rte_acl_add_rules;
@@ -375,8 +463,8 @@ symbols.
.. code-block:: c
- -BIND_DEFAULT_SYMBOL(rte_acl_create, _v20, 2.0);
- +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
+ -BIND_DEFAULT_SYMBOL(rte_acl_create, _v20, 20);
+ +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 21);
Lastly, any VERSION_SYMBOL macros that point to the old version node should be
removed, taking care to keep, where need old code in place to support newer
diff --git a/doc/guides/contributing/patches.rst b/doc/guides/contributing/patches.rst
index 9e1013b..d63c9f5 100644
--- a/doc/guides/contributing/patches.rst
+++ b/doc/guides/contributing/patches.rst
@@ -156,9 +156,9 @@ Make your planned changes in the cloned ``dpdk`` repo. Here are some guidelines
* For other PMDs and more info, refer to the ``MAINTAINERS`` file.
-* New external functions should be added to the local ``version.map`` file.
- See the :doc:`Guidelines for ABI policy and versioning </contributing/versioning>`.
- New external functions should also be added in alphabetical order.
+* New external functions should be added to the local ``version.map`` file. See
+ the :ref:`ABI policy <abi_policy>` and :ref:`ABI versioning <abi_versioning>`
+ guides. New external functions should also be added in alphabetical order.
* Important changes will require an addition to the release notes in ``doc/guides/rel_notes/``.
See the :ref:`Release Notes section of the Documentation Guidelines <doc_guidelines>` for details.
diff --git a/doc/guides/rel_notes/deprecation.rst b/doc/guides/rel_notes/deprecation.rst
index 0ee8533..2e163a5 100644
--- a/doc/guides/rel_notes/deprecation.rst
+++ b/doc/guides/rel_notes/deprecation.rst
@@ -4,7 +4,7 @@
ABI and API Deprecation
=======================
-See the :doc:`guidelines document for details of the ABI policy </contributing/versioning>`.
+See the :ref:`guidelines document for details of the ABI policy <abi_policy>`.
API and ABI deprecation notices are to be posted here.
--
2.7.4
^ permalink raw reply [relevance 30%]
* [dpdk-dev] [PATCH v5 1/4] doc: separate versioning.rst into version and policy
2019-09-27 16:30 10% [dpdk-dev] [PATCH v5 0/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
@ 2019-09-27 16:30 13% ` Ray Kinsella
2019-09-27 16:30 19% ` [dpdk-dev] [PATCH v5 2/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
` (2 subsequent siblings)
3 siblings, 0 replies; 200+ results
From: Ray Kinsella @ 2019-09-27 16:30 UTC (permalink / raw)
To: dev
Cc: mdr, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, nhorman,
maxime.coquelin, john.mcnamara, marko.kovacevic, hemant.agrawal,
ktraynor
Separate versioning.rst into abi versioning and abi policy guidance, in
preparation for adding more detail to the abi policy.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
doc/guides/contributing/abi_policy.rst | 169 +++++++++
doc/guides/contributing/abi_versioning.rst | 427 +++++++++++++++++++++
doc/guides/contributing/index.rst | 3 +-
doc/guides/contributing/versioning.rst | 591 -----------------------------
4 files changed, 598 insertions(+), 592 deletions(-)
create mode 100644 doc/guides/contributing/abi_policy.rst
create mode 100644 doc/guides/contributing/abi_versioning.rst
delete mode 100644 doc/guides/contributing/versioning.rst
diff --git a/doc/guides/contributing/abi_policy.rst b/doc/guides/contributing/abi_policy.rst
new file mode 100644
index 0000000..55bacb4
--- /dev/null
+++ b/doc/guides/contributing/abi_policy.rst
@@ -0,0 +1,169 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+ Copyright 2018 The DPDK contributors
+
+.. abi_api_policy:
+
+DPDK ABI/API policy
+===================
+
+Description
+-----------
+
+This document details some methods for handling ABI management in the DPDK.
+
+General Guidelines
+------------------
+
+#. Whenever possible, ABI should be preserved
+#. ABI/API may be changed with a deprecation process
+#. The modification of symbols can generally be managed with versioning
+#. Libraries or APIs marked in ``experimental`` state may change without constraint
+#. New APIs will be marked as ``experimental`` for at least one release to allow
+ any issues found by users of the new API to be fixed quickly
+#. The addition of symbols is generally not problematic
+#. The removal of symbols generally is an ABI break and requires bumping of the
+ LIBABIVER macro
+#. Updates to the minimum hardware requirements, which drop support for hardware which
+ was previously supported, should be treated as an ABI change.
+
+What is an ABI
+~~~~~~~~~~~~~~
+
+An ABI (Application Binary Interface) is the set of runtime interfaces exposed
+by a library. It is similar to an API (Application Programming Interface) but
+is the result of compilation. It is also effectively cloned when applications
+link to dynamic libraries. That is to say when an application is compiled to
+link against dynamic libraries, it is assumed that the ABI remains constant
+between the time the application is compiled/linked, and the time that it runs.
+Therefore, in the case of dynamic linking, it is critical that an ABI is
+preserved, or (when modified), done in such a way that the application is unable
+to behave improperly or in an unexpected fashion.
+
+
+ABI/API Deprecation
+-------------------
+
+The DPDK ABI policy
+~~~~~~~~~~~~~~~~~~~
+
+ABI versions are set at the time of major release labeling, and the ABI may
+change multiple times, without warning, between the last release label and the
+HEAD label of the git tree.
+
+ABI versions, once released, are available until such time as their
+deprecation has been noted in the Release Notes for at least one major release
+cycle. For example consider the case where the ABI for DPDK 2.0 has been
+shipped and then a decision is made to modify it during the development of
+DPDK 2.1. The decision will be recorded in the Release Notes for the DPDK 2.1
+release and the modification will be made available in the DPDK 2.2 release.
+
+ABI versions may be deprecated in whole or in part as needed by a given
+update.
+
+Some ABI changes may be too significant to reasonably maintain multiple
+versions. In those cases ABI's may be updated without backward compatibility
+being provided. The requirements for doing so are:
+
+#. At least 3 acknowledgments of the need to do so must be made on the
+ dpdk.org mailing list.
+
+ - The acknowledgment of the maintainer of the component is mandatory, or if
+ no maintainer is available for the component, the tree/sub-tree maintainer
+ for that component must acknowledge the ABI change instead.
+
+ - It is also recommended that acknowledgments from different "areas of
+ interest" be sought for each deprecation, for example: from NIC vendors,
+ CPU vendors, end-users, etc.
+
+#. The changes (including an alternative map file) can be included with
+ deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option,
+ to provide more details about oncoming changes.
+ ``RTE_NEXT_ABI`` wrapper will be removed when it become the default ABI.
+ More preferred way to provide this information is sending the feature
+ as a separate patch and reference it in deprecation notice.
+
+#. A full deprecation cycle, as explained above, must be made to offer
+ downstream consumers sufficient warning of the change.
+
+Note that the above process for ABI deprecation should not be undertaken
+lightly. ABI stability is extremely important for downstream consumers of the
+DPDK, especially when distributed in shared object form. Every effort should
+be made to preserve the ABI whenever possible. The ABI should only be changed
+for significant reasons, such as performance enhancements. ABI breakage due to
+changes such as reorganizing public structure fields for aesthetic or
+readability purposes should be avoided.
+
+.. note::
+
+ Updates to the minimum hardware requirements, which drop support for hardware
+ which was previously supported, should be treated as an ABI change, and
+ follow the relevant deprecation policy procedures as above: 3 acks and
+ announcement at least one release in advance.
+
+Examples of Deprecation Notices
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The following are some examples of ABI deprecation notices which would be
+added to the Release Notes:
+
+* The Macro ``#RTE_FOO`` is deprecated and will be removed with version 2.0,
+ to be replaced with the inline function ``rte_foo()``.
+
+* The function ``rte_mbuf_grok()`` has been updated to include a new parameter
+ in version 2.0. Backwards compatibility will be maintained for this function
+ until the release of version 2.1
+
+* The members of ``struct rte_foo`` have been reorganized in release 2.0 for
+ performance reasons. Existing binary applications will have backwards
+ compatibility in release 2.0, while newly built binaries will need to
+ reference the new structure variant ``struct rte_foo2``. Compatibility will
+ be removed in release 2.2, and all applications will require updating and
+ rebuilding to the new structure at that time, which will be renamed to the
+ original ``struct rte_foo``.
+
+* Significant ABI changes are planned for the ``librte_dostuff`` library. The
+ upcoming release 2.0 will not contain these changes, but release 2.1 will,
+ and no backwards compatibility is planned due to the extensive nature of
+ these changes. Binaries using this library built prior to version 2.1 will
+ require updating and recompilation.
+
+New API replacing previous one
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If a new API proposed functionally replaces an existing one, when the new API
+becomes non-experimental then the old one is marked with ``__rte_deprecated``.
+Deprecated APIs are removed completely just after the next LTS.
+
+Reminder that old API should follow deprecation process to be removed.
+
+
+Experimental APIs
+-----------------
+
+APIs marked as ``experimental`` are not considered part of the ABI and may
+change without warning at any time. Since changes to APIs are most likely
+immediately after their introduction, as users begin to take advantage of
+those new APIs and start finding issues with them, new DPDK APIs will be
+automatically marked as ``experimental`` to allow for a period of stabilization
+before they become part of a tracked ABI.
+
+Note that marking an API as experimental is a multi step process.
+To mark an API as experimental, the symbols which are desired to be exported
+must be placed in an EXPERIMENTAL version block in the corresponding libraries'
+version map script.
+Secondly, the corresponding prototypes of those exported functions (in the
+development header files), must be marked with the ``__rte_experimental`` tag
+(see ``rte_compat.h``).
+The DPDK build makefiles perform a check to ensure that the map file and the
+C code reflect the same list of symbols.
+This check can be circumvented by defining ``ALLOW_EXPERIMENTAL_API``
+during compilation in the corresponding library Makefile.
+
+In addition to tagging the code with ``__rte_experimental``,
+the doxygen markup must also contain the EXPERIMENTAL string,
+and the MAINTAINERS file should note the EXPERIMENTAL libraries.
+
+For removing the experimental tag associated with an API, deprecation notice
+is not required. Though, an API should remain in experimental state for at least
+one release. Thereafter, normal process of posting patch for review to mailing
+list can be followed.
diff --git a/doc/guides/contributing/abi_versioning.rst b/doc/guides/contributing/abi_versioning.rst
new file mode 100644
index 0000000..53e6ac0
--- /dev/null
+++ b/doc/guides/contributing/abi_versioning.rst
@@ -0,0 +1,427 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+ Copyright 2018 The DPDK contributors
+
+.. library_versioning:
+
+Library versioning
+------------------
+
+Downstreams might want to provide different DPDK releases at the same time to
+support multiple consumers of DPDK linked against older and newer sonames.
+
+Also due to the interdependencies that DPDK libraries can have applications
+might end up with an executable space in which multiple versions of a library
+are mapped by ld.so.
+
+Think of LibA that got an ABI bump and LibB that did not get an ABI bump but is
+depending on LibA.
+
+.. note::
+
+ Application
+ \-> LibA.old
+ \-> LibB.new -> LibA.new
+
+That is a conflict which can be avoided by setting ``CONFIG_RTE_MAJOR_ABI``.
+If set, the value of ``CONFIG_RTE_MAJOR_ABI`` overwrites all - otherwise per
+library - versions defined in the libraries ``LIBABIVER``.
+An example might be ``CONFIG_RTE_MAJOR_ABI=16.11`` which will make all libraries
+``librte<?>.so.16.11`` instead of ``librte<?>.so.<LIBABIVER>``.
+
+
+ABI versioning
+--------------
+
+Versioning Macros
+~~~~~~~~~~~~~~~~~
+
+When a symbol is exported from a library to provide an API, it also provides a
+calling convention (ABI) that is embodied in its name, return type and
+arguments. Occasionally that function may need to change to accommodate new
+functionality or behavior. When that occurs, it is desirable to allow for
+backward compatibility for a time with older binaries that are dynamically
+linked to the DPDK.
+
+To support backward compatibility the ``rte_compat.h``
+header file provides macros to use when updating exported functions. These
+macros are used in conjunction with the ``rte_<library>_version.map`` file for
+a given library to allow multiple versions of a symbol to exist in a shared
+library so that older binaries need not be immediately recompiled.
+
+The macros exported are:
+
+* ``VERSION_SYMBOL(b, e, n)``: Creates a symbol version table entry binding
+ versioned symbol ``b@DPDK_n`` to the internal function ``b_e``.
+
+* ``BIND_DEFAULT_SYMBOL(b, e, n)``: Creates a symbol version entry instructing
+ the linker to bind references to symbol ``b`` to the internal symbol
+ ``b_e``.
+
+* ``MAP_STATIC_SYMBOL(f, p)``: Declare the prototype ``f``, and map it to the
+ fully qualified function ``p``, so that if a symbol becomes versioned, it
+ can still be mapped back to the public symbol name.
+
+Examples of ABI Macro use
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Updating a public API
+_____________________
+
+Assume we have a function as follows
+
+.. code-block:: c
+
+ /*
+ * Create an acl context object for apps to
+ * manipulate
+ */
+ struct rte_acl_ctx *
+ rte_acl_create(const struct rte_acl_param *param)
+ {
+ ...
+ }
+
+
+Assume that struct rte_acl_ctx is a private structure, and that a developer
+wishes to enhance the acl api so that a debugging flag can be enabled on a
+per-context basis. This requires an addition to the structure (which, being
+private, is safe), but it also requires modifying the code as follows
+
+.. code-block:: c
+
+ /*
+ * Create an acl context object for apps to
+ * manipulate
+ */
+ struct rte_acl_ctx *
+ rte_acl_create(const struct rte_acl_param *param, int debug)
+ {
+ ...
+ }
+
+
+Note also that, being a public function, the header file prototype must also be
+changed, as must all the call sites, to reflect the new ABI footprint. We will
+maintain previous ABI versions that are accessible only to previously compiled
+binaries
+
+The addition of a parameter to the function is ABI breaking as the function is
+public, and existing application may use it in its current form. However, the
+compatibility macros in DPDK allow a developer to use symbol versioning so that
+multiple functions can be mapped to the same public symbol based on when an
+application was linked to it. To see how this is done, we start with the
+requisite libraries version map file. Initially the version map file for the
+acl library looks like this
+
+.. code-block:: none
+
+ DPDK_2.0 {
+ global:
+
+ rte_acl_add_rules;
+ rte_acl_build;
+ rte_acl_classify;
+ rte_acl_classify_alg;
+ rte_acl_classify_scalar;
+ rte_acl_create;
+ rte_acl_dump;
+ rte_acl_find_existing;
+ rte_acl_free;
+ rte_acl_ipv4vlan_add_rules;
+ rte_acl_ipv4vlan_build;
+ rte_acl_list_dump;
+ rte_acl_reset;
+ rte_acl_reset_rules;
+ rte_acl_set_ctx_classify;
+
+ local: *;
+ };
+
+This file needs to be modified as follows
+
+.. code-block:: none
+
+ DPDK_2.0 {
+ global:
+
+ rte_acl_add_rules;
+ rte_acl_build;
+ rte_acl_classify;
+ rte_acl_classify_alg;
+ rte_acl_classify_scalar;
+ rte_acl_create;
+ rte_acl_dump;
+ rte_acl_find_existing;
+ rte_acl_free;
+ rte_acl_ipv4vlan_add_rules;
+ rte_acl_ipv4vlan_build;
+ rte_acl_list_dump;
+ rte_acl_reset;
+ rte_acl_reset_rules;
+ rte_acl_set_ctx_classify;
+
+ local: *;
+ };
+
+ DPDK_2.1 {
+ global:
+ rte_acl_create;
+
+ } DPDK_2.0;
+
+The addition of the new block tells the linker that a new version node is
+available (DPDK_2.1), which contains the symbol rte_acl_create, and inherits the
+symbols from the DPDK_2.0 node. This list is directly translated into a list of
+exported symbols when DPDK is compiled as a shared library
+
+Next, we need to specify in the code which function map to the rte_acl_create
+symbol at which versions. First, at the site of the initial symbol definition,
+we need to update the function so that it is uniquely named, and not in conflict
+with the public symbol name
+
+.. code-block:: c
+
+ struct rte_acl_ctx *
+ -rte_acl_create(const struct rte_acl_param *param)
+ +rte_acl_create_v20(const struct rte_acl_param *param)
+ {
+ size_t sz;
+ struct rte_acl_ctx *ctx;
+ ...
+
+Note that the base name of the symbol was kept intact, as this is conducive to
+the macros used for versioning symbols. That is our next step, mapping this new
+symbol name to the initial symbol name at version node 2.0. Immediately after
+the function, we add this line of code
+
+.. code-block:: c
+
+ VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
+
+Remembering to also add the rte_compat.h header to the requisite c file where
+these changes are being made. The above macro instructs the linker to create a
+new symbol ``rte_acl_create@DPDK_2.0``, which matches the symbol created in older
+builds, but now points to the above newly named function. We have now mapped
+the original rte_acl_create symbol to the original function (but with a new
+name)
+
+Next, we need to create the 2.1 version of the symbol. We create a new function
+name, with a different suffix, and implement it appropriately
+
+.. code-block:: c
+
+ struct rte_acl_ctx *
+ rte_acl_create_v21(const struct rte_acl_param *param, int debug);
+ {
+ struct rte_acl_ctx *ctx = rte_acl_create_v20(param);
+
+ ctx->debug = debug;
+
+ return ctx;
+ }
+
+This code serves as our new API call. Its the same as our old call, but adds
+the new parameter in place. Next we need to map this function to the symbol
+``rte_acl_create@DPDK_2.1``. To do this, we modify the public prototype of the call
+in the header file, adding the macro there to inform all including applications,
+that on re-link, the default rte_acl_create symbol should point to this
+function. Note that we could do this by simply naming the function above
+rte_acl_create, and the linker would chose the most recent version tag to apply
+in the version script, but we can also do this in the header file
+
+.. code-block:: c
+
+ struct rte_acl_ctx *
+ -rte_acl_create(const struct rte_acl_param *param);
+ +rte_acl_create(const struct rte_acl_param *param, int debug);
+ +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
+
+The BIND_DEFAULT_SYMBOL macro explicitly tells applications that include this
+header, to link to the rte_acl_create_v21 function and apply the DPDK_2.1
+version node to it. This method is more explicit and flexible than just
+re-implementing the exact symbol name, and allows for other features (such as
+linking to the old symbol version by default, when the new ABI is to be opt-in
+for a period.
+
+One last thing we need to do. Note that we've taken what was a public symbol,
+and duplicated it into two uniquely and differently named symbols. We've then
+mapped each of those back to the public symbol ``rte_acl_create`` with different
+version tags. This only applies to dynamic linking, as static linking has no
+notion of versioning. That leaves this code in a position of no longer having a
+symbol simply named ``rte_acl_create`` and a static build will fail on that
+missing symbol.
+
+To correct this, we can simply map a function of our choosing back to the public
+symbol in the static build with the ``MAP_STATIC_SYMBOL`` macro. Generally the
+assumption is that the most recent version of the symbol is the one you want to
+map. So, back in the C file where, immediately after ``rte_acl_create_v21`` is
+defined, we add this
+
+.. code-block:: c
+
+ struct rte_acl_ctx *
+ rte_acl_create_v21(const struct rte_acl_param *param, int debug)
+ {
+ ...
+ }
+ MAP_STATIC_SYMBOL(struct rte_acl_ctx *rte_acl_create(const struct rte_acl_param *param, int debug), rte_acl_create_v21);
+
+That tells the compiler that, when building a static library, any calls to the
+symbol ``rte_acl_create`` should be linked to ``rte_acl_create_v21``
+
+That's it, on the next shared library rebuild, there will be two versions of
+rte_acl_create, an old DPDK_2.0 version, used by previously built applications,
+and a new DPDK_2.1 version, used by future built applications.
+
+
+Deprecating part of a public API
+________________________________
+
+Lets assume that you've done the above update, and after a few releases have
+passed you decide you would like to retire the old version of the function.
+After having gone through the ABI deprecation announcement process, removal is
+easy. Start by removing the symbol from the requisite version map file:
+
+.. code-block:: none
+
+ DPDK_2.0 {
+ global:
+
+ rte_acl_add_rules;
+ rte_acl_build;
+ rte_acl_classify;
+ rte_acl_classify_alg;
+ rte_acl_classify_scalar;
+ rte_acl_dump;
+ - rte_acl_create
+ rte_acl_find_existing;
+ rte_acl_free;
+ rte_acl_ipv4vlan_add_rules;
+ rte_acl_ipv4vlan_build;
+ rte_acl_list_dump;
+ rte_acl_reset;
+ rte_acl_reset_rules;
+ rte_acl_set_ctx_classify;
+
+ local: *;
+ };
+
+ DPDK_2.1 {
+ global:
+ rte_acl_create;
+ } DPDK_2.0;
+
+
+Next remove the corresponding versioned export.
+
+.. code-block:: c
+
+ -VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
+
+
+Note that the internal function definition could also be removed, but its used
+in our example by the newer version _v21, so we leave it in place. This is a
+coding style choice.
+
+Lastly, we need to bump the LIBABIVER number for this library in the Makefile to
+indicate to applications doing dynamic linking that this is a later, and
+possibly incompatible library version:
+
+.. code-block:: c
+
+ -LIBABIVER := 1
+ +LIBABIVER := 2
+
+Deprecating an entire ABI version
+_________________________________
+
+While removing a symbol from and ABI may be useful, it is often more practical
+to remove an entire version node at once. If a version node completely
+specifies an API, then removing part of it, typically makes it incomplete. In
+those cases it is better to remove the entire node
+
+To do this, start by modifying the version map file, such that all symbols from
+the node to be removed are merged into the next node in the map
+
+In the case of our map above, it would transform to look as follows
+
+.. code-block:: none
+
+ DPDK_2.1 {
+ global:
+
+ rte_acl_add_rules;
+ rte_acl_build;
+ rte_acl_classify;
+ rte_acl_classify_alg;
+ rte_acl_classify_scalar;
+ rte_acl_dump;
+ rte_acl_create
+ rte_acl_find_existing;
+ rte_acl_free;
+ rte_acl_ipv4vlan_add_rules;
+ rte_acl_ipv4vlan_build;
+ rte_acl_list_dump;
+ rte_acl_reset;
+ rte_acl_reset_rules;
+ rte_acl_set_ctx_classify;
+
+ local: *;
+ };
+
+Then any uses of BIND_DEFAULT_SYMBOL that pointed to the old node should be
+updated to point to the new version node in any header files for all affected
+symbols.
+
+.. code-block:: c
+
+ -BIND_DEFAULT_SYMBOL(rte_acl_create, _v20, 2.0);
+ +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
+
+Lastly, any VERSION_SYMBOL macros that point to the old version node should be
+removed, taking care to keep, where need old code in place to support newer
+versions of the symbol.
+
+
+Running the ABI Validator
+-------------------------
+
+The ``devtools`` directory in the DPDK source tree contains a utility program,
+``validate-abi.sh``, for validating the DPDK ABI based on the Linux `ABI
+Compliance Checker
+<http://ispras.linuxbase.org/index.php/ABI_compliance_checker>`_.
+
+This has a dependency on the ``abi-compliance-checker`` and ``and abi-dumper``
+utilities which can be installed via a package manager. For example::
+
+ sudo yum install abi-compliance-checker
+ sudo yum install abi-dumper
+
+The syntax of the ``validate-abi.sh`` utility is::
+
+ ./devtools/validate-abi.sh <REV1> <REV2>
+
+Where ``REV1`` and ``REV2`` are valid gitrevisions(7)
+https://www.kernel.org/pub/software/scm/git/docs/gitrevisions.html
+on the local repo.
+
+For example::
+
+ # Check between the previous and latest commit:
+ ./devtools/validate-abi.sh HEAD~1 HEAD
+
+ # Check on a specific compilation target:
+ ./devtools/validate-abi.sh -t x86_64-native-linux-gcc HEAD~1 HEAD
+
+ # Check between two tags:
+ ./devtools/validate-abi.sh v2.0.0 v2.1.0
+
+ # Check between git master and local topic-branch "vhost-hacking":
+ ./devtools/validate-abi.sh master vhost-hacking
+
+After the validation script completes (it can take a while since it need to
+compile both tags) it will create compatibility reports in the
+``./abi-check/compat_report`` directory. Listed incompatibilities can be found
+as follows::
+
+ grep -lr Incompatible abi-check/compat_reports/
diff --git a/doc/guides/contributing/index.rst b/doc/guides/contributing/index.rst
index e2608d3..2fefd91 100644
--- a/doc/guides/contributing/index.rst
+++ b/doc/guides/contributing/index.rst
@@ -10,7 +10,8 @@ Contributor's Guidelines
coding_style
design
- versioning
+ abi_policy
+ abi_versioning
documentation
patches
vulnerability
diff --git a/doc/guides/contributing/versioning.rst b/doc/guides/contributing/versioning.rst
deleted file mode 100644
index 3ab2c43..0000000
--- a/doc/guides/contributing/versioning.rst
+++ /dev/null
@@ -1,591 +0,0 @@
-.. SPDX-License-Identifier: BSD-3-Clause
- Copyright 2018 The DPDK contributors
-
-DPDK ABI/API policy
-===================
-
-Description
------------
-
-This document details some methods for handling ABI management in the DPDK.
-
-General Guidelines
-------------------
-
-#. Whenever possible, ABI should be preserved
-#. ABI/API may be changed with a deprecation process
-#. The modification of symbols can generally be managed with versioning
-#. Libraries or APIs marked in ``experimental`` state may change without constraint
-#. New APIs will be marked as ``experimental`` for at least one release to allow
- any issues found by users of the new API to be fixed quickly
-#. The addition of symbols is generally not problematic
-#. The removal of symbols generally is an ABI break and requires bumping of the
- LIBABIVER macro
-#. Updates to the minimum hardware requirements, which drop support for hardware which
- was previously supported, should be treated as an ABI change.
-
-What is an ABI
-~~~~~~~~~~~~~~
-
-An ABI (Application Binary Interface) is the set of runtime interfaces exposed
-by a library. It is similar to an API (Application Programming Interface) but
-is the result of compilation. It is also effectively cloned when applications
-link to dynamic libraries. That is to say when an application is compiled to
-link against dynamic libraries, it is assumed that the ABI remains constant
-between the time the application is compiled/linked, and the time that it runs.
-Therefore, in the case of dynamic linking, it is critical that an ABI is
-preserved, or (when modified), done in such a way that the application is unable
-to behave improperly or in an unexpected fashion.
-
-
-ABI/API Deprecation
--------------------
-
-The DPDK ABI policy
-~~~~~~~~~~~~~~~~~~~
-
-ABI versions are set at the time of major release labeling, and the ABI may
-change multiple times, without warning, between the last release label and the
-HEAD label of the git tree.
-
-ABI versions, once released, are available until such time as their
-deprecation has been noted in the Release Notes for at least one major release
-cycle. For example consider the case where the ABI for DPDK 2.0 has been
-shipped and then a decision is made to modify it during the development of
-DPDK 2.1. The decision will be recorded in the Release Notes for the DPDK 2.1
-release and the modification will be made available in the DPDK 2.2 release.
-
-ABI versions may be deprecated in whole or in part as needed by a given
-update.
-
-Some ABI changes may be too significant to reasonably maintain multiple
-versions. In those cases ABI's may be updated without backward compatibility
-being provided. The requirements for doing so are:
-
-#. At least 3 acknowledgments of the need to do so must be made on the
- dpdk.org mailing list.
-
- - The acknowledgment of the maintainer of the component is mandatory, or if
- no maintainer is available for the component, the tree/sub-tree maintainer
- for that component must acknowledge the ABI change instead.
-
- - It is also recommended that acknowledgments from different "areas of
- interest" be sought for each deprecation, for example: from NIC vendors,
- CPU vendors, end-users, etc.
-
-#. The changes (including an alternative map file) can be included with
- deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option,
- to provide more details about oncoming changes.
- ``RTE_NEXT_ABI`` wrapper will be removed when it become the default ABI.
- More preferred way to provide this information is sending the feature
- as a separate patch and reference it in deprecation notice.
-
-#. A full deprecation cycle, as explained above, must be made to offer
- downstream consumers sufficient warning of the change.
-
-Note that the above process for ABI deprecation should not be undertaken
-lightly. ABI stability is extremely important for downstream consumers of the
-DPDK, especially when distributed in shared object form. Every effort should
-be made to preserve the ABI whenever possible. The ABI should only be changed
-for significant reasons, such as performance enhancements. ABI breakage due to
-changes such as reorganizing public structure fields for aesthetic or
-readability purposes should be avoided.
-
-.. note::
-
- Updates to the minimum hardware requirements, which drop support for hardware
- which was previously supported, should be treated as an ABI change, and
- follow the relevant deprecation policy procedures as above: 3 acks and
- announcement at least one release in advance.
-
-Examples of Deprecation Notices
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The following are some examples of ABI deprecation notices which would be
-added to the Release Notes:
-
-* The Macro ``#RTE_FOO`` is deprecated and will be removed with version 2.0,
- to be replaced with the inline function ``rte_foo()``.
-
-* The function ``rte_mbuf_grok()`` has been updated to include a new parameter
- in version 2.0. Backwards compatibility will be maintained for this function
- until the release of version 2.1
-
-* The members of ``struct rte_foo`` have been reorganized in release 2.0 for
- performance reasons. Existing binary applications will have backwards
- compatibility in release 2.0, while newly built binaries will need to
- reference the new structure variant ``struct rte_foo2``. Compatibility will
- be removed in release 2.2, and all applications will require updating and
- rebuilding to the new structure at that time, which will be renamed to the
- original ``struct rte_foo``.
-
-* Significant ABI changes are planned for the ``librte_dostuff`` library. The
- upcoming release 2.0 will not contain these changes, but release 2.1 will,
- and no backwards compatibility is planned due to the extensive nature of
- these changes. Binaries using this library built prior to version 2.1 will
- require updating and recompilation.
-
-New API replacing previous one
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-If a new API proposed functionally replaces an existing one, when the new API
-becomes non-experimental then the old one is marked with ``__rte_deprecated``.
-Deprecated APIs are removed completely just after the next LTS.
-
-Reminder that old API should follow deprecation process to be removed.
-
-
-Experimental APIs
------------------
-
-APIs marked as ``experimental`` are not considered part of the ABI and may
-change without warning at any time. Since changes to APIs are most likely
-immediately after their introduction, as users begin to take advantage of
-those new APIs and start finding issues with them, new DPDK APIs will be
-automatically marked as ``experimental`` to allow for a period of stabilization
-before they become part of a tracked ABI.
-
-Note that marking an API as experimental is a multi step process.
-To mark an API as experimental, the symbols which are desired to be exported
-must be placed in an EXPERIMENTAL version block in the corresponding libraries'
-version map script.
-Secondly, the corresponding prototypes of those exported functions (in the
-development header files), must be marked with the ``__rte_experimental`` tag
-(see ``rte_compat.h``).
-The DPDK build makefiles perform a check to ensure that the map file and the
-C code reflect the same list of symbols.
-This check can be circumvented by defining ``ALLOW_EXPERIMENTAL_API``
-during compilation in the corresponding library Makefile.
-
-In addition to tagging the code with ``__rte_experimental``,
-the doxygen markup must also contain the EXPERIMENTAL string,
-and the MAINTAINERS file should note the EXPERIMENTAL libraries.
-
-For removing the experimental tag associated with an API, deprecation notice
-is not required. Though, an API should remain in experimental state for at least
-one release. Thereafter, normal process of posting patch for review to mailing
-list can be followed.
-
-
-Library versioning
-------------------
-
-Downstreams might want to provide different DPDK releases at the same time to
-support multiple consumers of DPDK linked against older and newer sonames.
-
-Also due to the interdependencies that DPDK libraries can have applications
-might end up with an executable space in which multiple versions of a library
-are mapped by ld.so.
-
-Think of LibA that got an ABI bump and LibB that did not get an ABI bump but is
-depending on LibA.
-
-.. note::
-
- Application
- \-> LibA.old
- \-> LibB.new -> LibA.new
-
-That is a conflict which can be avoided by setting ``CONFIG_RTE_MAJOR_ABI``.
-If set, the value of ``CONFIG_RTE_MAJOR_ABI`` overwrites all - otherwise per
-library - versions defined in the libraries ``LIBABIVER``.
-An example might be ``CONFIG_RTE_MAJOR_ABI=16.11`` which will make all libraries
-``librte<?>.so.16.11`` instead of ``librte<?>.so.<LIBABIVER>``.
-
-
-ABI versioning
---------------
-
-Versioning Macros
-~~~~~~~~~~~~~~~~~
-
-When a symbol is exported from a library to provide an API, it also provides a
-calling convention (ABI) that is embodied in its name, return type and
-arguments. Occasionally that function may need to change to accommodate new
-functionality or behavior. When that occurs, it is desirable to allow for
-backward compatibility for a time with older binaries that are dynamically
-linked to the DPDK.
-
-To support backward compatibility the ``rte_compat.h``
-header file provides macros to use when updating exported functions. These
-macros are used in conjunction with the ``rte_<library>_version.map`` file for
-a given library to allow multiple versions of a symbol to exist in a shared
-library so that older binaries need not be immediately recompiled.
-
-The macros exported are:
-
-* ``VERSION_SYMBOL(b, e, n)``: Creates a symbol version table entry binding
- versioned symbol ``b@DPDK_n`` to the internal function ``b_e``.
-
-* ``BIND_DEFAULT_SYMBOL(b, e, n)``: Creates a symbol version entry instructing
- the linker to bind references to symbol ``b`` to the internal symbol
- ``b_e``.
-
-* ``MAP_STATIC_SYMBOL(f, p)``: Declare the prototype ``f``, and map it to the
- fully qualified function ``p``, so that if a symbol becomes versioned, it
- can still be mapped back to the public symbol name.
-
-Examples of ABI Macro use
-^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Updating a public API
-_____________________
-
-Assume we have a function as follows
-
-.. code-block:: c
-
- /*
- * Create an acl context object for apps to
- * manipulate
- */
- struct rte_acl_ctx *
- rte_acl_create(const struct rte_acl_param *param)
- {
- ...
- }
-
-
-Assume that struct rte_acl_ctx is a private structure, and that a developer
-wishes to enhance the acl api so that a debugging flag can be enabled on a
-per-context basis. This requires an addition to the structure (which, being
-private, is safe), but it also requires modifying the code as follows
-
-.. code-block:: c
-
- /*
- * Create an acl context object for apps to
- * manipulate
- */
- struct rte_acl_ctx *
- rte_acl_create(const struct rte_acl_param *param, int debug)
- {
- ...
- }
-
-
-Note also that, being a public function, the header file prototype must also be
-changed, as must all the call sites, to reflect the new ABI footprint. We will
-maintain previous ABI versions that are accessible only to previously compiled
-binaries
-
-The addition of a parameter to the function is ABI breaking as the function is
-public, and existing application may use it in its current form. However, the
-compatibility macros in DPDK allow a developer to use symbol versioning so that
-multiple functions can be mapped to the same public symbol based on when an
-application was linked to it. To see how this is done, we start with the
-requisite libraries version map file. Initially the version map file for the
-acl library looks like this
-
-.. code-block:: none
-
- DPDK_2.0 {
- global:
-
- rte_acl_add_rules;
- rte_acl_build;
- rte_acl_classify;
- rte_acl_classify_alg;
- rte_acl_classify_scalar;
- rte_acl_create;
- rte_acl_dump;
- rte_acl_find_existing;
- rte_acl_free;
- rte_acl_ipv4vlan_add_rules;
- rte_acl_ipv4vlan_build;
- rte_acl_list_dump;
- rte_acl_reset;
- rte_acl_reset_rules;
- rte_acl_set_ctx_classify;
-
- local: *;
- };
-
-This file needs to be modified as follows
-
-.. code-block:: none
-
- DPDK_2.0 {
- global:
-
- rte_acl_add_rules;
- rte_acl_build;
- rte_acl_classify;
- rte_acl_classify_alg;
- rte_acl_classify_scalar;
- rte_acl_create;
- rte_acl_dump;
- rte_acl_find_existing;
- rte_acl_free;
- rte_acl_ipv4vlan_add_rules;
- rte_acl_ipv4vlan_build;
- rte_acl_list_dump;
- rte_acl_reset;
- rte_acl_reset_rules;
- rte_acl_set_ctx_classify;
-
- local: *;
- };
-
- DPDK_2.1 {
- global:
- rte_acl_create;
-
- } DPDK_2.0;
-
-The addition of the new block tells the linker that a new version node is
-available (DPDK_2.1), which contains the symbol rte_acl_create, and inherits the
-symbols from the DPDK_2.0 node. This list is directly translated into a list of
-exported symbols when DPDK is compiled as a shared library
-
-Next, we need to specify in the code which function map to the rte_acl_create
-symbol at which versions. First, at the site of the initial symbol definition,
-we need to update the function so that it is uniquely named, and not in conflict
-with the public symbol name
-
-.. code-block:: c
-
- struct rte_acl_ctx *
- -rte_acl_create(const struct rte_acl_param *param)
- +rte_acl_create_v20(const struct rte_acl_param *param)
- {
- size_t sz;
- struct rte_acl_ctx *ctx;
- ...
-
-Note that the base name of the symbol was kept intact, as this is conducive to
-the macros used for versioning symbols. That is our next step, mapping this new
-symbol name to the initial symbol name at version node 2.0. Immediately after
-the function, we add this line of code
-
-.. code-block:: c
-
- VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
-
-Remembering to also add the rte_compat.h header to the requisite c file where
-these changes are being made. The above macro instructs the linker to create a
-new symbol ``rte_acl_create@DPDK_2.0``, which matches the symbol created in older
-builds, but now points to the above newly named function. We have now mapped
-the original rte_acl_create symbol to the original function (but with a new
-name)
-
-Next, we need to create the 2.1 version of the symbol. We create a new function
-name, with a different suffix, and implement it appropriately
-
-.. code-block:: c
-
- struct rte_acl_ctx *
- rte_acl_create_v21(const struct rte_acl_param *param, int debug);
- {
- struct rte_acl_ctx *ctx = rte_acl_create_v20(param);
-
- ctx->debug = debug;
-
- return ctx;
- }
-
-This code serves as our new API call. Its the same as our old call, but adds
-the new parameter in place. Next we need to map this function to the symbol
-``rte_acl_create@DPDK_2.1``. To do this, we modify the public prototype of the call
-in the header file, adding the macro there to inform all including applications,
-that on re-link, the default rte_acl_create symbol should point to this
-function. Note that we could do this by simply naming the function above
-rte_acl_create, and the linker would chose the most recent version tag to apply
-in the version script, but we can also do this in the header file
-
-.. code-block:: c
-
- struct rte_acl_ctx *
- -rte_acl_create(const struct rte_acl_param *param);
- +rte_acl_create(const struct rte_acl_param *param, int debug);
- +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
-
-The BIND_DEFAULT_SYMBOL macro explicitly tells applications that include this
-header, to link to the rte_acl_create_v21 function and apply the DPDK_2.1
-version node to it. This method is more explicit and flexible than just
-re-implementing the exact symbol name, and allows for other features (such as
-linking to the old symbol version by default, when the new ABI is to be opt-in
-for a period.
-
-One last thing we need to do. Note that we've taken what was a public symbol,
-and duplicated it into two uniquely and differently named symbols. We've then
-mapped each of those back to the public symbol ``rte_acl_create`` with different
-version tags. This only applies to dynamic linking, as static linking has no
-notion of versioning. That leaves this code in a position of no longer having a
-symbol simply named ``rte_acl_create`` and a static build will fail on that
-missing symbol.
-
-To correct this, we can simply map a function of our choosing back to the public
-symbol in the static build with the ``MAP_STATIC_SYMBOL`` macro. Generally the
-assumption is that the most recent version of the symbol is the one you want to
-map. So, back in the C file where, immediately after ``rte_acl_create_v21`` is
-defined, we add this
-
-.. code-block:: c
-
- struct rte_acl_ctx *
- rte_acl_create_v21(const struct rte_acl_param *param, int debug)
- {
- ...
- }
- MAP_STATIC_SYMBOL(struct rte_acl_ctx *rte_acl_create(const struct rte_acl_param *param, int debug), rte_acl_create_v21);
-
-That tells the compiler that, when building a static library, any calls to the
-symbol ``rte_acl_create`` should be linked to ``rte_acl_create_v21``
-
-That's it, on the next shared library rebuild, there will be two versions of
-rte_acl_create, an old DPDK_2.0 version, used by previously built applications,
-and a new DPDK_2.1 version, used by future built applications.
-
-
-Deprecating part of a public API
-________________________________
-
-Lets assume that you've done the above update, and after a few releases have
-passed you decide you would like to retire the old version of the function.
-After having gone through the ABI deprecation announcement process, removal is
-easy. Start by removing the symbol from the requisite version map file:
-
-.. code-block:: none
-
- DPDK_2.0 {
- global:
-
- rte_acl_add_rules;
- rte_acl_build;
- rte_acl_classify;
- rte_acl_classify_alg;
- rte_acl_classify_scalar;
- rte_acl_dump;
- - rte_acl_create
- rte_acl_find_existing;
- rte_acl_free;
- rte_acl_ipv4vlan_add_rules;
- rte_acl_ipv4vlan_build;
- rte_acl_list_dump;
- rte_acl_reset;
- rte_acl_reset_rules;
- rte_acl_set_ctx_classify;
-
- local: *;
- };
-
- DPDK_2.1 {
- global:
- rte_acl_create;
- } DPDK_2.0;
-
-
-Next remove the corresponding versioned export.
-
-.. code-block:: c
-
- -VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
-
-
-Note that the internal function definition could also be removed, but its used
-in our example by the newer version _v21, so we leave it in place. This is a
-coding style choice.
-
-Lastly, we need to bump the LIBABIVER number for this library in the Makefile to
-indicate to applications doing dynamic linking that this is a later, and
-possibly incompatible library version:
-
-.. code-block:: c
-
- -LIBABIVER := 1
- +LIBABIVER := 2
-
-Deprecating an entire ABI version
-_________________________________
-
-While removing a symbol from and ABI may be useful, it is often more practical
-to remove an entire version node at once. If a version node completely
-specifies an API, then removing part of it, typically makes it incomplete. In
-those cases it is better to remove the entire node
-
-To do this, start by modifying the version map file, such that all symbols from
-the node to be removed are merged into the next node in the map
-
-In the case of our map above, it would transform to look as follows
-
-.. code-block:: none
-
- DPDK_2.1 {
- global:
-
- rte_acl_add_rules;
- rte_acl_build;
- rte_acl_classify;
- rte_acl_classify_alg;
- rte_acl_classify_scalar;
- rte_acl_dump;
- rte_acl_create
- rte_acl_find_existing;
- rte_acl_free;
- rte_acl_ipv4vlan_add_rules;
- rte_acl_ipv4vlan_build;
- rte_acl_list_dump;
- rte_acl_reset;
- rte_acl_reset_rules;
- rte_acl_set_ctx_classify;
-
- local: *;
- };
-
-Then any uses of BIND_DEFAULT_SYMBOL that pointed to the old node should be
-updated to point to the new version node in any header files for all affected
-symbols.
-
-.. code-block:: c
-
- -BIND_DEFAULT_SYMBOL(rte_acl_create, _v20, 2.0);
- +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
-
-Lastly, any VERSION_SYMBOL macros that point to the old version node should be
-removed, taking care to keep, where need old code in place to support newer
-versions of the symbol.
-
-
-Running the ABI Validator
--------------------------
-
-The ``devtools`` directory in the DPDK source tree contains a utility program,
-``validate-abi.sh``, for validating the DPDK ABI based on the Linux `ABI
-Compliance Checker
-<http://ispras.linuxbase.org/index.php/ABI_compliance_checker>`_.
-
-This has a dependency on the ``abi-compliance-checker`` and ``and abi-dumper``
-utilities which can be installed via a package manager. For example::
-
- sudo yum install abi-compliance-checker
- sudo yum install abi-dumper
-
-The syntax of the ``validate-abi.sh`` utility is::
-
- ./devtools/validate-abi.sh <REV1> <REV2>
-
-Where ``REV1`` and ``REV2`` are valid gitrevisions(7)
-https://www.kernel.org/pub/software/scm/git/docs/gitrevisions.html
-on the local repo.
-
-For example::
-
- # Check between the previous and latest commit:
- ./devtools/validate-abi.sh HEAD~1 HEAD
-
- # Check on a specific compilation target:
- ./devtools/validate-abi.sh -t x86_64-native-linux-gcc HEAD~1 HEAD
-
- # Check between two tags:
- ./devtools/validate-abi.sh v2.0.0 v2.1.0
-
- # Check between git master and local topic-branch "vhost-hacking":
- ./devtools/validate-abi.sh master vhost-hacking
-
-After the validation script completes (it can take a while since it need to
-compile both tags) it will create compatibility reports in the
-``./abi-check/compat_report`` directory. Listed incompatibilities can be found
-as follows::
-
- grep -lr Incompatible abi-check/compat_reports/
--
2.7.4
^ permalink raw reply [relevance 13%]
* [dpdk-dev] [PATCH v5 0/4] doc: changes to abi policy introducing major abi versions
@ 2019-09-27 16:30 10% Ray Kinsella
2019-09-27 16:30 13% ` [dpdk-dev] [PATCH v5 1/4] doc: separate versioning.rst into version and policy Ray Kinsella
` (3 more replies)
0 siblings, 4 replies; 200+ results
From: Ray Kinsella @ 2019-09-27 16:30 UTC (permalink / raw)
To: dev
Cc: mdr, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, nhorman,
maxime.coquelin, john.mcnamara, marko.kovacevic, hemant.agrawal,
ktraynor
TL;DR abbreviation:
A major ABI version that all DPDK releases during a one year period
support. ABI versioning is managed at a project-level, in place of library-level
management. ABI changes to add new features are permitted, as long as ABI
compatibility with the major ABI version is maintained.
Detail:
This patch introduces major ABI versions, supported for one year and released
aligned with the LTS release. This ABI version is then supported by all
subsequent releases within that one year period. The intention is that the one
year support period, will then be reviewed after the initial year with the
intention of lengthing the support period for the next ABI version.
ABI changes that preserve ABI compatibility with the major ABI version are
permitted in subsequent releases. ABI changes, follow similar approval rules as
before with the additional gate of now requiring technical board approval. The
merging and release of ABI breaking changes would now be pushed to the
declaration of the next major ABI version.
This change encourages developers to maintain ABI compatibility with the major
ABI version, by promoting a permissive culture around those changes that
preserve ABI compatibility. This approach begins to align DPDK with those
projects that declare major ABI versions (e.g. version 2.x, 3.x) and support
those versions for some period, typically two years or more.
To provide an example of how this might work in practice:
* DPDK v20 is declared as the supported ABI version for one year, aligned with
the DPDK v19.11 (LTS) release. All library sonames are updated to reflect the
new ABI version, e.g. librte_eal.so.20, librte_acl.so.20...
* DPDK v20.02 .. v20.08 releases are ABI compatible with the DPDK v20 ABI. ABI
changes are permitted from DPDK v20.02 onwards, with the condition that ABI
compatibility with DPDK v20 is preserved.
* DPDK v21 is declared as the new supported ABI version for two years, aligned
with the DPDK v20.11 (LTS) release. The DPDK v20 ABI is now depreciated,
library sonames are updated to v21 and ABI compatibility breaking changes may
be introduced.
v5
* Added figure to abi_policy.rst, mapping abi versions and abi compatibility to
DPDK releases. (as suggested by Neil Horman)
v4
* Removed changes to stable.rst, fixed typos and clarified the ABI policy
"warning".
v3
* Added myself as the maintainer of the ABI policy.
* Updated the policy and versioning guides to use the year of the LTS+1 (e.g.
v20), as the abi major version number.
v2
* Restructured the patch into 3 patches:
1. Splits the original versioning document into an ABI policy document
and ABI versioning document.
2. Add changes to the policy document introducing major ABI versions.
3. Fixes up the versioning document in light of major ABI versioning.
* Reduces the initial ABI stability from two years to one year, with a review
after the first year.
* Adds detail around ABI version handling for experimental libraries.
* Adds detail around chain of responsility for removing deprecated symbols.
Ray Kinsella (4):
doc: separate versioning.rst into version and policy
doc: changes to abi policy introducing major abi versions
doc: updates to versioning guide for abi versions
doc: add maintainer for abi policy
MAINTAINERS | 4 +
doc/guides/contributing/abi_policy.rst | 315 +++++++++++
doc/guides/contributing/abi_versioning.rst | 515 ++++++++++++++++++
.../contributing/img/abi_stability_policy.png | Bin 0 -> 61277 bytes
doc/guides/contributing/index.rst | 3 +-
doc/guides/contributing/patches.rst | 6 +-
doc/guides/contributing/stable.rst | 12 +-
doc/guides/contributing/versioning.rst | 591 ---------------------
doc/guides/rel_notes/deprecation.rst | 2 +-
9 files changed, 844 insertions(+), 604 deletions(-)
create mode 100644 doc/guides/contributing/abi_policy.rst
create mode 100644 doc/guides/contributing/abi_versioning.rst
create mode 100644 doc/guides/contributing/img/abi_stability_policy.png
delete mode 100644 doc/guides/contributing/versioning.rst
--
2.7.4
^ permalink raw reply [relevance 10%]
* Re: [dpdk-dev] [PATCH v4 1/4] doc: separate versioning.rst into version and policy
2019-09-25 10:23 13% ` [dpdk-dev] [PATCH v4 1/4] doc: separate versioning.rst into version and policy Ray Kinsella
2019-09-25 12:24 5% ` Neil Horman
@ 2019-09-27 15:43 4% ` Aaron Conole
2019-09-27 16:43 4% ` Ray Kinsella
1 sibling, 1 reply; 200+ results
From: Aaron Conole @ 2019-09-27 15:43 UTC (permalink / raw)
To: Ray Kinsella
Cc: dev, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, nhorman,
maxime.coquelin, john.mcnamara, marko.kovacevic, hemant.agrawal,
ktraynor
Ray Kinsella <mdr@ashroe.eu> writes:
> Separate versioning.rst into abi versioning and abi policy guidance, in
> preparation for adding more detail to the abi policy.
>
> Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
> ---
> doc/guides/contributing/abi_policy.rst | 169 +++++++++
> doc/guides/contributing/abi_versioning.rst | 427 +++++++++++++++++++++
> doc/guides/contributing/index.rst | 3 +-
> doc/guides/contributing/versioning.rst | 591 -----------------------------
> 4 files changed, 598 insertions(+), 592 deletions(-)
> create mode 100644 doc/guides/contributing/abi_policy.rst
> create mode 100644 doc/guides/contributing/abi_versioning.rst
> delete mode 100644 doc/guides/contributing/versioning.rst
>
> diff --git a/doc/guides/contributing/abi_policy.rst b/doc/guides/contributing/abi_policy.rst
> new file mode 100644
> index 0000000..55bacb4
> --- /dev/null
> +++ b/doc/guides/contributing/abi_policy.rst
> @@ -0,0 +1,169 @@
> +.. SPDX-License-Identifier: BSD-3-Clause
> + Copyright 2018 The DPDK contributors
> +
> +.. abi_api_policy:
> +
> +DPDK ABI/API policy
> +===================
> +
> +Description
> +-----------
> +
> +This document details some methods for handling ABI management in the DPDK.
> +
> +General Guidelines
> +------------------
> +
> +#. Whenever possible, ABI should be preserved
> +#. ABI/API may be changed with a deprecation process
> +#. The modification of symbols can generally be managed with versioning
> +#. Libraries or APIs marked in ``experimental`` state may change without constraint
> +#. New APIs will be marked as ``experimental`` for at least one release to allow
> + any issues found by users of the new API to be fixed quickly
> +#. The addition of symbols is generally not problematic
> +#. The removal of symbols generally is an ABI break and requires bumping of the
> + LIBABIVER macro
> +#. Updates to the minimum hardware requirements, which drop support for hardware which
> + was previously supported, should be treated as an ABI change.
> +
> +What is an ABI
> +~~~~~~~~~~~~~~
> +
> +An ABI (Application Binary Interface) is the set of runtime interfaces exposed
> +by a library. It is similar to an API (Application Programming Interface) but
> +is the result of compilation. It is also effectively cloned when applications
> +link to dynamic libraries. That is to say when an application is compiled to
> +link against dynamic libraries, it is assumed that the ABI remains constant
> +between the time the application is compiled/linked, and the time that it runs.
> +Therefore, in the case of dynamic linking, it is critical that an ABI is
> +preserved, or (when modified), done in such a way that the application is unable
> +to behave improperly or in an unexpected fashion.
> +
This section probably needs a bit more details. People still are
confused what exactly constitutes ABI vs. API (see
http://mails.dpdk.org/archives/dev/2018-January/085209.html for a
confusing example).
It's important that people know not just function signatures, but also
return codes, and even data structure layouts are all part of the ABI.
> +
> +ABI/API Deprecation
> +-------------------
> +
> +The DPDK ABI policy
> +~~~~~~~~~~~~~~~~~~~
> +
> +ABI versions are set at the time of major release labeling, and the ABI may
> +change multiple times, without warning, between the last release label and the
> +HEAD label of the git tree.
> +
> +ABI versions, once released, are available until such time as their
> +deprecation has been noted in the Release Notes for at least one major release
> +cycle. For example consider the case where the ABI for DPDK 2.0 has been
> +shipped and then a decision is made to modify it during the development of
> +DPDK 2.1. The decision will be recorded in the Release Notes for the DPDK 2.1
> +release and the modification will be made available in the DPDK 2.2 release.
> +
> +ABI versions may be deprecated in whole or in part as needed by a given
> +update.
> +
> +Some ABI changes may be too significant to reasonably maintain multiple
> +versions. In those cases ABI's may be updated without backward compatibility
> +being provided. The requirements for doing so are:
> +
> +#. At least 3 acknowledgments of the need to do so must be made on the
> + dpdk.org mailing list.
> +
> + - The acknowledgment of the maintainer of the component is mandatory, or if
> + no maintainer is available for the component, the tree/sub-tree maintainer
> + for that component must acknowledge the ABI change instead.
> +
> + - It is also recommended that acknowledgments from different "areas of
> + interest" be sought for each deprecation, for example: from NIC vendors,
> + CPU vendors, end-users, etc.
> +
> +#. The changes (including an alternative map file) can be included with
> + deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option,
> + to provide more details about oncoming changes.
> + ``RTE_NEXT_ABI`` wrapper will be removed when it become the default ABI.
> + More preferred way to provide this information is sending the feature
> + as a separate patch and reference it in deprecation notice.
> +
> +#. A full deprecation cycle, as explained above, must be made to offer
> + downstream consumers sufficient warning of the change.
> +
> +Note that the above process for ABI deprecation should not be undertaken
> +lightly. ABI stability is extremely important for downstream consumers of the
> +DPDK, especially when distributed in shared object form. Every effort should
> +be made to preserve the ABI whenever possible. The ABI should only be changed
> +for significant reasons, such as performance enhancements. ABI breakage due to
> +changes such as reorganizing public structure fields for aesthetic or
> +readability purposes should be avoided.
> +
> +.. note::
> +
> + Updates to the minimum hardware requirements, which drop support for hardware
> + which was previously supported, should be treated as an ABI change, and
> + follow the relevant deprecation policy procedures as above: 3 acks and
> + announcement at least one release in advance.
> +
> +Examples of Deprecation Notices
> +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
> +
> +The following are some examples of ABI deprecation notices which would be
> +added to the Release Notes:
> +
> +* The Macro ``#RTE_FOO`` is deprecated and will be removed with version 2.0,
> + to be replaced with the inline function ``rte_foo()``.
> +
> +* The function ``rte_mbuf_grok()`` has been updated to include a new parameter
> + in version 2.0. Backwards compatibility will be maintained for this function
> + until the release of version 2.1
> +
> +* The members of ``struct rte_foo`` have been reorganized in release 2.0 for
> + performance reasons. Existing binary applications will have backwards
> + compatibility in release 2.0, while newly built binaries will need to
> + reference the new structure variant ``struct rte_foo2``. Compatibility will
> + be removed in release 2.2, and all applications will require updating and
> + rebuilding to the new structure at that time, which will be renamed to the
> + original ``struct rte_foo``.
> +
> +* Significant ABI changes are planned for the ``librte_dostuff`` library. The
> + upcoming release 2.0 will not contain these changes, but release 2.1 will,
> + and no backwards compatibility is planned due to the extensive nature of
> + these changes. Binaries using this library built prior to version 2.1 will
> + require updating and recompilation.
> +
> +New API replacing previous one
> +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
> +
> +If a new API proposed functionally replaces an existing one, when the new API
> +becomes non-experimental then the old one is marked with ``__rte_deprecated``.
> +Deprecated APIs are removed completely just after the next LTS.
> +
> +Reminder that old API should follow deprecation process to be removed.
> +
> +
> +Experimental APIs
> +-----------------
> +
> +APIs marked as ``experimental`` are not considered part of the ABI and may
> +change without warning at any time. Since changes to APIs are most likely
> +immediately after their introduction, as users begin to take advantage of
> +those new APIs and start finding issues with them, new DPDK APIs will be
> +automatically marked as ``experimental`` to allow for a period of stabilization
> +before they become part of a tracked ABI.
> +
> +Note that marking an API as experimental is a multi step process.
> +To mark an API as experimental, the symbols which are desired to be exported
> +must be placed in an EXPERIMENTAL version block in the corresponding libraries'
> +version map script.
> +Secondly, the corresponding prototypes of those exported functions (in the
> +development header files), must be marked with the ``__rte_experimental`` tag
> +(see ``rte_compat.h``).
> +The DPDK build makefiles perform a check to ensure that the map file and the
> +C code reflect the same list of symbols.
> +This check can be circumvented by defining ``ALLOW_EXPERIMENTAL_API``
> +during compilation in the corresponding library Makefile.
> +
> +In addition to tagging the code with ``__rte_experimental``,
> +the doxygen markup must also contain the EXPERIMENTAL string,
> +and the MAINTAINERS file should note the EXPERIMENTAL libraries.
> +
> +For removing the experimental tag associated with an API, deprecation notice
> +is not required. Though, an API should remain in experimental state for at least
> +one release. Thereafter, normal process of posting patch for review to mailing
> +list can be followed.
> diff --git a/doc/guides/contributing/abi_versioning.rst b/doc/guides/contributing/abi_versioning.rst
> new file mode 100644
> index 0000000..53e6ac0
> --- /dev/null
> +++ b/doc/guides/contributing/abi_versioning.rst
> @@ -0,0 +1,427 @@
> +.. SPDX-License-Identifier: BSD-3-Clause
> + Copyright 2018 The DPDK contributors
> +
> +.. library_versioning:
> +
> +Library versioning
> +------------------
> +
> +Downstreams might want to provide different DPDK releases at the same time to
> +support multiple consumers of DPDK linked against older and newer sonames.
> +
> +Also due to the interdependencies that DPDK libraries can have applications
> +might end up with an executable space in which multiple versions of a library
> +are mapped by ld.so.
> +
> +Think of LibA that got an ABI bump and LibB that did not get an ABI bump but is
> +depending on LibA.
> +
> +.. note::
> +
> + Application
> + \-> LibA.old
> + \-> LibB.new -> LibA.new
> +
> +That is a conflict which can be avoided by setting ``CONFIG_RTE_MAJOR_ABI``.
> +If set, the value of ``CONFIG_RTE_MAJOR_ABI`` overwrites all - otherwise per
> +library - versions defined in the libraries ``LIBABIVER``.
> +An example might be ``CONFIG_RTE_MAJOR_ABI=16.11`` which will make all libraries
> +``librte<?>.so.16.11`` instead of ``librte<?>.so.<LIBABIVER>``.
> +
> +
> +ABI versioning
> +--------------
> +
> +Versioning Macros
> +~~~~~~~~~~~~~~~~~
> +
> +When a symbol is exported from a library to provide an API, it also provides a
> +calling convention (ABI) that is embodied in its name, return type and
> +arguments. Occasionally that function may need to change to accommodate new
> +functionality or behavior. When that occurs, it is desirable to allow for
> +backward compatibility for a time with older binaries that are dynamically
> +linked to the DPDK.
> +
> +To support backward compatibility the ``rte_compat.h``
> +header file provides macros to use when updating exported functions. These
> +macros are used in conjunction with the ``rte_<library>_version.map`` file for
> +a given library to allow multiple versions of a symbol to exist in a shared
> +library so that older binaries need not be immediately recompiled.
> +
> +The macros exported are:
> +
> +* ``VERSION_SYMBOL(b, e, n)``: Creates a symbol version table entry binding
> + versioned symbol ``b@DPDK_n`` to the internal function ``b_e``.
> +
> +* ``BIND_DEFAULT_SYMBOL(b, e, n)``: Creates a symbol version entry instructing
> + the linker to bind references to symbol ``b`` to the internal symbol
> + ``b_e``.
> +
> +* ``MAP_STATIC_SYMBOL(f, p)``: Declare the prototype ``f``, and map it to the
> + fully qualified function ``p``, so that if a symbol becomes versioned, it
> + can still be mapped back to the public symbol name.
> +
> +Examples of ABI Macro use
> +^^^^^^^^^^^^^^^^^^^^^^^^^
> +
> +Updating a public API
> +_____________________
> +
> +Assume we have a function as follows
> +
> +.. code-block:: c
> +
> + /*
> + * Create an acl context object for apps to
> + * manipulate
> + */
> + struct rte_acl_ctx *
> + rte_acl_create(const struct rte_acl_param *param)
> + {
> + ...
> + }
> +
> +
> +Assume that struct rte_acl_ctx is a private structure, and that a developer
> +wishes to enhance the acl api so that a debugging flag can be enabled on a
> +per-context basis. This requires an addition to the structure (which, being
> +private, is safe), but it also requires modifying the code as follows
> +
> +.. code-block:: c
> +
> + /*
> + * Create an acl context object for apps to
> + * manipulate
> + */
> + struct rte_acl_ctx *
> + rte_acl_create(const struct rte_acl_param *param, int debug)
> + {
> + ...
> + }
> +
> +
> +Note also that, being a public function, the header file prototype must also be
> +changed, as must all the call sites, to reflect the new ABI footprint. We will
> +maintain previous ABI versions that are accessible only to previously compiled
> +binaries
> +
> +The addition of a parameter to the function is ABI breaking as the function is
> +public, and existing application may use it in its current form. However, the
> +compatibility macros in DPDK allow a developer to use symbol versioning so that
> +multiple functions can be mapped to the same public symbol based on when an
> +application was linked to it. To see how this is done, we start with the
> +requisite libraries version map file. Initially the version map file for the
> +acl library looks like this
> +
> +.. code-block:: none
> +
> + DPDK_2.0 {
> + global:
> +
> + rte_acl_add_rules;
> + rte_acl_build;
> + rte_acl_classify;
> + rte_acl_classify_alg;
> + rte_acl_classify_scalar;
> + rte_acl_create;
> + rte_acl_dump;
> + rte_acl_find_existing;
> + rte_acl_free;
> + rte_acl_ipv4vlan_add_rules;
> + rte_acl_ipv4vlan_build;
> + rte_acl_list_dump;
> + rte_acl_reset;
> + rte_acl_reset_rules;
> + rte_acl_set_ctx_classify;
> +
> + local: *;
> + };
> +
> +This file needs to be modified as follows
> +
> +.. code-block:: none
> +
> + DPDK_2.0 {
> + global:
> +
> + rte_acl_add_rules;
> + rte_acl_build;
> + rte_acl_classify;
> + rte_acl_classify_alg;
> + rte_acl_classify_scalar;
> + rte_acl_create;
> + rte_acl_dump;
> + rte_acl_find_existing;
> + rte_acl_free;
> + rte_acl_ipv4vlan_add_rules;
> + rte_acl_ipv4vlan_build;
> + rte_acl_list_dump;
> + rte_acl_reset;
> + rte_acl_reset_rules;
> + rte_acl_set_ctx_classify;
> +
> + local: *;
> + };
> +
> + DPDK_2.1 {
> + global:
> + rte_acl_create;
> +
> + } DPDK_2.0;
> +
> +The addition of the new block tells the linker that a new version node is
> +available (DPDK_2.1), which contains the symbol rte_acl_create, and inherits the
> +symbols from the DPDK_2.0 node. This list is directly translated into a list of
> +exported symbols when DPDK is compiled as a shared library
> +
> +Next, we need to specify in the code which function map to the rte_acl_create
> +symbol at which versions. First, at the site of the initial symbol definition,
> +we need to update the function so that it is uniquely named, and not in conflict
> +with the public symbol name
> +
> +.. code-block:: c
> +
> + struct rte_acl_ctx *
> + -rte_acl_create(const struct rte_acl_param *param)
> + +rte_acl_create_v20(const struct rte_acl_param *param)
> + {
> + size_t sz;
> + struct rte_acl_ctx *ctx;
> + ...
> +
> +Note that the base name of the symbol was kept intact, as this is conducive to
> +the macros used for versioning symbols. That is our next step, mapping this new
> +symbol name to the initial symbol name at version node 2.0. Immediately after
> +the function, we add this line of code
> +
> +.. code-block:: c
> +
> + VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
> +
> +Remembering to also add the rte_compat.h header to the requisite c file where
> +these changes are being made. The above macro instructs the linker to create a
> +new symbol ``rte_acl_create@DPDK_2.0``, which matches the symbol created in older
> +builds, but now points to the above newly named function. We have now mapped
> +the original rte_acl_create symbol to the original function (but with a new
> +name)
> +
> +Next, we need to create the 2.1 version of the symbol. We create a new function
> +name, with a different suffix, and implement it appropriately
> +
> +.. code-block:: c
> +
> + struct rte_acl_ctx *
> + rte_acl_create_v21(const struct rte_acl_param *param, int debug);
> + {
> + struct rte_acl_ctx *ctx = rte_acl_create_v20(param);
> +
> + ctx->debug = debug;
> +
> + return ctx;
> + }
> +
> +This code serves as our new API call. Its the same as our old call, but adds
> +the new parameter in place. Next we need to map this function to the symbol
> +``rte_acl_create@DPDK_2.1``. To do this, we modify the public prototype of the call
> +in the header file, adding the macro there to inform all including applications,
> +that on re-link, the default rte_acl_create symbol should point to this
> +function. Note that we could do this by simply naming the function above
> +rte_acl_create, and the linker would chose the most recent version tag to apply
> +in the version script, but we can also do this in the header file
> +
> +.. code-block:: c
> +
> + struct rte_acl_ctx *
> + -rte_acl_create(const struct rte_acl_param *param);
> + +rte_acl_create(const struct rte_acl_param *param, int debug);
> + +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
> +
> +The BIND_DEFAULT_SYMBOL macro explicitly tells applications that include this
> +header, to link to the rte_acl_create_v21 function and apply the DPDK_2.1
> +version node to it. This method is more explicit and flexible than just
> +re-implementing the exact symbol name, and allows for other features (such as
> +linking to the old symbol version by default, when the new ABI is to be opt-in
> +for a period.
> +
> +One last thing we need to do. Note that we've taken what was a public symbol,
> +and duplicated it into two uniquely and differently named symbols. We've then
> +mapped each of those back to the public symbol ``rte_acl_create`` with different
> +version tags. This only applies to dynamic linking, as static linking has no
> +notion of versioning. That leaves this code in a position of no longer having a
> +symbol simply named ``rte_acl_create`` and a static build will fail on that
> +missing symbol.
> +
> +To correct this, we can simply map a function of our choosing back to the public
> +symbol in the static build with the ``MAP_STATIC_SYMBOL`` macro. Generally the
> +assumption is that the most recent version of the symbol is the one you want to
> +map. So, back in the C file where, immediately after ``rte_acl_create_v21`` is
> +defined, we add this
> +
> +.. code-block:: c
> +
> + struct rte_acl_ctx *
> + rte_acl_create_v21(const struct rte_acl_param *param, int debug)
> + {
> + ...
> + }
> + MAP_STATIC_SYMBOL(struct rte_acl_ctx *rte_acl_create(const struct rte_acl_param *param, int debug), rte_acl_create_v21);
> +
> +That tells the compiler that, when building a static library, any calls to the
> +symbol ``rte_acl_create`` should be linked to ``rte_acl_create_v21``
> +
> +That's it, on the next shared library rebuild, there will be two versions of
> +rte_acl_create, an old DPDK_2.0 version, used by previously built applications,
> +and a new DPDK_2.1 version, used by future built applications.
> +
> +
> +Deprecating part of a public API
> +________________________________
> +
> +Lets assume that you've done the above update, and after a few releases have
> +passed you decide you would like to retire the old version of the function.
> +After having gone through the ABI deprecation announcement process, removal is
> +easy. Start by removing the symbol from the requisite version map file:
> +
> +.. code-block:: none
> +
> + DPDK_2.0 {
> + global:
> +
> + rte_acl_add_rules;
> + rte_acl_build;
> + rte_acl_classify;
> + rte_acl_classify_alg;
> + rte_acl_classify_scalar;
> + rte_acl_dump;
> + - rte_acl_create
> + rte_acl_find_existing;
> + rte_acl_free;
> + rte_acl_ipv4vlan_add_rules;
> + rte_acl_ipv4vlan_build;
> + rte_acl_list_dump;
> + rte_acl_reset;
> + rte_acl_reset_rules;
> + rte_acl_set_ctx_classify;
> +
> + local: *;
> + };
> +
> + DPDK_2.1 {
> + global:
> + rte_acl_create;
> + } DPDK_2.0;
> +
> +
> +Next remove the corresponding versioned export.
> +
> +.. code-block:: c
> +
> + -VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
> +
> +
> +Note that the internal function definition could also be removed, but its used
> +in our example by the newer version _v21, so we leave it in place. This is a
> +coding style choice.
> +
> +Lastly, we need to bump the LIBABIVER number for this library in the Makefile to
> +indicate to applications doing dynamic linking that this is a later, and
> +possibly incompatible library version:
> +
> +.. code-block:: c
> +
> + -LIBABIVER := 1
> + +LIBABIVER := 2
> +
> +Deprecating an entire ABI version
> +_________________________________
> +
> +While removing a symbol from and ABI may be useful, it is often more practical
> +to remove an entire version node at once. If a version node completely
> +specifies an API, then removing part of it, typically makes it incomplete. In
> +those cases it is better to remove the entire node
> +
> +To do this, start by modifying the version map file, such that all symbols from
> +the node to be removed are merged into the next node in the map
> +
> +In the case of our map above, it would transform to look as follows
> +
> +.. code-block:: none
> +
> + DPDK_2.1 {
> + global:
> +
> + rte_acl_add_rules;
> + rte_acl_build;
> + rte_acl_classify;
> + rte_acl_classify_alg;
> + rte_acl_classify_scalar;
> + rte_acl_dump;
> + rte_acl_create
> + rte_acl_find_existing;
> + rte_acl_free;
> + rte_acl_ipv4vlan_add_rules;
> + rte_acl_ipv4vlan_build;
> + rte_acl_list_dump;
> + rte_acl_reset;
> + rte_acl_reset_rules;
> + rte_acl_set_ctx_classify;
> +
> + local: *;
> + };
> +
> +Then any uses of BIND_DEFAULT_SYMBOL that pointed to the old node should be
> +updated to point to the new version node in any header files for all affected
> +symbols.
> +
> +.. code-block:: c
> +
> + -BIND_DEFAULT_SYMBOL(rte_acl_create, _v20, 2.0);
> + +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
> +
> +Lastly, any VERSION_SYMBOL macros that point to the old version node should be
> +removed, taking care to keep, where need old code in place to support newer
> +versions of the symbol.
> +
> +
> +Running the ABI Validator
> +-------------------------
> +
> +The ``devtools`` directory in the DPDK source tree contains a utility program,
> +``validate-abi.sh``, for validating the DPDK ABI based on the Linux `ABI
> +Compliance Checker
> +<http://ispras.linuxbase.org/index.php/ABI_compliance_checker>`_.
> +
> +This has a dependency on the ``abi-compliance-checker`` and ``and abi-dumper``
> +utilities which can be installed via a package manager. For example::
> +
> + sudo yum install abi-compliance-checker
> + sudo yum install abi-dumper
> +
> +The syntax of the ``validate-abi.sh`` utility is::
> +
> + ./devtools/validate-abi.sh <REV1> <REV2>
> +
> +Where ``REV1`` and ``REV2`` are valid gitrevisions(7)
> +https://www.kernel.org/pub/software/scm/git/docs/gitrevisions.html
> +on the local repo.
> +
> +For example::
> +
> + # Check between the previous and latest commit:
> + ./devtools/validate-abi.sh HEAD~1 HEAD
> +
> + # Check on a specific compilation target:
> + ./devtools/validate-abi.sh -t x86_64-native-linux-gcc HEAD~1 HEAD
> +
> + # Check between two tags:
> + ./devtools/validate-abi.sh v2.0.0 v2.1.0
> +
> + # Check between git master and local topic-branch "vhost-hacking":
> + ./devtools/validate-abi.sh master vhost-hacking
> +
> +After the validation script completes (it can take a while since it need to
> +compile both tags) it will create compatibility reports in the
> +``./abi-check/compat_report`` directory. Listed incompatibilities can be found
> +as follows::
> +
> + grep -lr Incompatible abi-check/compat_reports/
> diff --git a/doc/guides/contributing/index.rst b/doc/guides/contributing/index.rst
> index e2608d3..2fefd91 100644
> --- a/doc/guides/contributing/index.rst
> +++ b/doc/guides/contributing/index.rst
> @@ -10,7 +10,8 @@ Contributor's Guidelines
>
> coding_style
> design
> - versioning
> + abi_policy
> + abi_versioning
> documentation
> patches
> vulnerability
> diff --git a/doc/guides/contributing/versioning.rst b/doc/guides/contributing/versioning.rst
> deleted file mode 100644
> index 3ab2c43..0000000
> --- a/doc/guides/contributing/versioning.rst
> +++ /dev/null
> @@ -1,591 +0,0 @@
> -.. SPDX-License-Identifier: BSD-3-Clause
> - Copyright 2018 The DPDK contributors
> -
> -DPDK ABI/API policy
> -===================
> -
> -Description
> ------------
> -
> -This document details some methods for handling ABI management in the DPDK.
> -
> -General Guidelines
> -------------------
> -
> -#. Whenever possible, ABI should be preserved
> -#. ABI/API may be changed with a deprecation process
> -#. The modification of symbols can generally be managed with versioning
> -#. Libraries or APIs marked in ``experimental`` state may change without constraint
> -#. New APIs will be marked as ``experimental`` for at least one release to allow
> - any issues found by users of the new API to be fixed quickly
> -#. The addition of symbols is generally not problematic
> -#. The removal of symbols generally is an ABI break and requires bumping of the
> - LIBABIVER macro
> -#. Updates to the minimum hardware requirements, which drop support for hardware which
> - was previously supported, should be treated as an ABI change.
> -
> -What is an ABI
> -~~~~~~~~~~~~~~
> -
> -An ABI (Application Binary Interface) is the set of runtime interfaces exposed
> -by a library. It is similar to an API (Application Programming Interface) but
> -is the result of compilation. It is also effectively cloned when applications
> -link to dynamic libraries. That is to say when an application is compiled to
> -link against dynamic libraries, it is assumed that the ABI remains constant
> -between the time the application is compiled/linked, and the time that it runs.
> -Therefore, in the case of dynamic linking, it is critical that an ABI is
> -preserved, or (when modified), done in such a way that the application is unable
> -to behave improperly or in an unexpected fashion.
> -
> -
> -ABI/API Deprecation
> --------------------
> -
> -The DPDK ABI policy
> -~~~~~~~~~~~~~~~~~~~
> -
> -ABI versions are set at the time of major release labeling, and the ABI may
> -change multiple times, without warning, between the last release label and the
> -HEAD label of the git tree.
> -
> -ABI versions, once released, are available until such time as their
> -deprecation has been noted in the Release Notes for at least one major release
> -cycle. For example consider the case where the ABI for DPDK 2.0 has been
> -shipped and then a decision is made to modify it during the development of
> -DPDK 2.1. The decision will be recorded in the Release Notes for the DPDK 2.1
> -release and the modification will be made available in the DPDK 2.2 release.
> -
> -ABI versions may be deprecated in whole or in part as needed by a given
> -update.
> -
> -Some ABI changes may be too significant to reasonably maintain multiple
> -versions. In those cases ABI's may be updated without backward compatibility
> -being provided. The requirements for doing so are:
> -
> -#. At least 3 acknowledgments of the need to do so must be made on the
> - dpdk.org mailing list.
> -
> - - The acknowledgment of the maintainer of the component is mandatory, or if
> - no maintainer is available for the component, the tree/sub-tree maintainer
> - for that component must acknowledge the ABI change instead.
> -
> - - It is also recommended that acknowledgments from different "areas of
> - interest" be sought for each deprecation, for example: from NIC vendors,
> - CPU vendors, end-users, etc.
> -
> -#. The changes (including an alternative map file) can be included with
> - deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option,
> - to provide more details about oncoming changes.
> - ``RTE_NEXT_ABI`` wrapper will be removed when it become the default ABI.
> - More preferred way to provide this information is sending the feature
> - as a separate patch and reference it in deprecation notice.
> -
> -#. A full deprecation cycle, as explained above, must be made to offer
> - downstream consumers sufficient warning of the change.
> -
> -Note that the above process for ABI deprecation should not be undertaken
> -lightly. ABI stability is extremely important for downstream consumers of the
> -DPDK, especially when distributed in shared object form. Every effort should
> -be made to preserve the ABI whenever possible. The ABI should only be changed
> -for significant reasons, such as performance enhancements. ABI breakage due to
> -changes such as reorganizing public structure fields for aesthetic or
> -readability purposes should be avoided.
> -
> -.. note::
> -
> - Updates to the minimum hardware requirements, which drop support for hardware
> - which was previously supported, should be treated as an ABI change, and
> - follow the relevant deprecation policy procedures as above: 3 acks and
> - announcement at least one release in advance.
> -
> -Examples of Deprecation Notices
> -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
> -
> -The following are some examples of ABI deprecation notices which would be
> -added to the Release Notes:
> -
> -* The Macro ``#RTE_FOO`` is deprecated and will be removed with version 2.0,
> - to be replaced with the inline function ``rte_foo()``.
> -
> -* The function ``rte_mbuf_grok()`` has been updated to include a new parameter
> - in version 2.0. Backwards compatibility will be maintained for this function
> - until the release of version 2.1
> -
> -* The members of ``struct rte_foo`` have been reorganized in release 2.0 for
> - performance reasons. Existing binary applications will have backwards
> - compatibility in release 2.0, while newly built binaries will need to
> - reference the new structure variant ``struct rte_foo2``. Compatibility will
> - be removed in release 2.2, and all applications will require updating and
> - rebuilding to the new structure at that time, which will be renamed to the
> - original ``struct rte_foo``.
> -
> -* Significant ABI changes are planned for the ``librte_dostuff`` library. The
> - upcoming release 2.0 will not contain these changes, but release 2.1 will,
> - and no backwards compatibility is planned due to the extensive nature of
> - these changes. Binaries using this library built prior to version 2.1 will
> - require updating and recompilation.
> -
> -New API replacing previous one
> -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
> -
> -If a new API proposed functionally replaces an existing one, when the new API
> -becomes non-experimental then the old one is marked with ``__rte_deprecated``.
> -Deprecated APIs are removed completely just after the next LTS.
> -
> -Reminder that old API should follow deprecation process to be removed.
> -
> -
> -Experimental APIs
> ------------------
> -
> -APIs marked as ``experimental`` are not considered part of the ABI and may
> -change without warning at any time. Since changes to APIs are most likely
> -immediately after their introduction, as users begin to take advantage of
> -those new APIs and start finding issues with them, new DPDK APIs will be
> -automatically marked as ``experimental`` to allow for a period of stabilization
> -before they become part of a tracked ABI.
> -
> -Note that marking an API as experimental is a multi step process.
> -To mark an API as experimental, the symbols which are desired to be exported
> -must be placed in an EXPERIMENTAL version block in the corresponding libraries'
> -version map script.
> -Secondly, the corresponding prototypes of those exported functions (in the
> -development header files), must be marked with the ``__rte_experimental`` tag
> -(see ``rte_compat.h``).
> -The DPDK build makefiles perform a check to ensure that the map file and the
> -C code reflect the same list of symbols.
> -This check can be circumvented by defining ``ALLOW_EXPERIMENTAL_API``
> -during compilation in the corresponding library Makefile.
> -
> -In addition to tagging the code with ``__rte_experimental``,
> -the doxygen markup must also contain the EXPERIMENTAL string,
> -and the MAINTAINERS file should note the EXPERIMENTAL libraries.
> -
> -For removing the experimental tag associated with an API, deprecation notice
> -is not required. Though, an API should remain in experimental state for at least
> -one release. Thereafter, normal process of posting patch for review to mailing
> -list can be followed.
> -
> -
> -Library versioning
> -------------------
> -
> -Downstreams might want to provide different DPDK releases at the same time to
> -support multiple consumers of DPDK linked against older and newer sonames.
> -
> -Also due to the interdependencies that DPDK libraries can have applications
> -might end up with an executable space in which multiple versions of a library
> -are mapped by ld.so.
> -
> -Think of LibA that got an ABI bump and LibB that did not get an ABI bump but is
> -depending on LibA.
> -
> -.. note::
> -
> - Application
> - \-> LibA.old
> - \-> LibB.new -> LibA.new
> -
> -That is a conflict which can be avoided by setting ``CONFIG_RTE_MAJOR_ABI``.
> -If set, the value of ``CONFIG_RTE_MAJOR_ABI`` overwrites all - otherwise per
> -library - versions defined in the libraries ``LIBABIVER``.
> -An example might be ``CONFIG_RTE_MAJOR_ABI=16.11`` which will make all libraries
> -``librte<?>.so.16.11`` instead of ``librte<?>.so.<LIBABIVER>``.
> -
> -
> -ABI versioning
> ---------------
> -
> -Versioning Macros
> -~~~~~~~~~~~~~~~~~
> -
> -When a symbol is exported from a library to provide an API, it also provides a
> -calling convention (ABI) that is embodied in its name, return type and
> -arguments. Occasionally that function may need to change to accommodate new
> -functionality or behavior. When that occurs, it is desirable to allow for
> -backward compatibility for a time with older binaries that are dynamically
> -linked to the DPDK.
> -
> -To support backward compatibility the ``rte_compat.h``
> -header file provides macros to use when updating exported functions. These
> -macros are used in conjunction with the ``rte_<library>_version.map`` file for
> -a given library to allow multiple versions of a symbol to exist in a shared
> -library so that older binaries need not be immediately recompiled.
> -
> -The macros exported are:
> -
> -* ``VERSION_SYMBOL(b, e, n)``: Creates a symbol version table entry binding
> - versioned symbol ``b@DPDK_n`` to the internal function ``b_e``.
> -
> -* ``BIND_DEFAULT_SYMBOL(b, e, n)``: Creates a symbol version entry instructing
> - the linker to bind references to symbol ``b`` to the internal symbol
> - ``b_e``.
> -
> -* ``MAP_STATIC_SYMBOL(f, p)``: Declare the prototype ``f``, and map it to the
> - fully qualified function ``p``, so that if a symbol becomes versioned, it
> - can still be mapped back to the public symbol name.
> -
> -Examples of ABI Macro use
> -^^^^^^^^^^^^^^^^^^^^^^^^^
> -
> -Updating a public API
> -_____________________
> -
> -Assume we have a function as follows
> -
> -.. code-block:: c
> -
> - /*
> - * Create an acl context object for apps to
> - * manipulate
> - */
> - struct rte_acl_ctx *
> - rte_acl_create(const struct rte_acl_param *param)
> - {
> - ...
> - }
> -
> -
> -Assume that struct rte_acl_ctx is a private structure, and that a developer
> -wishes to enhance the acl api so that a debugging flag can be enabled on a
> -per-context basis. This requires an addition to the structure (which, being
> -private, is safe), but it also requires modifying the code as follows
> -
> -.. code-block:: c
> -
> - /*
> - * Create an acl context object for apps to
> - * manipulate
> - */
> - struct rte_acl_ctx *
> - rte_acl_create(const struct rte_acl_param *param, int debug)
> - {
> - ...
> - }
> -
> -
> -Note also that, being a public function, the header file prototype must also be
> -changed, as must all the call sites, to reflect the new ABI footprint. We will
> -maintain previous ABI versions that are accessible only to previously compiled
> -binaries
> -
> -The addition of a parameter to the function is ABI breaking as the function is
> -public, and existing application may use it in its current form. However, the
> -compatibility macros in DPDK allow a developer to use symbol versioning so that
> -multiple functions can be mapped to the same public symbol based on when an
> -application was linked to it. To see how this is done, we start with the
> -requisite libraries version map file. Initially the version map file for the
> -acl library looks like this
> -
> -.. code-block:: none
> -
> - DPDK_2.0 {
> - global:
> -
> - rte_acl_add_rules;
> - rte_acl_build;
> - rte_acl_classify;
> - rte_acl_classify_alg;
> - rte_acl_classify_scalar;
> - rte_acl_create;
> - rte_acl_dump;
> - rte_acl_find_existing;
> - rte_acl_free;
> - rte_acl_ipv4vlan_add_rules;
> - rte_acl_ipv4vlan_build;
> - rte_acl_list_dump;
> - rte_acl_reset;
> - rte_acl_reset_rules;
> - rte_acl_set_ctx_classify;
> -
> - local: *;
> - };
> -
> -This file needs to be modified as follows
> -
> -.. code-block:: none
> -
> - DPDK_2.0 {
> - global:
> -
> - rte_acl_add_rules;
> - rte_acl_build;
> - rte_acl_classify;
> - rte_acl_classify_alg;
> - rte_acl_classify_scalar;
> - rte_acl_create;
> - rte_acl_dump;
> - rte_acl_find_existing;
> - rte_acl_free;
> - rte_acl_ipv4vlan_add_rules;
> - rte_acl_ipv4vlan_build;
> - rte_acl_list_dump;
> - rte_acl_reset;
> - rte_acl_reset_rules;
> - rte_acl_set_ctx_classify;
> -
> - local: *;
> - };
> -
> - DPDK_2.1 {
> - global:
> - rte_acl_create;
> -
> - } DPDK_2.0;
> -
> -The addition of the new block tells the linker that a new version node is
> -available (DPDK_2.1), which contains the symbol rte_acl_create, and inherits the
> -symbols from the DPDK_2.0 node. This list is directly translated into a list of
> -exported symbols when DPDK is compiled as a shared library
> -
> -Next, we need to specify in the code which function map to the rte_acl_create
> -symbol at which versions. First, at the site of the initial symbol definition,
> -we need to update the function so that it is uniquely named, and not in conflict
> -with the public symbol name
> -
> -.. code-block:: c
> -
> - struct rte_acl_ctx *
> - -rte_acl_create(const struct rte_acl_param *param)
> - +rte_acl_create_v20(const struct rte_acl_param *param)
> - {
> - size_t sz;
> - struct rte_acl_ctx *ctx;
> - ...
> -
> -Note that the base name of the symbol was kept intact, as this is conducive to
> -the macros used for versioning symbols. That is our next step, mapping this new
> -symbol name to the initial symbol name at version node 2.0. Immediately after
> -the function, we add this line of code
> -
> -.. code-block:: c
> -
> - VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
> -
> -Remembering to also add the rte_compat.h header to the requisite c file where
> -these changes are being made. The above macro instructs the linker to create a
> -new symbol ``rte_acl_create@DPDK_2.0``, which matches the symbol created in older
> -builds, but now points to the above newly named function. We have now mapped
> -the original rte_acl_create symbol to the original function (but with a new
> -name)
> -
> -Next, we need to create the 2.1 version of the symbol. We create a new function
> -name, with a different suffix, and implement it appropriately
> -
> -.. code-block:: c
> -
> - struct rte_acl_ctx *
> - rte_acl_create_v21(const struct rte_acl_param *param, int debug);
> - {
> - struct rte_acl_ctx *ctx = rte_acl_create_v20(param);
> -
> - ctx->debug = debug;
> -
> - return ctx;
> - }
> -
> -This code serves as our new API call. Its the same as our old call, but adds
> -the new parameter in place. Next we need to map this function to the symbol
> -``rte_acl_create@DPDK_2.1``. To do this, we modify the public prototype of the call
> -in the header file, adding the macro there to inform all including applications,
> -that on re-link, the default rte_acl_create symbol should point to this
> -function. Note that we could do this by simply naming the function above
> -rte_acl_create, and the linker would chose the most recent version tag to apply
> -in the version script, but we can also do this in the header file
> -
> -.. code-block:: c
> -
> - struct rte_acl_ctx *
> - -rte_acl_create(const struct rte_acl_param *param);
> - +rte_acl_create(const struct rte_acl_param *param, int debug);
> - +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
> -
> -The BIND_DEFAULT_SYMBOL macro explicitly tells applications that include this
> -header, to link to the rte_acl_create_v21 function and apply the DPDK_2.1
> -version node to it. This method is more explicit and flexible than just
> -re-implementing the exact symbol name, and allows for other features (such as
> -linking to the old symbol version by default, when the new ABI is to be opt-in
> -for a period.
> -
> -One last thing we need to do. Note that we've taken what was a public symbol,
> -and duplicated it into two uniquely and differently named symbols. We've then
> -mapped each of those back to the public symbol ``rte_acl_create`` with different
> -version tags. This only applies to dynamic linking, as static linking has no
> -notion of versioning. That leaves this code in a position of no longer having a
> -symbol simply named ``rte_acl_create`` and a static build will fail on that
> -missing symbol.
> -
> -To correct this, we can simply map a function of our choosing back to the public
> -symbol in the static build with the ``MAP_STATIC_SYMBOL`` macro. Generally the
> -assumption is that the most recent version of the symbol is the one you want to
> -map. So, back in the C file where, immediately after ``rte_acl_create_v21`` is
> -defined, we add this
> -
> -.. code-block:: c
> -
> - struct rte_acl_ctx *
> - rte_acl_create_v21(const struct rte_acl_param *param, int debug)
> - {
> - ...
> - }
> - MAP_STATIC_SYMBOL(struct rte_acl_ctx *rte_acl_create(const struct rte_acl_param *param, int debug), rte_acl_create_v21);
> -
> -That tells the compiler that, when building a static library, any calls to the
> -symbol ``rte_acl_create`` should be linked to ``rte_acl_create_v21``
> -
> -That's it, on the next shared library rebuild, there will be two versions of
> -rte_acl_create, an old DPDK_2.0 version, used by previously built applications,
> -and a new DPDK_2.1 version, used by future built applications.
> -
> -
> -Deprecating part of a public API
> -________________________________
> -
> -Lets assume that you've done the above update, and after a few releases have
> -passed you decide you would like to retire the old version of the function.
> -After having gone through the ABI deprecation announcement process, removal is
> -easy. Start by removing the symbol from the requisite version map file:
> -
> -.. code-block:: none
> -
> - DPDK_2.0 {
> - global:
> -
> - rte_acl_add_rules;
> - rte_acl_build;
> - rte_acl_classify;
> - rte_acl_classify_alg;
> - rte_acl_classify_scalar;
> - rte_acl_dump;
> - - rte_acl_create
> - rte_acl_find_existing;
> - rte_acl_free;
> - rte_acl_ipv4vlan_add_rules;
> - rte_acl_ipv4vlan_build;
> - rte_acl_list_dump;
> - rte_acl_reset;
> - rte_acl_reset_rules;
> - rte_acl_set_ctx_classify;
> -
> - local: *;
> - };
> -
> - DPDK_2.1 {
> - global:
> - rte_acl_create;
> - } DPDK_2.0;
> -
> -
> -Next remove the corresponding versioned export.
> -
> -.. code-block:: c
> -
> - -VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
> -
> -
> -Note that the internal function definition could also be removed, but its used
> -in our example by the newer version _v21, so we leave it in place. This is a
> -coding style choice.
> -
> -Lastly, we need to bump the LIBABIVER number for this library in the Makefile to
> -indicate to applications doing dynamic linking that this is a later, and
> -possibly incompatible library version:
> -
> -.. code-block:: c
> -
> - -LIBABIVER := 1
> - +LIBABIVER := 2
> -
> -Deprecating an entire ABI version
> -_________________________________
> -
> -While removing a symbol from and ABI may be useful, it is often more practical
> -to remove an entire version node at once. If a version node completely
> -specifies an API, then removing part of it, typically makes it incomplete. In
> -those cases it is better to remove the entire node
> -
> -To do this, start by modifying the version map file, such that all symbols from
> -the node to be removed are merged into the next node in the map
> -
> -In the case of our map above, it would transform to look as follows
> -
> -.. code-block:: none
> -
> - DPDK_2.1 {
> - global:
> -
> - rte_acl_add_rules;
> - rte_acl_build;
> - rte_acl_classify;
> - rte_acl_classify_alg;
> - rte_acl_classify_scalar;
> - rte_acl_dump;
> - rte_acl_create
> - rte_acl_find_existing;
> - rte_acl_free;
> - rte_acl_ipv4vlan_add_rules;
> - rte_acl_ipv4vlan_build;
> - rte_acl_list_dump;
> - rte_acl_reset;
> - rte_acl_reset_rules;
> - rte_acl_set_ctx_classify;
> -
> - local: *;
> - };
> -
> -Then any uses of BIND_DEFAULT_SYMBOL that pointed to the old node should be
> -updated to point to the new version node in any header files for all affected
> -symbols.
> -
> -.. code-block:: c
> -
> - -BIND_DEFAULT_SYMBOL(rte_acl_create, _v20, 2.0);
> - +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
> -
> -Lastly, any VERSION_SYMBOL macros that point to the old version node should be
> -removed, taking care to keep, where need old code in place to support newer
> -versions of the symbol.
> -
> -
> -Running the ABI Validator
> --------------------------
> -
> -The ``devtools`` directory in the DPDK source tree contains a utility program,
> -``validate-abi.sh``, for validating the DPDK ABI based on the Linux `ABI
> -Compliance Checker
> -<http://ispras.linuxbase.org/index.php/ABI_compliance_checker>`_.
> -
> -This has a dependency on the ``abi-compliance-checker`` and ``and abi-dumper``
> -utilities which can be installed via a package manager. For example::
> -
> - sudo yum install abi-compliance-checker
> - sudo yum install abi-dumper
> -
> -The syntax of the ``validate-abi.sh`` utility is::
> -
> - ./devtools/validate-abi.sh <REV1> <REV2>
> -
> -Where ``REV1`` and ``REV2`` are valid gitrevisions(7)
> -https://www.kernel.org/pub/software/scm/git/docs/gitrevisions.html
> -on the local repo.
> -
> -For example::
> -
> - # Check between the previous and latest commit:
> - ./devtools/validate-abi.sh HEAD~1 HEAD
> -
> - # Check on a specific compilation target:
> - ./devtools/validate-abi.sh -t x86_64-native-linux-gcc HEAD~1 HEAD
> -
> - # Check between two tags:
> - ./devtools/validate-abi.sh v2.0.0 v2.1.0
> -
> - # Check between git master and local topic-branch "vhost-hacking":
> - ./devtools/validate-abi.sh master vhost-hacking
> -
> -After the validation script completes (it can take a while since it need to
> -compile both tags) it will create compatibility reports in the
> -``./abi-check/compat_report`` directory. Listed incompatibilities can be found
> -as follows::
> -
> - grep -lr Incompatible abi-check/compat_reports/
^ permalink raw reply [relevance 4%]
* Re: [dpdk-dev] [PATCH v4 1/4] doc: separate versioning.rst into version and policy
2019-09-25 12:24 5% ` Neil Horman
2019-09-25 13:01 3% ` Ray Kinsella
@ 2019-09-27 15:22 4% ` Ray Kinsella
1 sibling, 0 replies; 200+ results
From: Ray Kinsella @ 2019-09-27 15:22 UTC (permalink / raw)
To: Neil Horman
Cc: dev, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, maxime.coquelin,
john.mcnamara, marko.kovacevic, hemant.agrawal, ktraynor
On 25/09/2019 13:24, Neil Horman wrote:
> On Wed, Sep 25, 2019 at 11:23:53AM +0100, Ray Kinsella wrote:
> This seems..confusing. In patch 0:
> =================================================================
> * DPDK v20 is declared as the supported ABI version for one year, aligned with
> the DPDK v19.11 (LTS) release. All library sonames are updated to reflect the
> new ABI version, e.g. librte_eal.so.20, librte_acl.so.20...
> * DPDK v20.02 .. v20.08 releases are ABI compatible with the DPDK v20 ABI. ABI
> changes are permitted from DPDK v20.02 onwards, with the condition that ABI
> compatibility with DPDK v20 is preserved.
> * DPDK v21 is declared as the new supported ABI version for two years, aligned
> with the DPDK v20.11 (LTS) release. The DPDK v20 ABI is now depreciated,
> library sonames are updated to v21 and ABI compatibility breaking changes may
> be introduced.
> ===================================================================
Actually Neil,
Apologies now I am taking a closer look at your email.
I understand what is going on.
Patch 1/4 separates the ABI Policy from the detail of ABI Version
management, but it is still old policy that your where reading at this
point, hence all the references to DPDK 2.0 and why it doesn't
Patch 2/4 is the _new policy_, where you will see that all references to
2.0 have been removed.
You other points around the needing a taxonomy still are 100% valid
though and I am working on adding a diagram.
Ray K
^ permalink raw reply [relevance 4%]
* [dpdk-dev] Minutes of Technical Board Meeting, 2019-09-25
@ 2019-09-27 14:02 5% Bruce Richardson
0 siblings, 0 replies; 200+ results
From: Bruce Richardson @ 2019-09-27 14:02 UTC (permalink / raw)
To: dev; +Cc: techboard
Members Attending
-----------------
Bruce (chair)
Ferruh
Hemant
Jerin
Konstantin
Maxime
Stephen
Thomas
NOTE: The technical board meetings every second Wednesday on IRC channel
#dpdk-board, at 3pm UTC. Meetings are public and DPDK community members
are welcome to attend.
NOTE: Next meeting will be on Wednesday 2019-10-09 @3pm UTC, and will be
chaired by Ferruh.
Summary of meeting:
-------------------
1. Removal of examples (standing agenda item):
Bruce reported that he is working on this, with a patchset in
progress for submission upstream shortly
2. SPDX licenses (standing agenda item):
Hemant provided an update on the gaps to be closed for achieving
use of SPDX tags alone (i.e. no license text in files) in 19.11.
Key gaps:
* Issue identified with pmdinfogen using GPL derived code, meaning
it needs a license exception. Tech board has approved the
exception and the request will now be passed to the governing
board for final approval. Thereafter an update will be made to
the file "license/exceptions.txt" in the DPDK repo
* Two drivers and various miscelaneous files are missing license
header updates. Hemant and Stephen will work together to produce
complete list of files needing update and will ping the
maintainers of the drivers in question to see if they can get a
mass update from them.
3. ABI Policy Discussion Following Userspace:
Ray Kinsella has provided an update via email on the output of the
discussions at userspace. Discussion followed on how ABI breaks are
to be managed, including use of staging trees, as well as
discussion on the releases for which the ABI policy will be
new complied with. Also discussed was allowable changes for 19.11
in advance of the policy coming into effect. At the end of the
discussion a number of key proposals were agreed upon:
1. New DPDK ABI policy sent and discussed previously is approved
for use in a future DPDK release. [1]
2. DPDK 19.11 release will remain targetted at 19.11 and will not
be postponed to a 19.12 release because of ABI changes.
3. The new ABI policy will take effect from 19.11 release, meaning
that DPDK 20.02 and 20.05 releases must be ABI compatible with
the 19.11 release.
The decision to introduce policy from 19.11 may be reviewed at a
future techboard meeting in advance of the 19.11 release, if
maintainers/contributors identify a serious issue that requires
additional changes to the existing ABI ahead of stabilization, and
which cannot be done in the 19.11 timeframe.
** It is therefore STRONGLY RECOMMENDED by the technical board,
that all maintainers, committers and contributors, assess the
impact of the new ABI policy on their planned work for next year,
and flag any major issues to the technical board ahead of the 19.11
release.**
4. Technical board composition.
Discussion of adding new members to the technical board was not
possible at the meeting due to time constraints. This will be
discussed by the board over email, and any updates to the technical
board membership will be sent out to the community via email once
the discussion has concluded.
[1] http://patches.dpdk.org/project/dpdk/list/?series=6524
^ permalink raw reply [relevance 5%]
* Re: [dpdk-dev] [RFC PATCH 1/9] security: introduce CPU Crypto action type and API
2019-09-25 18:24 4% ` Ananyev, Konstantin
@ 2019-09-27 9:26 0% ` Akhil Goyal
0 siblings, 0 replies; 200+ results
From: Akhil Goyal @ 2019-09-27 9:26 UTC (permalink / raw)
To: Ananyev, Konstantin, 'dev@dpdk.org',
De Lara Guarch, Pablo, 'Thomas Monjalon'
Cc: Zhang, Roy Fan, Doherty, Declan, 'Anoob Joseph'
Hi Konstantin,
> -----Original Message-----
> From: Ananyev, Konstantin <konstantin.ananyev@intel.com>
> Sent: Wednesday, September 25, 2019 11:54 PM
> To: Akhil Goyal <akhil.goyal@nxp.com>; 'dev@dpdk.org' <dev@dpdk.org>; De
> Lara Guarch, Pablo <pablo.de.lara.guarch@intel.com>; 'Thomas Monjalon'
> <thomas@monjalon.net>
> Cc: Zhang, Roy Fan <roy.fan.zhang@intel.com>; Doherty, Declan
> <declan.doherty@intel.com>; 'Anoob Joseph' <anoobj@marvell.com>
> Subject: RE: [RFC PATCH 1/9] security: introduce CPU Crypto action type and API
>
>
> > > > > > > > > > This action type allows the burst of symmetric crypto workload
> using
> > > > the
> > > > > > > > same
> > > > > > > > > > algorithm, key, and direction being processed by CPU cycles
> > > > > > synchronously.
> > > > > > > > > > This flexible action type does not require external hardware
> > > > involvement,
> > > > > > > > > > having the crypto workload processed synchronously, and is
> more
> > > > > > > > performant
> > > > > > > > > > than Cryptodev SW PMD due to the saved cycles on removed
> "async
> > > > > > mode
> > > > > > > > > > simulation" as well as 3 cacheline access of the crypto ops.
> > > > > > > > >
> > > > > > > > > Does that mean application will not call the
> cryptodev_enqueue_burst
> > > > and
> > > > > > > > corresponding dequeue burst.
> > > > > > > >
> > > > > > > > Yes, instead it just call rte_security_process_cpu_crypto_bulk(...)
> > > > > > > >
> > > > > > > > > It would be a new API something like process_packets and it will
> have
> > > > the
> > > > > > > > crypto processed packets while returning from the API?
> > > > > > > >
> > > > > > > > Yes, though the plan is that API will operate on raw data buffers,
> not
> > > > mbufs.
> > > > > > > >
> > > > > > > > >
> > > > > > > > > I still do not understand why we cannot do with the conventional
> > > > crypto lib
> > > > > > > > only.
> > > > > > > > > As far as I can understand, you are not doing any protocol
> processing
> > > > or
> > > > > > any
> > > > > > > > value add
> > > > > > > > > To the crypto processing. IMO, you just need a synchronous
> crypto
> > > > > > processing
> > > > > > > > API which
> > > > > > > > > Can be defined in cryptodev, you don't need to re-create a crypto
> > > > session
> > > > > > in
> > > > > > > > the name of
> > > > > > > > > Security session in the driver just to do a synchronous processing.
> > > > > > > >
> > > > > > > > I suppose your question is why not to have
> > > > > > > > rte_crypot_process_cpu_crypto_bulk(...) instead?
> > > > > > > > The main reason is that would require disruptive changes in existing
> > > > > > cryptodev
> > > > > > > > API
> > > > > > > > (would cause ABI/API breakage).
> > > > > > > > Session for RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO need
> some
> > > > extra
> > > > > > > > information
> > > > > > > > that normal crypto_sym_xform doesn't contain
> > > > > > > > (cipher offset from the start of the buffer, might be something extra
> in
> > > > > > future).
> > > > > > >
> > > > > > > Cipher offset will be part of rte_crypto_op.
> > > > > >
> > > > > > fill/read (+ alloc/free) is one of the main things that slowdown current
> > > > crypto-op
> > > > > > approach.
> > > > > > That's why the general idea - have all data that wouldn't change from
> packet
> > > > to
> > > > > > packet
> > > > > > included into the session and setup it once at session_init().
> > > > >
> > > > > I agree that you cannot use crypto-op.
> > > > > You can have the new API in crypto.
> > > > > As per the current patch, you only need cipher_offset which you can have
> it as
> > > > a parameter until
> > > > > You get it approved in the crypto xform. I believe it will be beneficial in
> case of
> > > > other crypto cases as well.
> > > > > We can have cipher offset at both places(crypto-op and cipher_xform). It
> will
> > > > give flexibility to the user to
> > > > > override it.
> > > >
> > > > After having another thought on your proposal:
> > > > Probably we can introduce new rte_crypto_sym_xform_types for CPU
> related
> > > > stuff here?
> > >
> > > I also thought of adding new xforms, but that wont serve the purpose for
> may be all the cases.
> > > You would be needing all information currently available in the current
> xforms.
> > > So if you are adding new fields in the new xform, the size will be more than
> that of the union of xforms.
> > > ABI breakage would still be there.
> > >
> > > If you think a valid compression of the AEAD xform can be done, then that
> can be done for each of the
> > > Xforms and we can have a solution to this issue.
> >
> > I think that we can re-use iv.offset for our purposes (for crypto offset).
> > So for now we can make that path work without any ABI breakage.
> > Fan, please feel free to correct me here, if I missed something.
> > If in future we would need to add some extra information it might
> > require ABI breakage, though by now I don't envision anything particular to
> add.
> > Anyway, if there is no objection to go that way, we can try to make
> > these changes for v2.
> >
>
> Actually, after looking at it more deeply it appears not that easy as I thought it
> would be :)
> Below is a very draft version of proposed API additions.
> I think it avoids ABI breakages right now and provides enough flexibility for
> future extensions (if any).
> For now, it doesn't address your comments about naming conventions (_CPU_
> vs _SYNC_) , etc.
> but I suppose is comprehensive enough to provide a main idea beyond it.
> Akhil and other interested parties, please try to review and provide feedback
> ASAP,
> as related changes would take some time and we still like to hit 19.11 deadline.
> Konstantin
>
> diff --git a/lib/librte_cryptodev/rte_crypto_sym.h
> b/lib/librte_cryptodev/rte_crypto_sym.h
> index bc8da2466..c03069e23 100644
> --- a/lib/librte_cryptodev/rte_crypto_sym.h
> +++ b/lib/librte_cryptodev/rte_crypto_sym.h
> @@ -103,6 +103,9 @@ rte_crypto_cipher_operation_strings[];
> *
> * This structure contains data relating to Cipher (Encryption and Decryption)
> * use to create a session.
> + * Actually I was wrong saying that we don't have free space inside xforms.
> + * Making key struct packed (see below) allow us to regain 6B that could be
> + * used for future extensions.
> */
> struct rte_crypto_cipher_xform {
> enum rte_crypto_cipher_operation op;
> @@ -116,7 +119,25 @@ struct rte_crypto_cipher_xform {
> struct {
> const uint8_t *data; /**< pointer to key data */
> uint16_t length; /**< key length in bytes */
> - } key;
> + } __attribute__((__packed__)) key;
> +
> + /**
> + * offset for cipher to start within user provided data buffer.
> + * Fan suggested another (and less space consuming way) -
> + * reuse iv.offset space below, by changing:
> + * struct {uint16_t offset, length;} iv;
> + * to uunamed union:
> + * union {
> + * struct {uint16_t offset, length;} iv;
> + * struct {uint16_t iv_len, crypto_offset} cpu_crypto_param;
> + * };
> + * Both approaches seems ok to me in general.
No strong opinions here. OK with this one.
> + * Comments/suggestions are welcome.
> + */
> + uint16_t offset;
> +
> + uint8_t reserved1[4];
> +
> /**< Cipher key
> *
> * For the RTE_CRYPTO_CIPHER_AES_F8 mode of operation, key.data will
> @@ -284,7 +305,7 @@ struct rte_crypto_auth_xform {
> struct {
> const uint8_t *data; /**< pointer to key data */
> uint16_t length; /**< key length in bytes */
> - } key;
> + } __attribute__((__packed__)) key;
> /**< Authentication key data.
> * The authentication key length MUST be less than or equal to the
> * block size of the algorithm. It is the callers responsibility to
> @@ -292,6 +313,8 @@ struct rte_crypto_auth_xform {
> * (for example RFC 2104, FIPS 198a).
> */
>
> + uint8_t reserved1[6];
> +
> struct {
> uint16_t offset;
> /**< Starting point for Initialisation Vector or Counter,
> @@ -376,7 +399,12 @@ struct rte_crypto_aead_xform {
> struct {
> const uint8_t *data; /**< pointer to key data */
> uint16_t length; /**< key length in bytes */
> - } key;
> + } __attribute__((__packed__)) key;
> +
> + /** offset for cipher to start within data buffer */
> + uint16_t cipher_offset;
> +
> + uint8_t reserved1[4];
>
> struct {
> uint16_t offset;
> diff --git a/lib/librte_cryptodev/rte_cryptodev.h
> b/lib/librte_cryptodev/rte_cryptodev.h
> index e175b838c..c0c7bfed7 100644
> --- a/lib/librte_cryptodev/rte_cryptodev.h
> +++ b/lib/librte_cryptodev/rte_cryptodev.h
> @@ -1272,6 +1272,101 @@ void *
> rte_cryptodev_sym_session_get_user_data(
> struct rte_cryptodev_sym_session *sess);
>
> +/*
> + * After several thoughts decided not to try to squeeze CPU_CRYPTO
> + * into existing rte_crypto_sym_session structure/API, but instead
> + * introduce an extentsion to it via new fully opaque
> + * struct rte_crypto_cpu_sym_session and additional related API.
What all things do we need to squeeze?
In this proposal I do not see the new struct cpu_sym_session defined here.
I believe you will have same lib API/struct for cpu_sym_session and sym_session.
I am not sure if that would be needed.
It would be internal to the driver that if synchronous processing is supported(from feature flag) and
Have relevant fields in xform(the newly added ones which are packed as per your suggestions) set,
It will create that type of session.
> + * Main points:
> + * - Current crypto-dev API is reasonably mature and it is desirable
> + * to keep it unchanged (API/ABI stability). From other side, this
> + * new sync API is new one and probably would require extra changes.
> + * Having it as a new one allows to mark it as experimental, without
> + * affecting existing one.
> + * - Fully opaque cpu_sym_session structure gives more flexibility
> + * to the PMD writers and again allows to avoid ABI breakages in future.
> + * - process() function per set of xforms
> + * allows to expose different process() functions for different
> + * xform combinations. PMD writer can decide, does he wants to
> + * push all supported algorithms into one process() function,
> + * or spread it across several ones.
> + * I.E. More flexibility for PMD writer.
Which process function should be chosen is internal to PMD, how would that info
be visible to the application or the library. These will get stored in the session private
data. It would be upto the PMD writer, to store the per session process function in
the session private data.
Process function would be a dev ops just like enc/deq operations and it should call
The respective process API stored in the session private data.
I am not sure if you would need a new session init API for this as nothing would be visible to
the app or lib.
> + * - Not storing process() pointer inside the session -
> + * Allows user to choose does he want to store a process() pointer
> + * per session, or per group of sessions for that device that share
> + * the same input xforms. I.E. extra flexibility for the user,
> + * plus allows us to keep cpu_sym_session totally opaque, see above.
If multiple sessions need to be processed via the same process function,
PMD would save the same process in all the sessions, I don't think there would
be any perf overhead with that.
> + * Sketched usage model:
> + * ....
> + * /* control path, alloc/init session */
> + * int32_t sz = rte_crypto_cpu_sym_session_size(dev_id, &xform);
> + * struct rte_crypto_cpu_sym_session *ses = user_alloc(..., sz);
> + * rte_crypto_cpu_sym_process_t process =
> + * rte_crypto_cpu_sym_session_func(dev_id, &xform);
> + * rte_crypto_cpu_sym_session_init(dev_id, ses, &xform);
> + * ...
> + * /* data-path*/
> + * process(ses, ....);
> + * ....
> + * /* control path, termiante/free session */
> + * rte_crypto_cpu_sym_session_fini(dev_id, ses);
> + */
> +
> +/**
> + * vector structure, contains pointer to vector array and the length
> + * of the array
> + */
> +struct rte_crypto_vec {
> + struct iovec *vec;
> + uint32_t num;
> +};
> +
> +/*
> + * Data-path bulk process crypto function.
> + */
> +typedef void (*rte_crypto_cpu_sym_process_t)(
> + struct rte_crypto_cpu_sym_session *sess,
> + struct rte_crypto_vec buf[], void *iv[], void *aad[],
> + void *digest[], int status[], uint32_t num);
> +/*
> + * for given device return process function specific to input xforms
> + * on error - return NULL and set rte_errno value.
> + * Note that for same input xfroms for the same device should return
> + * the same process function.
> + */
> +__rte_experimental
> +rte_crypto_cpu_sym_process_t
> +rte_crypto_cpu_sym_session_func(uint8_t dev_id,
> + const struct rte_crypto_sym_xform *xforms);
> +
> +/*
> + * Return required session size in bytes for given set of xforms.
> + * if xforms == NULL, then return the max possible session size,
> + * that would fit session for any supported by the device algorithm.
> + * if CPU mode is not supported at all, or requeted in xform
> + * algorithm is not supported, then return -ENOTSUP.
> + */
> +__rte_experimental
> +int
> +rte_crypto_cpu_sym_session_size(uint8_t dev_id,
> + const struct rte_crypto_sym_xform *xforms);
> +
> +/*
> + * Initialize session.
> + * It is caller responsibility to allocate enough space for it.
> + * See rte_crypto_cpu_sym_session_size above.
> + */
> +__rte_experimental
> +int rte_crypto_cpu_sym_session_init(uint8_t dev_id,
> + struct rte_crypto_cpu_sym_session *sess,
> + const struct rte_crypto_sym_xform *xforms);
> +
> +__rte_experimental
> +void
> +rte_crypto_cpu_sym_session_fini(uint8_t dev_id,
> + struct rte_crypto_cpu_sym_session *sess);
> +
> +
> #ifdef __cplusplus
> }
> #endif
> diff --git a/lib/librte_cryptodev/rte_cryptodev_pmd.h
> b/lib/librte_cryptodev/rte_cryptodev_pmd.h
> index defe05ea0..ed7e63fab 100644
> --- a/lib/librte_cryptodev/rte_cryptodev_pmd.h
> +++ b/lib/librte_cryptodev/rte_cryptodev_pmd.h
> @@ -310,6 +310,20 @@ typedef void (*cryptodev_sym_free_session_t)(struct
> rte_cryptodev *dev,
> typedef void (*cryptodev_asym_free_session_t)(struct rte_cryptodev *dev,
> struct rte_cryptodev_asym_session *sess);
>
> +typedef int (*cryptodev_cpu_sym_session_size_t) (struct rte_cryptodev *dev,
> + const struct rte_crypto_sym_xform *xforms);
> +
> +typedef int (*cryptodev_cpu_sym_session_init_t) (struct rte_cryptodev *dev,
> + struct rte_crypto_cpu_sym_session *sess,
> + const struct rte_crypto_sym_xform *xforms);
> +
> +typedef void (*cryptodev_cpu_sym_session_fini_t) (struct rte_cryptodev *dev,
> + struct rte_crypto_cpu_sym_session *sess);
> +
> +typedef rte_crypto_cpu_sym_process_t (*cryptodev_cpu_sym_session_func_t)
> (
> + struct rte_cryptodev *dev,
> + const struct rte_crypto_sym_xform *xforms);
> +
> /** Crypto device operations function pointer table */
> struct rte_cryptodev_ops {
> cryptodev_configure_t dev_configure; /**< Configure device. */
> @@ -343,6 +357,11 @@ struct rte_cryptodev_ops {
> /**< Clear a Crypto sessions private data. */
> cryptodev_asym_free_session_t asym_session_clear;
> /**< Clear a Crypto sessions private data. */
> +
> + cryptodev_cpu_sym_session_size_t sym_cpu_session_get_size;
> + cryptodev_cpu_sym_session_func_t sym_cpu_session_get_func;
> + cryptodev_cpu_sym_session_init_t sym_cpu_session_init;
> + cryptodev_cpu_sym_session_fini_t sym_cpu_session_fini;
> };
>
>
>
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [PATCH v1 0/4] get Rx/Tx packet burst mode information
2019-09-26 17:15 3% ` Stephen Hemminger
2019-09-26 17:36 0% ` Ferruh Yigit
@ 2019-09-27 1:17 0% ` Wang, Haiyue
1 sibling, 0 replies; 200+ results
From: Wang, Haiyue @ 2019-09-27 1:17 UTC (permalink / raw)
To: Stephen Hemminger
Cc: dev, Yigit, Ferruh, Ye, Xiaolong, Kinsella, Ray, Iremonger,
Bernard, Sun, Chenmin
> -----Original Message-----
> From: Stephen Hemminger [mailto:stephen@networkplumber.org]
> Sent: Friday, September 27, 2019 01:15
> To: Wang, Haiyue <haiyue.wang@intel.com>
> Cc: dev@dpdk.org; Yigit, Ferruh <ferruh.yigit@intel.com>; Ye, Xiaolong <xiaolong.ye@intel.com>;
> Kinsella, Ray <ray.kinsella@intel.com>; Iremonger, Bernard <bernard.iremonger@intel.com>; Sun, Chenmin
> <chenmin.sun@intel.com>
> Subject: Re: [dpdk-dev] [PATCH v1 0/4] get Rx/Tx packet burst mode information
>
> On Thu, 26 Sep 2019 16:36:09 +0000
> "Wang, Haiyue" <haiyue.wang@intel.com> wrote:
>
> > Hi Stephen,
> >
> > > -----Original Message-----
> > > From: Stephen Hemminger [mailto:stephen@networkplumber.org]
> > > Sent: Thursday, September 26, 2019 23:57
> > > To: Wang, Haiyue <haiyue.wang@intel.com>
> > > Cc: dev@dpdk.org; Yigit, Ferruh <ferruh.yigit@intel.com>; Ye, Xiaolong <xiaolong.ye@intel.com>;
> > > Kinsella, Ray <ray.kinsella@intel.com>; Iremonger, Bernard <bernard.iremonger@intel.com>; Sun,
> Chenmin
> > > <chenmin.sun@intel.com>
> > > Subject: Re: [dpdk-dev] [PATCH v1 0/4] get Rx/Tx packet burst mode information
> > >
> > > On Thu, 26 Sep 2019 19:48:14 +0800
> > > Haiyue Wang <haiyue.wang@intel.com> wrote:
> > >
> > > > RFCv3 -> v1:
> > > > https://patchwork.dpdk.org/patch/59103/
> > > > https://patchwork.dpdk.org/patch/59104/
> > > > https://patchwork.dpdk.org/patch/59105/
> > > > https://patchwork.dpdk.org/patch/59106/
> > > > 1). Use the function 'rte_bsf64' to iterate the options for
> > > > getting the name.
> > > >
> > > > Haiyue Wang (4):
> > > > ethdev: add the API for getting burst mode information
> > > > net/i40e: support to get the Rx/Tx burst mode
> > > > net/ice: support to get the Rx/Tx burst mode
> > > > app/testpmd: show the Rx/Tx burst mode description
> > > >
> > > > app/test-pmd/config.c | 29 +++++++++
> > > > doc/guides/rel_notes/release_19_11.rst | 9 +++
> > > > drivers/net/i40e/i40e_ethdev.c | 2 +
> > > > drivers/net/i40e/i40e_ethdev.h | 4 ++
> > > > drivers/net/i40e/i40e_rxtx.c | 72 +++++++++++++++++++++
> > > > drivers/net/ice/ice_ethdev.c | 2 +
> > > > drivers/net/ice/ice_rxtx.c | 54 ++++++++++++++++
> > > > drivers/net/ice/ice_rxtx.h | 4 ++
> > > > lib/librte_ethdev/rte_ethdev.c | 75 ++++++++++++++++++++++
> > > > lib/librte_ethdev/rte_ethdev.h | 82 ++++++++++++++++++++++++
> > > > lib/librte_ethdev/rte_ethdev_core.h | 5 ++
> > > > lib/librte_ethdev/rte_ethdev_version.map | 5 ++
> > > > 12 files changed, 343 insertions(+)
> > > >
> > >
> > > A couple of meta comments:
> > > 1) Could this be part of dev_info_get somehow?
> > >
> >
> > https://patchwork.dpdk.org/patch/57624/
> > 'Think of a better way that doesn't break ABI.' ;-)
>
> That comment was made relative to 19.08, but 19.11 is the time where
> API/ABI breakage is allowed.
>
Since 'rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info)'
focuses on 'port' level, the new rx/tx_burst_mode API can support 'queue' level:
'rte_eth_tx_burst_mode_get(uint16_t port_id, uint16_t queue_id ...', in other
words, PMD can optimize their queues in Vector/Scalar/... modes for each queue,
not have to just one mode for all queues at the same time, this API can return
"Per Queue" information.
> > > 2) Why should application care? Is this just a test hook?
> >
> > https://patches.dpdk.org/cover/57623/
> > This is from FD.io VPP's bug, and finally, we come out
> > this API for application accessing the burst mode information.
> > It can be used as a simple trace or something like performance
> > analysis like why slow ? Not in vector, anyway, application can
> > get this burst mode information now, not just open PMD debug log
> > level.
>
> From an architecture perspective, diagnostics are good but VPP is probably
> taking that too far. It is possible to expose local symbols if they
> want to keep using dlsym() by adjusting linker flags. It is more that VPP
> is stripping everything. Since VPP has chosen to go their own
> way is fixable inside VPP without changing DPDK. Also, long term VPP is
> going away from using DPDK drivers. Probably soon they will have their
> own drivers for i40e and ice anyway.
>
>
> The basis of my concern is that this is one of those kind of API's
> that creates long term technical debt around supporting it as other
> things change.
At first, we use 'string format' to make VPP happy, now, we use bit-fields
for general use.
People come, people go, even VPP left DPDK, now, testpmd is the first
user, and I think it is good for 'test' the PMD with friendly information:
testpmd> show rxq info 0 0
********************* Infos for port 0 , RX queue 0 *********************
Mempool: mbuf_pool_socket_0
RX prefetch threshold: 0
RX host threshold: 0
RX writeback threshold: 0
RX free threshold: 32
RX drop packets: off
RX deferred start: off
RX scattered packets: off
Number of RXDs: 1024
Burst mode: Vector AVX2 =============> direct information, not have to check the
code and open debug to make sure every setting
is right.
testpmd> show txq info 0 0
********************* Infos for port 0 , TX queue 0 *********************
TX prefetch threshold: 32
TX host threshold: 0
TX writeback threshold: 0
TX RS threshold: 32
TX free threshold: 32
TX deferred start: off
Number of TXDs: 1024
Burst mode: Vector AVX2 =============>
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [PATCH v2 15/17] net/hinic: add hinic PMD doc files
2019-09-25 14:30 4% ` [dpdk-dev] [PATCH v2 15/17] net/hinic: add hinic PMD doc files Xiaoyun wang
@ 2019-09-26 18:51 0% ` Ferruh Yigit
0 siblings, 0 replies; 200+ results
From: Ferruh Yigit @ 2019-09-26 18:51 UTC (permalink / raw)
To: Xiaoyun wang
Cc: dev, xuanziyang2, shahar.belkar, luoxianjun, tanya.brokhman,
zhouguoyang, wulike1
On 9/25/2019 3:30 PM, Xiaoyun wang wrote:
> Add doc files about new features and modification.
>
> Signed-off-by: Xiaoyun wang <cloud.wangxiaoyun@huawei.com>
> ---
> doc/guides/nics/features/hinic.ini | 12 ++++++++-
> doc/guides/nics/hinic.rst | 6 +++++
> doc/guides/rel_notes/release_19_11.rst | 45 ++++++----------------------------
> 3 files changed, 25 insertions(+), 38 deletions(-)
>
> diff --git a/doc/guides/nics/features/hinic.ini b/doc/guides/nics/features/hinic.ini
> index fe063d6..dc02b4b 100644
> --- a/doc/guides/nics/features/hinic.ini
> +++ b/doc/guides/nics/features/hinic.ini
> @@ -9,16 +9,22 @@ Link status = Y
> Link status event = Y
> Free Tx mbuf on demand = Y
> Queue start/stop = Y
> -Jumbo frame = N
> +MTU update = Y
> +Jumbo frame = Y
> Scattered Rx = Y
> TSO = Y
> +LRO = Y
> Promiscuous mode = Y
> +Allmulticast mode = Y
> Unicast MAC filter = Y
> Multicast MAC filter = Y
> RSS hash = Y
> RSS key update = Y
> RSS reta update = Y
> Inner RSS = Y
> +SR-IOV = Y
> +VLAN filter = Y
> +VLAN offload = Y
> CRC offload = Y
> L3 checksum offload = Y
> L4 checksum offload = Y
> @@ -27,6 +33,10 @@ Inner L4 checksum = Y
> Basic stats = Y
> Extended stats = Y
> Stats per queue = Y
> +Flow director = Y
> +Flow control = Y
> +FW version = Y
> +Multiprocess aware = Y
> Linux UIO = Y
> Linux VFIO = Y
> BSD nic_uio = N
> diff --git a/doc/guides/nics/hinic.rst b/doc/guides/nics/hinic.rst
> index c9329bc..f036fc5 100644
> --- a/doc/guides/nics/hinic.rst
> +++ b/doc/guides/nics/hinic.rst
> @@ -24,6 +24,12 @@ Features
> - Link state information
> - Link flow control
> - Scattered and gather for TX and RX
> +- SR�CIOV - Partially supported at this point, VFIO only
Can you fix the char is SR-IOV?
> +- Allmulticast mode
> +- Unicast MAC filter
> +- Multicast MAC filter
> +- FW version
> +- Flow director
>
> Prerequisites
> -------------
> diff --git a/doc/guides/rel_notes/release_19_11.rst b/doc/guides/rel_notes/release_19_11.rst
> index 65361c4..6c6f27f 100644
> --- a/doc/guides/rel_notes/release_19_11.rst
> +++ b/doc/guides/rel_notes/release_19_11.rst
> @@ -56,11 +56,15 @@ New Features
> Also, make sure to start the actual text at the margin.
> =========================================================
>
> -* **Updated the Intel ice driver.**
> +* **Updated the Huawei hinic driver.**
>
> - Updated the Intel ice driver with new features and improvements, including:
> + Updated the Huawei hinic driver with new features and improvements, including:
>
> - * Added support for device-specific DDP package loading.
> + * Enabled SR-IOV - Partially supported at this point, VFIO only.
> + * Supported VLAN filter and VLAN offload.
> + * Supported Unicast MAC filter and Multicast MAC filter.
> + * Supported FW version get.
> + * Supported Flow director for LACP, VRRP, BGP and so on.
Can you please distribute the doc patches in to the related patches that
introduces the feature, for all three document, it helps by documenting what has
been added in the patch.
>
> Removed Items
> -------------
> @@ -99,30 +103,6 @@ API Changes
> Also, make sure to start the actual text at the margin.
> =========================================================
>
> -* ethdev: changed ``rte_eth_dev_infos_get`` return value from ``void`` to
> - ``int`` to provide a way to report various error conditions.
> -
> -* ethdev: changed ``rte_eth_promiscuous_enable`` and
> - ``rte_eth_promiscuous_disable`` return value from ``void`` to ``int`` to
> - provide a way to report various error conditions.
> -
> -* ethdev: changed ``rte_eth_allmulticast_enable`` and
> - ``rte_eth_allmulticast_disable`` return value from ``void`` to ``int`` to
> - provide a way to report various error conditions.
> -
> -* ethdev: changed ``rte_eth_dev_xstats_reset`` return value from ``void`` to
> - ``int`` to provide a way to report various error conditions.
> -
> -* ethdev: changed ``rte_eth_link_get`` and ``rte_eth_link_get_nowait``
> - return value from ``void`` to ``int`` to provide a way to report various
> - error conditions.
> -
> -* ethdev: changed ``rte_eth_macaddr_get`` return value from ``void`` to
> - ``int`` to provide a way to report various error conditions.
> -
> -* ethdev: changed ``rte_eth_dev_owner_delete`` return value from ``void`` to
> - ``int`` to provide a way to report various error conditions.
> -
>
> ABI Changes
> -----------
> @@ -174,7 +154,7 @@ The libraries prepended with a plus sign were incremented in this version.
> librte_distributor.so.1
> librte_eal.so.11
> librte_efd.so.1
> - + librte_ethdev.so.13
> + librte_ethdev.so.12
> librte_eventdev.so.7
> librte_flow_classify.so.1
> librte_gro.so.1
> @@ -252,12 +232,3 @@ Tested Platforms
> Also, make sure to start the actual text at the margin.
> =========================================================
>
> -* **Updated Mellanox mlx5 driver.**
> -
> - Updated Mellanox mlx5 driver with new features and improvements, including:
> -
> - * Added support for VLAN pop flow offload command.
> - * Added support for VLAN push flow offload command.
> - * Added support for VLAN set PCP offload command.
> - * Added support for VLAN set VID offload command.
> -
>
I guess above changes are git mistake, please check in next version.
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [PATCH v1 0/4] get Rx/Tx packet burst mode information
2019-09-26 17:15 3% ` Stephen Hemminger
@ 2019-09-26 17:36 0% ` Ferruh Yigit
2019-09-27 1:17 0% ` Wang, Haiyue
1 sibling, 0 replies; 200+ results
From: Ferruh Yigit @ 2019-09-26 17:36 UTC (permalink / raw)
To: Stephen Hemminger, Wang, Haiyue
Cc: dev, Ye, Xiaolong, Kinsella, Ray, Iremonger, Bernard, Sun, Chenmin
On 9/26/2019 6:15 PM, Stephen Hemminger wrote:
> On Thu, 26 Sep 2019 16:36:09 +0000
> "Wang, Haiyue" <haiyue.wang@intel.com> wrote:
>
>> Hi Stephen,
>>
>>> -----Original Message-----
>>> From: Stephen Hemminger [mailto:stephen@networkplumber.org]
>>> Sent: Thursday, September 26, 2019 23:57
>>> To: Wang, Haiyue <haiyue.wang@intel.com>
>>> Cc: dev@dpdk.org; Yigit, Ferruh <ferruh.yigit@intel.com>; Ye, Xiaolong <xiaolong.ye@intel.com>;
>>> Kinsella, Ray <ray.kinsella@intel.com>; Iremonger, Bernard <bernard.iremonger@intel.com>; Sun, Chenmin
>>> <chenmin.sun@intel.com>
>>> Subject: Re: [dpdk-dev] [PATCH v1 0/4] get Rx/Tx packet burst mode information
>>>
>>> On Thu, 26 Sep 2019 19:48:14 +0800
>>> Haiyue Wang <haiyue.wang@intel.com> wrote:
>>>
>>>> RFCv3 -> v1:
>>>> https://patchwork.dpdk.org/patch/59103/
>>>> https://patchwork.dpdk.org/patch/59104/
>>>> https://patchwork.dpdk.org/patch/59105/
>>>> https://patchwork.dpdk.org/patch/59106/
>>>> 1). Use the function 'rte_bsf64' to iterate the options for
>>>> getting the name.
>>>>
>>>> Haiyue Wang (4):
>>>> ethdev: add the API for getting burst mode information
>>>> net/i40e: support to get the Rx/Tx burst mode
>>>> net/ice: support to get the Rx/Tx burst mode
>>>> app/testpmd: show the Rx/Tx burst mode description
>>>>
>>>> app/test-pmd/config.c | 29 +++++++++
>>>> doc/guides/rel_notes/release_19_11.rst | 9 +++
>>>> drivers/net/i40e/i40e_ethdev.c | 2 +
>>>> drivers/net/i40e/i40e_ethdev.h | 4 ++
>>>> drivers/net/i40e/i40e_rxtx.c | 72 +++++++++++++++++++++
>>>> drivers/net/ice/ice_ethdev.c | 2 +
>>>> drivers/net/ice/ice_rxtx.c | 54 ++++++++++++++++
>>>> drivers/net/ice/ice_rxtx.h | 4 ++
>>>> lib/librte_ethdev/rte_ethdev.c | 75 ++++++++++++++++++++++
>>>> lib/librte_ethdev/rte_ethdev.h | 82 ++++++++++++++++++++++++
>>>> lib/librte_ethdev/rte_ethdev_core.h | 5 ++
>>>> lib/librte_ethdev/rte_ethdev_version.map | 5 ++
>>>> 12 files changed, 343 insertions(+)
>>>>
>>>
>>> A couple of meta comments:
>>> 1) Could this be part of dev_info_get somehow?
>>>
>>
>> https://patchwork.dpdk.org/patch/57624/
>> 'Think of a better way that doesn't break ABI.' ;-)
>
> That comment was made relative to 19.08, but 19.11 is the time where
> API/ABI breakage is allowed.
'rte_eth_dev_info_get()' is already a little messy, yes this can be part of it
but I think separate, smaller, better defined APIs are better.
Also almost everybody interested in 'rte_eth_dev_info_get()', but not sure
everyone will be interested in this info.
>
>>> 2) Why should application care? Is this just a test hook?
>>
>> https://patches.dpdk.org/cover/57623/
>> This is from FD.io VPP's bug, and finally, we come out
>> this API for application accessing the burst mode information.
>> It can be used as a simple trace or something like performance
>> analysis like why slow ? Not in vector, anyway, application can
>> get this burst mode information now, not just open PMD debug log
>> level.
>
> From an architecture perspective, diagnostics are good but VPP is probably
> taking that too far. It is possible to expose local symbols if they
> want to keep using dlsym() by adjusting linker flags. It is more that VPP
> is stripping everything. Since VPP has chosen to go their own
> way is fixable inside VPP without changing DPDK. Also, long term VPP is
> going away from using DPDK drivers. Probably soon they will have their
> own drivers for i40e and ice anyway.
>
>
> The basis of my concern is that this is one of those kind of API's
> that creates long term technical debt around supporting it as other
> things change.
>
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [PATCH v1 0/4] get Rx/Tx packet burst mode information
2019-09-26 16:36 3% ` Wang, Haiyue
@ 2019-09-26 17:15 3% ` Stephen Hemminger
2019-09-26 17:36 0% ` Ferruh Yigit
2019-09-27 1:17 0% ` Wang, Haiyue
0 siblings, 2 replies; 200+ results
From: Stephen Hemminger @ 2019-09-26 17:15 UTC (permalink / raw)
To: Wang, Haiyue
Cc: dev, Yigit, Ferruh, Ye, Xiaolong, Kinsella, Ray, Iremonger,
Bernard, Sun, Chenmin
On Thu, 26 Sep 2019 16:36:09 +0000
"Wang, Haiyue" <haiyue.wang@intel.com> wrote:
> Hi Stephen,
>
> > -----Original Message-----
> > From: Stephen Hemminger [mailto:stephen@networkplumber.org]
> > Sent: Thursday, September 26, 2019 23:57
> > To: Wang, Haiyue <haiyue.wang@intel.com>
> > Cc: dev@dpdk.org; Yigit, Ferruh <ferruh.yigit@intel.com>; Ye, Xiaolong <xiaolong.ye@intel.com>;
> > Kinsella, Ray <ray.kinsella@intel.com>; Iremonger, Bernard <bernard.iremonger@intel.com>; Sun, Chenmin
> > <chenmin.sun@intel.com>
> > Subject: Re: [dpdk-dev] [PATCH v1 0/4] get Rx/Tx packet burst mode information
> >
> > On Thu, 26 Sep 2019 19:48:14 +0800
> > Haiyue Wang <haiyue.wang@intel.com> wrote:
> >
> > > RFCv3 -> v1:
> > > https://patchwork.dpdk.org/patch/59103/
> > > https://patchwork.dpdk.org/patch/59104/
> > > https://patchwork.dpdk.org/patch/59105/
> > > https://patchwork.dpdk.org/patch/59106/
> > > 1). Use the function 'rte_bsf64' to iterate the options for
> > > getting the name.
> > >
> > > Haiyue Wang (4):
> > > ethdev: add the API for getting burst mode information
> > > net/i40e: support to get the Rx/Tx burst mode
> > > net/ice: support to get the Rx/Tx burst mode
> > > app/testpmd: show the Rx/Tx burst mode description
> > >
> > > app/test-pmd/config.c | 29 +++++++++
> > > doc/guides/rel_notes/release_19_11.rst | 9 +++
> > > drivers/net/i40e/i40e_ethdev.c | 2 +
> > > drivers/net/i40e/i40e_ethdev.h | 4 ++
> > > drivers/net/i40e/i40e_rxtx.c | 72 +++++++++++++++++++++
> > > drivers/net/ice/ice_ethdev.c | 2 +
> > > drivers/net/ice/ice_rxtx.c | 54 ++++++++++++++++
> > > drivers/net/ice/ice_rxtx.h | 4 ++
> > > lib/librte_ethdev/rte_ethdev.c | 75 ++++++++++++++++++++++
> > > lib/librte_ethdev/rte_ethdev.h | 82 ++++++++++++++++++++++++
> > > lib/librte_ethdev/rte_ethdev_core.h | 5 ++
> > > lib/librte_ethdev/rte_ethdev_version.map | 5 ++
> > > 12 files changed, 343 insertions(+)
> > >
> >
> > A couple of meta comments:
> > 1) Could this be part of dev_info_get somehow?
> >
>
> https://patchwork.dpdk.org/patch/57624/
> 'Think of a better way that doesn't break ABI.' ;-)
That comment was made relative to 19.08, but 19.11 is the time where
API/ABI breakage is allowed.
> > 2) Why should application care? Is this just a test hook?
>
> https://patches.dpdk.org/cover/57623/
> This is from FD.io VPP's bug, and finally, we come out
> this API for application accessing the burst mode information.
> It can be used as a simple trace or something like performance
> analysis like why slow ? Not in vector, anyway, application can
> get this burst mode information now, not just open PMD debug log
> level.
From an architecture perspective, diagnostics are good but VPP is probably
taking that too far. It is possible to expose local symbols if they
want to keep using dlsym() by adjusting linker flags. It is more that VPP
is stripping everything. Since VPP has chosen to go their own
way is fixable inside VPP without changing DPDK. Also, long term VPP is
going away from using DPDK drivers. Probably soon they will have their
own drivers for i40e and ice anyway.
The basis of my concern is that this is one of those kind of API's
that creates long term technical debt around supporting it as other
things change.
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] [PATCH v1 0/4] get Rx/Tx packet burst mode information
@ 2019-09-26 16:36 3% ` Wang, Haiyue
2019-09-26 17:15 3% ` Stephen Hemminger
0 siblings, 1 reply; 200+ results
From: Wang, Haiyue @ 2019-09-26 16:36 UTC (permalink / raw)
To: Stephen Hemminger
Cc: dev, Yigit, Ferruh, Ye, Xiaolong, Kinsella, Ray, Iremonger,
Bernard, Sun, Chenmin
Hi Stephen,
> -----Original Message-----
> From: Stephen Hemminger [mailto:stephen@networkplumber.org]
> Sent: Thursday, September 26, 2019 23:57
> To: Wang, Haiyue <haiyue.wang@intel.com>
> Cc: dev@dpdk.org; Yigit, Ferruh <ferruh.yigit@intel.com>; Ye, Xiaolong <xiaolong.ye@intel.com>;
> Kinsella, Ray <ray.kinsella@intel.com>; Iremonger, Bernard <bernard.iremonger@intel.com>; Sun, Chenmin
> <chenmin.sun@intel.com>
> Subject: Re: [dpdk-dev] [PATCH v1 0/4] get Rx/Tx packet burst mode information
>
> On Thu, 26 Sep 2019 19:48:14 +0800
> Haiyue Wang <haiyue.wang@intel.com> wrote:
>
> > RFCv3 -> v1:
> > https://patchwork.dpdk.org/patch/59103/
> > https://patchwork.dpdk.org/patch/59104/
> > https://patchwork.dpdk.org/patch/59105/
> > https://patchwork.dpdk.org/patch/59106/
> > 1). Use the function 'rte_bsf64' to iterate the options for
> > getting the name.
> >
> > Haiyue Wang (4):
> > ethdev: add the API for getting burst mode information
> > net/i40e: support to get the Rx/Tx burst mode
> > net/ice: support to get the Rx/Tx burst mode
> > app/testpmd: show the Rx/Tx burst mode description
> >
> > app/test-pmd/config.c | 29 +++++++++
> > doc/guides/rel_notes/release_19_11.rst | 9 +++
> > drivers/net/i40e/i40e_ethdev.c | 2 +
> > drivers/net/i40e/i40e_ethdev.h | 4 ++
> > drivers/net/i40e/i40e_rxtx.c | 72 +++++++++++++++++++++
> > drivers/net/ice/ice_ethdev.c | 2 +
> > drivers/net/ice/ice_rxtx.c | 54 ++++++++++++++++
> > drivers/net/ice/ice_rxtx.h | 4 ++
> > lib/librte_ethdev/rte_ethdev.c | 75 ++++++++++++++++++++++
> > lib/librte_ethdev/rte_ethdev.h | 82 ++++++++++++++++++++++++
> > lib/librte_ethdev/rte_ethdev_core.h | 5 ++
> > lib/librte_ethdev/rte_ethdev_version.map | 5 ++
> > 12 files changed, 343 insertions(+)
> >
>
> A couple of meta comments:
> 1) Could this be part of dev_info_get somehow?
>
https://patchwork.dpdk.org/patch/57624/
'Think of a better way that doesn't break ABI.' ;-)
> 2) Why should application care? Is this just a test hook?
https://patches.dpdk.org/cover/57623/
This is from FD.io VPP's bug, and finally, we come out
this API for application accessing the burst mode information.
It can be used as a simple trace or something like performance
analysis like why slow ? Not in vector, anyway, application can
get this burst mode information now, not just open PMD debug log
level.
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] [PATCH v5 1/2] ethdev: expose basic xstats for driver use
2019-09-26 12:46 3% ` Andrew Rybchenko
@ 2019-09-26 16:09 0% ` Stephen Hemminger
0 siblings, 0 replies; 200+ results
From: Stephen Hemminger @ 2019-09-26 16:09 UTC (permalink / raw)
To: Andrew Rybchenko; +Cc: dev
On Thu, 26 Sep 2019 15:46:52 +0300
Andrew Rybchenko <arybchenko@solarflare.com> wrote:
> On 9/19/19 4:17 PM, Stephen Hemminger wrote:
> > Avoid duplication by having generic basic xstats available
> > for use by drivers. A later patch uses this for failsafe
> > driver.
> >
> > Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
> > Acked-by: Gaetan Rivet <gaetan.rivet@6wind.com>
>
> Acked-by: Andrew Rybchenko <arybchenko@solarflare.com>
>
> > diff --git a/lib/librte_ethdev/rte_ethdev_driver.h b/lib/librte_ethdev/rte_ethdev_driver.h
> > index 936ff8c98651..489889a72203 100644
> > --- a/lib/librte_ethdev/rte_ethdev_driver.h
> > +++ b/lib/librte_ethdev/rte_ethdev_driver.h
> > @@ -208,6 +208,71 @@ rte_eth_linkstatus_get(const struct rte_eth_dev *dev,
> > #endif
> > }
> >
> > +/**
> > + * @internal
> > + * Get basic stats part of xstats for an ethernet device.
> > + *
> > + * @param dev
> > + * Pointer to struct rte_eth_dev.
> > + */
> > +__rte_experimental
>
> Does it make sense to mark internal API as experimental?
> I thought that @internal is not a part of API/ABI.
agree, but checkpatch doesn't understand @internal tag.
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [PATCH v7 10/10] vhost: add vhost-user-blk example which support inflight
2019-09-26 14:29 0% ` Yu, Jin
@ 2019-09-26 14:40 0% ` Tiwei Bie
0 siblings, 0 replies; 200+ results
From: Tiwei Bie @ 2019-09-26 14:40 UTC (permalink / raw)
To: Yu, Jin; +Cc: dev, Liu, Changpeng, maxime.coquelin, Wang, Zhihong
On Thu, Sep 26, 2019 at 10:29:19PM +0800, Yu, Jin wrote:
> > -----Original Message-----
> > From: Bie, Tiwei
> > Sent: Wednesday, September 25, 2019 10:46 PM
> > To: Yu, Jin <jin.yu@intel.com>
> > Cc: dev@dpdk.org; Liu, Changpeng <changpeng.liu@intel.com>;
> > maxime.coquelin@redhat.com; Wang, Zhihong <zhihong.wang@intel.com>
> > Subject: Re: [PATCH v7 10/10] vhost: add vhost-user-blk example which support
> > inflight
> >
> > On Fri, Sep 20, 2019 at 08:01:02PM +0800, Jin Yu wrote:
> > > A vhost-user-blk example that support inflight feature. It uses the
> > > new APIs that introduced in the first patch, so It can show how there
> >
> > s/It/it/
> > s/there/these/
>
> Got it. Thanks.
> >
> > > APIs work to support inflight feature.
> > >
> > > Signed-off-by: Jin Yu <jin.yu@intel.com>
> > > ---
> > > V1 - add the case.
> > > V2 - add the rte_vhost prefix.
> > > V3 - add packed ring support
> > > ---
> > > examples/vhost_blk/Makefile | 67 ++
> > > examples/vhost_blk/blk.c | 125 +++
> > > examples/vhost_blk/blk_spec.h | 95 ++
> > > examples/vhost_blk/meson.build | 20 +
> > > examples/vhost_blk/vhost_blk.c | 1313 +++++++++++++++++++++++++
> > > examples/vhost_blk/vhost_blk.h | 116 +++
> > > examples/vhost_blk/vhost_blk_compat.c | 195 ++++
> > > 7 files changed, 1931 insertions(+)
> >
> > I met some build issues when trying this example.
> >
> > examples/vhost_blk/vhost_blk.c: In function ‘descriptor_get_next_packed’:
> > examples/vhost_blk/vhost_blk.c:71:21: error: invalid use of undefined type
> > ‘struct vring_packed_desc’
> > if (vq->desc_packed[*idx % vq->size].flags & VIRTQ_DESC_F_NEXT) {
> > ^
> > examples/vhost_blk/vhost_blk.c:71:21: error: dereferencing pointer to
> > incomplete type ‘struct vring_packed_desc’
> > examples/vhost_blk/vhost_blk.c:73:26: error: invalid use of undefined type
> > ‘struct vring_packed_desc’
> > return &vq->desc_packed[*idx % vq->size];
>
> The new version of virtio_ring.h have defined the vring_packed_desc.
> For the compatibility， I will add the definition in the rte_vhost.h.
No. They should be added in this example instead of
the vhost API.
> Thanks.
>
> > ^
> > examples/vhost_blk/vhost_blk.c: In function
> > ‘inflight_submit_completion_packed’:
> > examples/vhost_blk/vhost_blk.c:165:2: warning:
> > ‘rte_vhost_set_last_inflight_io_packed’ is deprecated: Symbol is not yet part of
> > stable ABI [-Wdeprecated-declarations]
>
> Should I ignore this warning？ I'm not sure about this.
> The reason is the _rte_experiment keyword?
You need something like this:
https://github.com/DPDK/dpdk/blob/bd253daa7717835f88bbc58b09a94d0060380396/examples/vhost/Makefile#L29
https://github.com/DPDK/dpdk/blob/bd253daa7717835f88bbc58b09a94d0060380396/examples/vhost/meson.build#L13
> Thanks.
> > ret = rte_vhost_set_last_inflight_io_packed(ctrlr->bdev->vid, q_idx,
> > ^~~
> > In file included from examples/vhost_blk/vhost_blk.c:17:0:
> > x86_64-native-linuxapp-gcc/include/rte_vhost.h:810:1: note: declared here
> > rte_vhost_set_last_inflight_io_packed(int vid,
> > ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
> > ^
> > ...
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [PATCH v7 10/10] vhost: add vhost-user-blk example which support inflight
2019-09-25 14:45 3% ` Tiwei Bie
@ 2019-09-26 14:29 0% ` Yu, Jin
2019-09-26 14:40 0% ` Tiwei Bie
0 siblings, 1 reply; 200+ results
From: Yu, Jin @ 2019-09-26 14:29 UTC (permalink / raw)
To: Bie, Tiwei; +Cc: dev, Liu, Changpeng, maxime.coquelin, Wang, Zhihong
> -----Original Message-----
> From: Bie, Tiwei
> Sent: Wednesday, September 25, 2019 10:46 PM
> To: Yu, Jin <jin.yu@intel.com>
> Cc: dev@dpdk.org; Liu, Changpeng <changpeng.liu@intel.com>;
> maxime.coquelin@redhat.com; Wang, Zhihong <zhihong.wang@intel.com>
> Subject: Re: [PATCH v7 10/10] vhost: add vhost-user-blk example which support
> inflight
>
> On Fri, Sep 20, 2019 at 08:01:02PM +0800, Jin Yu wrote:
> > A vhost-user-blk example that support inflight feature. It uses the
> > new APIs that introduced in the first patch, so It can show how there
>
> s/It/it/
> s/there/these/
Got it. Thanks.
>
> > APIs work to support inflight feature.
> >
> > Signed-off-by: Jin Yu <jin.yu@intel.com>
> > ---
> > V1 - add the case.
> > V2 - add the rte_vhost prefix.
> > V3 - add packed ring support
> > ---
> > examples/vhost_blk/Makefile | 67 ++
> > examples/vhost_blk/blk.c | 125 +++
> > examples/vhost_blk/blk_spec.h | 95 ++
> > examples/vhost_blk/meson.build | 20 +
> > examples/vhost_blk/vhost_blk.c | 1313 +++++++++++++++++++++++++
> > examples/vhost_blk/vhost_blk.h | 116 +++
> > examples/vhost_blk/vhost_blk_compat.c | 195 ++++
> > 7 files changed, 1931 insertions(+)
>
> I met some build issues when trying this example.
>
> examples/vhost_blk/vhost_blk.c: In function ‘descriptor_get_next_packed’:
> examples/vhost_blk/vhost_blk.c:71:21: error: invalid use of undefined type
> ‘struct vring_packed_desc’
> if (vq->desc_packed[*idx % vq->size].flags & VIRTQ_DESC_F_NEXT) {
> ^
> examples/vhost_blk/vhost_blk.c:71:21: error: dereferencing pointer to
> incomplete type ‘struct vring_packed_desc’
> examples/vhost_blk/vhost_blk.c:73:26: error: invalid use of undefined type
> ‘struct vring_packed_desc’
> return &vq->desc_packed[*idx % vq->size];
The new version of virtio_ring.h have defined the vring_packed_desc.
For the compatibility， I will add the definition in the rte_vhost.h.
Thanks.
> ^
> examples/vhost_blk/vhost_blk.c: In function
> ‘inflight_submit_completion_packed’:
> examples/vhost_blk/vhost_blk.c:165:2: warning:
> ‘rte_vhost_set_last_inflight_io_packed’ is deprecated: Symbol is not yet part of
> stable ABI [-Wdeprecated-declarations]
Should I ignore this warning？ I'm not sure about this.
The reason is the _rte_experiment keyword?
Thanks.
> ret = rte_vhost_set_last_inflight_io_packed(ctrlr->bdev->vid, q_idx,
> ^~~
> In file included from examples/vhost_blk/vhost_blk.c:17:0:
> x86_64-native-linuxapp-gcc/include/rte_vhost.h:810:1: note: declared here
> rte_vhost_set_last_inflight_io_packed(int vid,
> ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
> ^
> ...
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [PATCH v5 1/2] ethdev: expose basic xstats for driver use
@ 2019-09-26 12:46 3% ` Andrew Rybchenko
2019-09-26 16:09 0% ` Stephen Hemminger
0 siblings, 1 reply; 200+ results
From: Andrew Rybchenko @ 2019-09-26 12:46 UTC (permalink / raw)
To: Stephen Hemminger, dev
On 9/19/19 4:17 PM, Stephen Hemminger wrote:
> Avoid duplication by having generic basic xstats available
> for use by drivers. A later patch uses this for failsafe
> driver.
>
> Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
> Acked-by: Gaetan Rivet <gaetan.rivet@6wind.com>
Acked-by: Andrew Rybchenko <arybchenko@solarflare.com>
> diff --git a/lib/librte_ethdev/rte_ethdev_driver.h b/lib/librte_ethdev/rte_ethdev_driver.h
> index 936ff8c98651..489889a72203 100644
> --- a/lib/librte_ethdev/rte_ethdev_driver.h
> +++ b/lib/librte_ethdev/rte_ethdev_driver.h
> @@ -208,6 +208,71 @@ rte_eth_linkstatus_get(const struct rte_eth_dev *dev,
> #endif
> }
>
> +/**
> + * @internal
> + * Get basic stats part of xstats for an ethernet device.
> + *
> + * @param dev
> + * Pointer to struct rte_eth_dev.
> + */
> +__rte_experimental
Does it make sense to mark internal API as experimental?
I thought that @internal is not a part of API/ABI.
[snip]
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] RFC: hiding struct rte_eth_dev
2019-09-26 11:50 0% ` David Marchand
@ 2019-09-26 11:52 0% ` David Marchand
0 siblings, 0 replies; 200+ results
From: David Marchand @ 2019-09-26 11:52 UTC (permalink / raw)
To: Andrew Rybchenko, Thomas Monjalon
Cc: Ananyev, Konstantin, Jerin Jacob, Ray Kinsella, dpdk-dev,
Richardson, Bruce, Jerin Jacob Kollanukkaran, Hemant Agrawal,
Stephen Hemminger, Yigit, Ferruh, maxime.coquelin, Zapolski,
MarcinX A, Ian Stokes, Ilya Maximets
Fixed Ilya address.
On Thu, Sep 26, 2019 at 1:50 PM David Marchand
<david.marchand@redhat.com> wrote:
>
> On Thu, Sep 26, 2019 at 1:13 PM Andrew Rybchenko
> <arybchenko@solarflare.com> wrote:
> >
> > On 9/24/19 7:50 PM, Ananyev, Konstantin wrote:
> >
> > Hi everyone,
> >
> > Hi folks,
> >
> > The ABI Stability proposals should be pretty well known at this point.
> > The latest rev is here ...
> >
> > http://inbox.dpdk.org/dev/1565864619-17206-1-git-send-email-mdr@ashroe.eu/
> >
> > As has been discussed public data structure's are risky for ABI
> > stability, as any changes to a data structure can change the ABI. As a
> > general rule you want to expose as few as possible (ideally none), and
> > keep them as small as possible.
> >
> > One of the key data structures in DPDK is `struct rte_eth_dev`. In this
> > case, rte_eth_dev is exposed public-ally, as a side-effect of the
> > inlining of the [rx,tx]_burst functions.
> >
> > Marcin Zapolski has been looking at what to do about it, with no current
> > consensus on a path forward. The options on our table is:-
> >
> > 1. Do nothing, live with the risk to DPDK v20 ABI stability.
> >
> > 2. Pad rte_eth_dev, add some extra bytes to the structure "in case" we
> > need to add a field during the v20 ABI (through to 20.11).
> >
> > 3. Break rte_eth_dev into public and private structs.
> > - See
> > http://inbox.dpdk.org/dev/20190906131813.1343-1-marcinx.a.zapolski@intel.com/
> > - This ends up quiet an invasive patch, late in the cycle, however it
> > does have no performance penalty.
> >
> > 4. Uninline [rx,tx]_burst functions
> > - See
> > http://inbox.dpdk.org/dev/20190730124950.1293-1-marcinx.a.zapolski@intel.com/
> > - This has a performance penalty of ~2% with testpmd, impact on a "real
> > workload" is likely to be in the noise.
> >
> > We need to agree an approach for v19.11, and that may be we agree to do
> > nothing. My personal vote is 4. as the simplest with minimal impact.
> >
> > My preference NOT to do #4. Reasons are:
> > - I have seen performance drop from 1.5% to 3.5% based on the arm64
> > cores in use(Embedded vs Server cores)
> > - We need the correct approach to cater to cryptodev and eventdev as
> > well. If #4 is checked in, We will
> > take shotcut for cryptodev and eventdev
> >
> > My preference #1, do nothing, is probably ok and could live with #2,
> > adding padding,
> > and fix properly with #3 as when needed and use #3 scheme for crypto
> > dev and eventdev as well.
> >
> >
> > My preference would be #4 also.
> > If that's not an option, then I suppose #1 for 19.11 and #3 for next release
> > when ABI breakage would be allowed.
> > BTW, good point that we need similar thing for other dev types too.
> > Konstantin
> >
> >
> > My preference would be #4 or #1.
> > #2 and #3 are both tradeoffs and do not resolve ABI breaking completely.
> > #3 is really invasive, it requires changes of driverRx/Tx burst prototypes and
> > uninline descriptor status functions (may be it would be better to change
> > callback prototypes as well, but keep functions inline).
> > #4 is better since it is really a step to ABI stability and it still allow to
> > do many generic checks (dev->data dependent) on ethdev API level.
>
> Did we ensure that external users have all the required api before
> hiding the rte_eth_dev struct?
> ovs still accesses rte_eth_devices[].
>
> CC Ian and Ilya.
--
David Marchand
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] RFC: hiding struct rte_eth_dev
2019-09-26 11:13 4% ` Andrew Rybchenko
@ 2019-09-26 11:50 0% ` David Marchand
2019-09-26 11:52 0% ` David Marchand
0 siblings, 1 reply; 200+ results
From: David Marchand @ 2019-09-26 11:50 UTC (permalink / raw)
To: Andrew Rybchenko, Thomas Monjalon
Cc: Ananyev, Konstantin, Jerin Jacob, Ray Kinsella, dpdk-dev,
Richardson, Bruce, Jerin Jacob Kollanukkaran, Hemant Agrawal,
Stephen Hemminger, Yigit, Ferruh, maxime.coquelin, Zapolski,
MarcinX A, Ian Stokes, Ilya Maximets
On Thu, Sep 26, 2019 at 1:13 PM Andrew Rybchenko
<arybchenko@solarflare.com> wrote:
>
> On 9/24/19 7:50 PM, Ananyev, Konstantin wrote:
>
> Hi everyone,
>
> Hi folks,
>
> The ABI Stability proposals should be pretty well known at this point.
> The latest rev is here ...
>
> http://inbox.dpdk.org/dev/1565864619-17206-1-git-send-email-mdr@ashroe.eu/
>
> As has been discussed public data structure's are risky for ABI
> stability, as any changes to a data structure can change the ABI. As a
> general rule you want to expose as few as possible (ideally none), and
> keep them as small as possible.
>
> One of the key data structures in DPDK is `struct rte_eth_dev`. In this
> case, rte_eth_dev is exposed public-ally, as a side-effect of the
> inlining of the [rx,tx]_burst functions.
>
> Marcin Zapolski has been looking at what to do about it, with no current
> consensus on a path forward. The options on our table is:-
>
> 1. Do nothing, live with the risk to DPDK v20 ABI stability.
>
> 2. Pad rte_eth_dev, add some extra bytes to the structure "in case" we
> need to add a field during the v20 ABI (through to 20.11).
>
> 3. Break rte_eth_dev into public and private structs.
> - See
> http://inbox.dpdk.org/dev/20190906131813.1343-1-marcinx.a.zapolski@intel.com/
> - This ends up quiet an invasive patch, late in the cycle, however it
> does have no performance penalty.
>
> 4. Uninline [rx,tx]_burst functions
> - See
> http://inbox.dpdk.org/dev/20190730124950.1293-1-marcinx.a.zapolski@intel.com/
> - This has a performance penalty of ~2% with testpmd, impact on a "real
> workload" is likely to be in the noise.
>
> We need to agree an approach for v19.11, and that may be we agree to do
> nothing. My personal vote is 4. as the simplest with minimal impact.
>
> My preference NOT to do #4. Reasons are:
> - I have seen performance drop from 1.5% to 3.5% based on the arm64
> cores in use(Embedded vs Server cores)
> - We need the correct approach to cater to cryptodev and eventdev as
> well. If #4 is checked in, We will
> take shotcut for cryptodev and eventdev
>
> My preference #1, do nothing, is probably ok and could live with #2,
> adding padding,
> and fix properly with #3 as when needed and use #3 scheme for crypto
> dev and eventdev as well.
>
>
> My preference would be #4 also.
> If that's not an option, then I suppose #1 for 19.11 and #3 for next release
> when ABI breakage would be allowed.
> BTW, good point that we need similar thing for other dev types too.
> Konstantin
>
>
> My preference would be #4 or #1.
> #2 and #3 are both tradeoffs and do not resolve ABI breaking completely.
> #3 is really invasive, it requires changes of driverRx/Tx burst prototypes and
> uninline descriptor status functions (may be it would be better to change
> callback prototypes as well, but keep functions inline).
> #4 is better since it is really a step to ABI stability and it still allow to
> do many generic checks (dev->data dependent) on ethdev API level.
Did we ensure that external users have all the required api before
hiding the rte_eth_dev struct?
ovs still accesses rte_eth_devices[].
CC Ian and Ilya.
--
David Marchand
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] RFC: hiding struct rte_eth_dev
2019-09-24 16:50 3% ` Ananyev, Konstantin
@ 2019-09-26 11:13 4% ` Andrew Rybchenko
2019-09-26 11:50 0% ` David Marchand
0 siblings, 1 reply; 200+ results
From: Andrew Rybchenko @ 2019-09-26 11:13 UTC (permalink / raw)
To: Ananyev, Konstantin, Jerin Jacob, Ray Kinsella
Cc: dpdk-dev, Richardson, Bruce, Jerin Jacob Kollanukkaran,
Hemant Agrawal, Thomas Monjalon, Stephen Hemminger, Yigit,
Ferruh, maxime.coquelin, David Marchand, Zapolski, MarcinX A
On 9/24/19 7:50 PM, Ananyev, Konstantin wrote:
> Hi everyone,
>
>>> Hi folks,
>>>
>>> The ABI Stability proposals should be pretty well known at this point.
>>> The latest rev is here ...
>>>
>>> http://inbox.dpdk.org/dev/1565864619-17206-1-git-send-email-mdr@ashroe.eu/
>>>
>>> As has been discussed public data structure's are risky for ABI
>>> stability, as any changes to a data structure can change the ABI. As a
>>> general rule you want to expose as few as possible (ideally none), and
>>> keep them as small as possible.
>>>
>>> One of the key data structures in DPDK is `struct rte_eth_dev`. In this
>>> case, rte_eth_dev is exposed public-ally, as a side-effect of the
>>> inlining of the [rx,tx]_burst functions.
>>>
>>> Marcin Zapolski has been looking at what to do about it, with no current
>>> consensus on a path forward. The options on our table is:-
>>>
>>> 1. Do nothing, live with the risk to DPDK v20 ABI stability.
>>>
>>> 2. Pad rte_eth_dev, add some extra bytes to the structure "in case" we
>>> need to add a field during the v20 ABI (through to 20.11).
>>>
>>> 3. Break rte_eth_dev into public and private structs.
>>> - See
>>> http://inbox.dpdk.org/dev/20190906131813.1343-1-marcinx.a.zapolski@intel.com/
>>> - This ends up quiet an invasive patch, late in the cycle, however it
>>> does have no performance penalty.
>>>
>>> 4. Uninline [rx,tx]_burst functions
>>> - See
>>> http://inbox.dpdk.org/dev/20190730124950.1293-1-marcinx.a.zapolski@intel.com/
>>> - This has a performance penalty of ~2% with testpmd, impact on a "real
>>> workload" is likely to be in the noise.
>>>
>>> We need to agree an approach for v19.11, and that may be we agree to do
>>> nothing. My personal vote is 4. as the simplest with minimal impact.
>> My preference NOT to do #4. Reasons are:
>> - I have seen performance drop from 1.5% to 3.5% based on the arm64
>> cores in use(Embedded vs Server cores)
>> - We need the correct approach to cater to cryptodev and eventdev as
>> well. If #4 is checked in, We will
>> take shotcut for cryptodev and eventdev
>>
>> My preference #1, do nothing, is probably ok and could live with #2,
>> adding padding,
>> and fix properly with #3 as when needed and use #3 scheme for crypto
>> dev and eventdev as well.
>>
>>
> My preference would be #4 also.
> If that's not an option, then I suppose #1 for 19.11 and #3 for next release
> when ABI breakage would be allowed.
> BTW, good point that we need similar thing for other dev types too.
> Konstantin
My preference would be #4 or #1.
#2 and #3 are both tradeoffs and do not resolve ABI breaking completely.
#3 is really invasive, it requires changes of driverRx/Tx burst
prototypes and
uninline descriptor status functions (may be it would be better to change
callback prototypes as well, but keep functions inline).
#4 is better since it is really a step to ABI stability and it still
allow to
do many generic checks (dev->data dependent) on ethdev API level.
Andrew
^ permalink raw reply [relevance 4%]
* [dpdk-dev] [PATCH v3 15/15] sched: remove redundant code
@ 2019-09-26 8:52 4% ` Jasvinder Singh
0 siblings, 0 replies; 200+ results
From: Jasvinder Singh @ 2019-09-26 8:52 UTC (permalink / raw)
To: dev; +Cc: cristian.dumitrescu, Lukasz Krakowiak
Remove redundant data structure fields from port level data
structures and update release notes.
Signed-off-by: Jasvinder Singh <jasvinder.singh@intel.com>
Signed-off-by: Lukasz Krakowiak <lukaszx.krakowiak@intel.com>
---
doc/guides/rel_notes/release_19_11.rst | 6 +++-
lib/librte_sched/rte_sched.c | 42 +-------------------------
lib/librte_sched/rte_sched.h | 22 --------------
3 files changed, 6 insertions(+), 64 deletions(-)
diff --git a/doc/guides/rel_notes/release_19_11.rst b/doc/guides/rel_notes/release_19_11.rst
index 27cfbd9e3..dd122f00a 100644
--- a/doc/guides/rel_notes/release_19_11.rst
+++ b/doc/guides/rel_notes/release_19_11.rst
@@ -94,6 +94,10 @@ API Changes
Also, make sure to start the actual text at the margin.
=========================================================
+* sched: The pipe nodes configuration parameters such as number of pipes,
+ pipe queue sizes, pipe profiles, etc., are moved from port level structure
+ to subport level. This allows different subports of the same port to
+ have different configuration for the pipe nodes.
ABI Changes
-----------
@@ -181,7 +185,7 @@ The libraries prepended with a plus sign were incremented in this version.
librte_rcu.so.1
librte_reorder.so.1
librte_ring.so.2
- librte_sched.so.3
+ + librte_sched.so.4
librte_security.so.2
librte_stack.so.1
librte_table.so.3
diff --git a/lib/librte_sched/rte_sched.c b/lib/librte_sched/rte_sched.c
index 1faa580d0..710ecf65a 100644
--- a/lib/librte_sched/rte_sched.c
+++ b/lib/librte_sched/rte_sched.c
@@ -216,13 +216,6 @@ struct rte_sched_port {
uint32_t mtu;
uint32_t frame_overhead;
int socket;
- uint16_t qsize[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
- uint32_t n_pipe_profiles;
- uint32_t n_max_pipe_profiles;
- uint32_t pipe_tc_be_rate_max;
-#ifdef RTE_SCHED_RED
- struct rte_red_config red_config[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE][RTE_COLORS];
-#endif
/* Timing */
uint64_t time_cpu_cycles; /* Current CPU time measured in CPU cyles */
@@ -230,48 +223,15 @@ struct rte_sched_port {
uint64_t time; /* Current NIC TX time measured in bytes */
struct rte_reciprocal inv_cycles_per_byte; /* CPU cycles per byte */
- /* Scheduling loop detection */
- uint32_t pipe_loop;
- uint32_t pipe_exhaustion;
-
- /* Bitmap */
- struct rte_bitmap *bmp;
- uint32_t grinder_base_bmp_pos[RTE_SCHED_PORT_N_GRINDERS] __rte_aligned_16;
-
/* Grinders */
- struct rte_sched_grinder grinder[RTE_SCHED_PORT_N_GRINDERS];
- uint32_t busy_grinders;
struct rte_mbuf **pkts_out;
uint32_t n_pkts_out;
uint32_t subport_id;
- /* Queue base calculation */
- uint32_t qsize_add[RTE_SCHED_QUEUES_PER_PIPE];
- uint32_t qsize_sum;
-
/* Large data structures */
- struct rte_sched_subport *subports[0];
- struct rte_sched_subport *subport;
- struct rte_sched_pipe *pipe;
- struct rte_sched_queue *queue;
- struct rte_sched_queue_extra *queue_extra;
- struct rte_sched_pipe_profile *pipe_profiles;
- uint8_t *bmp_array;
- struct rte_mbuf **queue_array;
- uint8_t memory[0] __rte_cache_aligned;
+ struct rte_sched_subport *subports[0] __rte_cache_aligned;
} __rte_cache_aligned;
-enum rte_sched_port_array {
- e_RTE_SCHED_PORT_ARRAY_SUBPORT = 0,
- e_RTE_SCHED_PORT_ARRAY_PIPE,
- e_RTE_SCHED_PORT_ARRAY_QUEUE,
- e_RTE_SCHED_PORT_ARRAY_QUEUE_EXTRA,
- e_RTE_SCHED_PORT_ARRAY_PIPE_PROFILES,
- e_RTE_SCHED_PORT_ARRAY_BMP_ARRAY,
- e_RTE_SCHED_PORT_ARRAY_QUEUE_ARRAY,
- e_RTE_SCHED_PORT_ARRAY_TOTAL,
-};
-
enum rte_sched_subport_array {
e_RTE_SCHED_SUBPORT_ARRAY_PIPE = 0,
e_RTE_SCHED_SUBPORT_ARRAY_QUEUE,
diff --git a/lib/librte_sched/rte_sched.h b/lib/librte_sched/rte_sched.h
index 40f02f124..c82c23c14 100644
--- a/lib/librte_sched/rte_sched.h
+++ b/lib/librte_sched/rte_sched.h
@@ -260,28 +260,6 @@ struct rte_sched_port_params {
* the subports of the same port.
*/
uint32_t n_pipes_per_subport;
-
- /** Packet queue size for each traffic class.
- * All the pipes within the same subport share the similar
- * configuration for the queues.
- */
- uint16_t qsize[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
-
- /** Pipe profile table.
- * Every pipe is configured using one of the profiles from this table.
- */
- struct rte_sched_pipe_params *pipe_profiles;
-
- /** Profiles in the pipe profile table */
- uint32_t n_pipe_profiles;
-
- /** Max profiles allowed in the pipe profile table */
- uint32_t n_max_pipe_profiles;
-
-#ifdef RTE_SCHED_RED
- /** RED parameters */
- struct rte_red_params red_params[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE][RTE_COLORS];
-#endif
};
/*
--
2.21.0
^ permalink raw reply [relevance 4%]
* Re: [dpdk-dev] [PATCH v3 13/15] vhost: cache address translation result
@ 2019-09-26 5:32 3% ` Tiwei Bie
0 siblings, 0 replies; 200+ results
From: Tiwei Bie @ 2019-09-26 5:32 UTC (permalink / raw)
To: Marvin Liu; +Cc: maxime.coquelin, zhihong.wang, stephen, gavin.hu, dev
On Thu, Sep 26, 2019 at 01:13:27AM +0800, Marvin Liu wrote:
> Cache address translation result and use it in next translation. Due
> to limited regions are supported, buffers are most likely in same
> region when doing data transmission.
>
> Signed-off-by: Marvin Liu <yong.liu@intel.com>
>
> diff --git a/lib/librte_vhost/rte_vhost.h b/lib/librte_vhost/rte_vhost.h
> index 7fb172912..d90235cd6 100644
> --- a/lib/librte_vhost/rte_vhost.h
> +++ b/lib/librte_vhost/rte_vhost.h
> @@ -91,10 +91,18 @@ struct rte_vhost_mem_region {
> int fd;
> };
>
> +struct rte_vhost_mem_region_cache {
> + uint64_t guest_phys_addr;
> + uint64_t guest_phys_addr_end;
> + int64_t host_user_addr_offset;
> + uint64_t size;
> +};
> +
> /**
> * Memory structure includes region and mapping information.
> */
> struct rte_vhost_memory {
> + struct rte_vhost_mem_region_cache cache_region;
This breaks ABI.
> uint32_t nregions;
> struct rte_vhost_mem_region regions[];
> };
> @@ -232,11 +240,30 @@ rte_vhost_va_from_guest_pa(struct rte_vhost_memory *mem,
> struct rte_vhost_mem_region *r;
> uint32_t i;
>
> + struct rte_vhost_mem_region_cache *r_cache;
> + /* check with cached region */
> + r_cache = &mem->cache_region;
> + if (likely(gpa >= r_cache->guest_phys_addr && gpa <
> + r_cache->guest_phys_addr_end)) {
> + if (unlikely(*len > r_cache->guest_phys_addr_end - gpa))
> + *len = r_cache->guest_phys_addr_end - gpa;
> +
> + return gpa - r_cache->host_user_addr_offset;
> + }
Does this help a lot in performance?
We can implement this caching for builtin backend first.
> +
> +
> for (i = 0; i < mem->nregions; i++) {
> r = &mem->regions[i];
> if (gpa >= r->guest_phys_addr &&
> gpa < r->guest_phys_addr + r->size) {
>
> + r_cache->guest_phys_addr = r->guest_phys_addr;
> + r_cache->guest_phys_addr_end = r->guest_phys_addr +
> + r->size;
> + r_cache->size = r->size;
> + r_cache->host_user_addr_offset = r->guest_phys_addr -
> + r->host_user_addr;
> +
> if (unlikely(*len > r->guest_phys_addr + r->size - gpa))
> *len = r->guest_phys_addr + r->size - gpa;
>
> --
> 2.17.1
>
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] [RFC PATCH 1/9] security: introduce CPU Crypto action type and API
2019-09-18 7:44 4% ` Ananyev, Konstantin
@ 2019-09-25 18:24 4% ` Ananyev, Konstantin
2019-09-27 9:26 0% ` Akhil Goyal
0 siblings, 1 reply; 200+ results
From: Ananyev, Konstantin @ 2019-09-25 18:24 UTC (permalink / raw)
To: 'Akhil Goyal', 'dev@dpdk.org',
De Lara Guarch, Pablo, 'Thomas Monjalon'
Cc: Zhang, Roy Fan, Doherty, Declan, 'Anoob Joseph'
> > > > > > > > > This action type allows the burst of symmetric crypto workload using
> > > the
> > > > > > > same
> > > > > > > > > algorithm, key, and direction being processed by CPU cycles
> > > > > synchronously.
> > > > > > > > > This flexible action type does not require external hardware
> > > involvement,
> > > > > > > > > having the crypto workload processed synchronously, and is more
> > > > > > > performant
> > > > > > > > > than Cryptodev SW PMD due to the saved cycles on removed "async
> > > > > mode
> > > > > > > > > simulation" as well as 3 cacheline access of the crypto ops.
> > > > > > > >
> > > > > > > > Does that mean application will not call the cryptodev_enqueue_burst
> > > and
> > > > > > > corresponding dequeue burst.
> > > > > > >
> > > > > > > Yes, instead it just call rte_security_process_cpu_crypto_bulk(...)
> > > > > > >
> > > > > > > > It would be a new API something like process_packets and it will have
> > > the
> > > > > > > crypto processed packets while returning from the API?
> > > > > > >
> > > > > > > Yes, though the plan is that API will operate on raw data buffers, not
> > > mbufs.
> > > > > > >
> > > > > > > >
> > > > > > > > I still do not understand why we cannot do with the conventional
> > > crypto lib
> > > > > > > only.
> > > > > > > > As far as I can understand, you are not doing any protocol processing
> > > or
> > > > > any
> > > > > > > value add
> > > > > > > > To the crypto processing. IMO, you just need a synchronous crypto
> > > > > processing
> > > > > > > API which
> > > > > > > > Can be defined in cryptodev, you don't need to re-create a crypto
> > > session
> > > > > in
> > > > > > > the name of
> > > > > > > > Security session in the driver just to do a synchronous processing.
> > > > > > >
> > > > > > > I suppose your question is why not to have
> > > > > > > rte_crypot_process_cpu_crypto_bulk(...) instead?
> > > > > > > The main reason is that would require disruptive changes in existing
> > > > > cryptodev
> > > > > > > API
> > > > > > > (would cause ABI/API breakage).
> > > > > > > Session for RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO need some
> > > extra
> > > > > > > information
> > > > > > > that normal crypto_sym_xform doesn't contain
> > > > > > > (cipher offset from the start of the buffer, might be something extra in
> > > > > future).
> > > > > >
> > > > > > Cipher offset will be part of rte_crypto_op.
> > > > >
> > > > > fill/read (+ alloc/free) is one of the main things that slowdown current
> > > crypto-op
> > > > > approach.
> > > > > That's why the general idea - have all data that wouldn't change from packet
> > > to
> > > > > packet
> > > > > included into the session and setup it once at session_init().
> > > >
> > > > I agree that you cannot use crypto-op.
> > > > You can have the new API in crypto.
> > > > As per the current patch, you only need cipher_offset which you can have it as
> > > a parameter until
> > > > You get it approved in the crypto xform. I believe it will be beneficial in case of
> > > other crypto cases as well.
> > > > We can have cipher offset at both places(crypto-op and cipher_xform). It will
> > > give flexibility to the user to
> > > > override it.
> > >
> > > After having another thought on your proposal:
> > > Probably we can introduce new rte_crypto_sym_xform_types for CPU related
> > > stuff here?
> >
> > I also thought of adding new xforms, but that wont serve the purpose for may be all the cases.
> > You would be needing all information currently available in the current xforms.
> > So if you are adding new fields in the new xform, the size will be more than that of the union of xforms.
> > ABI breakage would still be there.
> >
> > If you think a valid compression of the AEAD xform can be done, then that can be done for each of the
> > Xforms and we can have a solution to this issue.
>
> I think that we can re-use iv.offset for our purposes (for crypto offset).
> So for now we can make that path work without any ABI breakage.
> Fan, please feel free to correct me here, if I missed something.
> If in future we would need to add some extra information it might
> require ABI breakage, though by now I don't envision anything particular to add.
> Anyway, if there is no objection to go that way, we can try to make
> these changes for v2.
>
Actually, after looking at it more deeply it appears not that easy as I thought it would be :)
Below is a very draft version of proposed API additions.
I think it avoids ABI breakages right now and provides enough flexibility for future extensions (if any).
For now, it doesn't address your comments about naming conventions (_CPU_ vs _SYNC_) , etc.
but I suppose is comprehensive enough to provide a main idea beyond it.
Akhil and other interested parties, please try to review and provide feedback ASAP,
as related changes would take some time and we still like to hit 19.11 deadline.
Konstantin
diff --git a/lib/librte_cryptodev/rte_crypto_sym.h b/lib/librte_cryptodev/rte_crypto_sym.h
index bc8da2466..c03069e23 100644
--- a/lib/librte_cryptodev/rte_crypto_sym.h
+++ b/lib/librte_cryptodev/rte_crypto_sym.h
@@ -103,6 +103,9 @@ rte_crypto_cipher_operation_strings[];
*
* This structure contains data relating to Cipher (Encryption and Decryption)
* use to create a session.
+ * Actually I was wrong saying that we don't have free space inside xforms.
+ * Making key struct packed (see below) allow us to regain 6B that could be
+ * used for future extensions.
*/
struct rte_crypto_cipher_xform {
enum rte_crypto_cipher_operation op;
@@ -116,7 +119,25 @@ struct rte_crypto_cipher_xform {
struct {
const uint8_t *data; /**< pointer to key data */
uint16_t length; /**< key length in bytes */
- } key;
+ } __attribute__((__packed__)) key;
+
+ /**
+ * offset for cipher to start within user provided data buffer.
+ * Fan suggested another (and less space consuming way) -
+ * reuse iv.offset space below, by changing:
+ * struct {uint16_t offset, length;} iv;
+ * to uunamed union:
+ * union {
+ * struct {uint16_t offset, length;} iv;
+ * struct {uint16_t iv_len, crypto_offset} cpu_crypto_param;
+ * };
+ * Both approaches seems ok to me in general.
+ * Comments/suggestions are welcome.
+ */
+ uint16_t offset;
+
+ uint8_t reserved1[4];
+
/**< Cipher key
*
* For the RTE_CRYPTO_CIPHER_AES_F8 mode of operation, key.data will
@@ -284,7 +305,7 @@ struct rte_crypto_auth_xform {
struct {
const uint8_t *data; /**< pointer to key data */
uint16_t length; /**< key length in bytes */
- } key;
+ } __attribute__((__packed__)) key;
/**< Authentication key data.
* The authentication key length MUST be less than or equal to the
* block size of the algorithm. It is the callers responsibility to
@@ -292,6 +313,8 @@ struct rte_crypto_auth_xform {
* (for example RFC 2104, FIPS 198a).
*/
+ uint8_t reserved1[6];
+
struct {
uint16_t offset;
/**< Starting point for Initialisation Vector or Counter,
@@ -376,7 +399,12 @@ struct rte_crypto_aead_xform {
struct {
const uint8_t *data; /**< pointer to key data */
uint16_t length; /**< key length in bytes */
- } key;
+ } __attribute__((__packed__)) key;
+
+ /** offset for cipher to start within data buffer */
+ uint16_t cipher_offset;
+
+ uint8_t reserved1[4];
struct {
uint16_t offset;
diff --git a/lib/librte_cryptodev/rte_cryptodev.h b/lib/librte_cryptodev/rte_cryptodev.h
index e175b838c..c0c7bfed7 100644
--- a/lib/librte_cryptodev/rte_cryptodev.h
+++ b/lib/librte_cryptodev/rte_cryptodev.h
@@ -1272,6 +1272,101 @@ void *
rte_cryptodev_sym_session_get_user_data(
struct rte_cryptodev_sym_session *sess);
+/*
+ * After several thoughts decided not to try to squeeze CPU_CRYPTO
+ * into existing rte_crypto_sym_session structure/API, but instead
+ * introduce an extentsion to it via new fully opaque
+ * struct rte_crypto_cpu_sym_session and additional related API.
+ * Main points:
+ * - Current crypto-dev API is reasonably mature and it is desirable
+ * to keep it unchanged (API/ABI stability). From other side, this
+ * new sync API is new one and probably would require extra changes.
+ * Having it as a new one allows to mark it as experimental, without
+ * affecting existing one.
+ * - Fully opaque cpu_sym_session structure gives more flexibility
+ * to the PMD writers and again allows to avoid ABI breakages in future.
+ * - process() function per set of xforms
+ * allows to expose different process() functions for different
+ * xform combinations. PMD writer can decide, does he wants to
+ * push all supported algorithms into one process() function,
+ * or spread it across several ones.
+ * I.E. More flexibility for PMD writer.
+ * - Not storing process() pointer inside the session -
+ * Allows user to choose does he want to store a process() pointer
+ * per session, or per group of sessions for that device that share
+ * the same input xforms. I.E. extra flexibility for the user,
+ * plus allows us to keep cpu_sym_session totally opaque, see above.
+ * Sketched usage model:
+ * ....
+ * /* control path, alloc/init session */
+ * int32_t sz = rte_crypto_cpu_sym_session_size(dev_id, &xform);
+ * struct rte_crypto_cpu_sym_session *ses = user_alloc(..., sz);
+ * rte_crypto_cpu_sym_process_t process =
+ * rte_crypto_cpu_sym_session_func(dev_id, &xform);
+ * rte_crypto_cpu_sym_session_init(dev_id, ses, &xform);
+ * ...
+ * /* data-path*/
+ * process(ses, ....);
+ * ....
+ * /* control path, termiante/free session */
+ * rte_crypto_cpu_sym_session_fini(dev_id, ses);
+ */
+
+/**
+ * vector structure, contains pointer to vector array and the length
+ * of the array
+ */
+struct rte_crypto_vec {
+ struct iovec *vec;
+ uint32_t num;
+};
+
+/*
+ * Data-path bulk process crypto function.
+ */
+typedef void (*rte_crypto_cpu_sym_process_t)(
+ struct rte_crypto_cpu_sym_session *sess,
+ struct rte_crypto_vec buf[], void *iv[], void *aad[],
+ void *digest[], int status[], uint32_t num);
+/*
+ * for given device return process function specific to input xforms
+ * on error - return NULL and set rte_errno value.
+ * Note that for same input xfroms for the same device should return
+ * the same process function.
+ */
+__rte_experimental
+rte_crypto_cpu_sym_process_t
+rte_crypto_cpu_sym_session_func(uint8_t dev_id,
+ const struct rte_crypto_sym_xform *xforms);
+
+/*
+ * Return required session size in bytes for given set of xforms.
+ * if xforms == NULL, then return the max possible session size,
+ * that would fit session for any supported by the device algorithm.
+ * if CPU mode is not supported at all, or requeted in xform
+ * algorithm is not supported, then return -ENOTSUP.
+ */
+__rte_experimental
+int
+rte_crypto_cpu_sym_session_size(uint8_t dev_id,
+ const struct rte_crypto_sym_xform *xforms);
+
+/*
+ * Initialize session.
+ * It is caller responsibility to allocate enough space for it.
+ * See rte_crypto_cpu_sym_session_size above.
+ */
+__rte_experimental
+int rte_crypto_cpu_sym_session_init(uint8_t dev_id,
+ struct rte_crypto_cpu_sym_session *sess,
+ const struct rte_crypto_sym_xform *xforms);
+
+__rte_experimental
+void
+rte_crypto_cpu_sym_session_fini(uint8_t dev_id,
+ struct rte_crypto_cpu_sym_session *sess);
+
+
#ifdef __cplusplus
}
#endif
diff --git a/lib/librte_cryptodev/rte_cryptodev_pmd.h b/lib/librte_cryptodev/rte_cryptodev_pmd.h
index defe05ea0..ed7e63fab 100644
--- a/lib/librte_cryptodev/rte_cryptodev_pmd.h
+++ b/lib/librte_cryptodev/rte_cryptodev_pmd.h
@@ -310,6 +310,20 @@ typedef void (*cryptodev_sym_free_session_t)(struct rte_cryptodev *dev,
typedef void (*cryptodev_asym_free_session_t)(struct rte_cryptodev *dev,
struct rte_cryptodev_asym_session *sess);
+typedef int (*cryptodev_cpu_sym_session_size_t) (struct rte_cryptodev *dev,
+ const struct rte_crypto_sym_xform *xforms);
+
+typedef int (*cryptodev_cpu_sym_session_init_t) (struct rte_cryptodev *dev,
+ struct rte_crypto_cpu_sym_session *sess,
+ const struct rte_crypto_sym_xform *xforms);
+
+typedef void (*cryptodev_cpu_sym_session_fini_t) (struct rte_cryptodev *dev,
+ struct rte_crypto_cpu_sym_session *sess);
+
+typedef rte_crypto_cpu_sym_process_t (*cryptodev_cpu_sym_session_func_t) (
+ struct rte_cryptodev *dev,
+ const struct rte_crypto_sym_xform *xforms);
+
/** Crypto device operations function pointer table */
struct rte_cryptodev_ops {
cryptodev_configure_t dev_configure; /**< Configure device. */
@@ -343,6 +357,11 @@ struct rte_cryptodev_ops {
/**< Clear a Crypto sessions private data. */
cryptodev_asym_free_session_t asym_session_clear;
/**< Clear a Crypto sessions private data. */
+
+ cryptodev_cpu_sym_session_size_t sym_cpu_session_get_size;
+ cryptodev_cpu_sym_session_func_t sym_cpu_session_get_func;
+ cryptodev_cpu_sym_session_init_t sym_cpu_session_init;
+ cryptodev_cpu_sym_session_fini_t sym_cpu_session_fini;
};
^ permalink raw reply [relevance 4%]
* [dpdk-dev] [PATCH v7] eal: make lcore_config private
@ 2019-09-25 16:10 3% Stephen Hemminger
0 siblings, 0 replies; 200+ results
From: Stephen Hemminger @ 2019-09-25 16:10 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger
The internal structure of lcore_config is no longer be part of
visible API/ABI. Make it private to EAL.
Rearrange and resize the fields in the structure so it takes
less memory (and cache footprint).
Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
v7 - add eal_private.h to windows
lib/librte_eal/common/eal_common_launch.c | 2 ++
lib/librte_eal/common/eal_private.h | 22 +++++++++++++++++++++
lib/librte_eal/common/include/rte_lcore.h | 24 -----------------------
lib/librte_eal/common/rte_service.c | 2 ++
lib/librte_eal/rte_eal_version.map | 1 -
lib/librte_eal/windows/eal/eal_thread.c | 1 +
6 files changed, 27 insertions(+), 25 deletions(-)
diff --git a/lib/librte_eal/common/eal_common_launch.c b/lib/librte_eal/common/eal_common_launch.c
index fe0ba3f0d617..cf52d717f68e 100644
--- a/lib/librte_eal/common/eal_common_launch.c
+++ b/lib/librte_eal/common/eal_common_launch.c
@@ -15,6 +15,8 @@
#include <rte_per_lcore.h>
#include <rte_lcore.h>
+#include "eal_private.h"
+
/*
* Wait until a lcore finished its job.
*/
diff --git a/lib/librte_eal/common/eal_private.h b/lib/librte_eal/common/eal_private.h
index 798ede553b21..25e80547904f 100644
--- a/lib/librte_eal/common/eal_private.h
+++ b/lib/librte_eal/common/eal_private.h
@@ -10,6 +10,28 @@
#include <stdio.h>
#include <rte_dev.h>
+#include <rte_lcore.h>
+
+/**
+ * Structure storing internal configuration (per-lcore)
+ */
+struct lcore_config {
+ uint32_t core_id; /**< core number on socket for this lcore */
+ uint32_t core_index; /**< relative index, starting from 0 */
+ uint16_t socket_id; /**< physical socket id for this lcore */
+ uint8_t core_role; /**< role of core eg: OFF, RTE, SERVICE */
+ uint8_t detected; /**< true if lcore was detected */
+ volatile enum rte_lcore_state_t state; /**< lcore state */
+ rte_cpuset_t cpuset; /**< cpu set which the lcore affinity to */
+ pthread_t thread_id; /**< pthread identifier */
+ int pipe_master2slave[2]; /**< communication pipe with master */
+ int pipe_slave2master[2]; /**< communication pipe with master */
+ lcore_function_t * volatile f; /**< function to call */
+ void * volatile arg; /**< argument of function */
+ volatile int ret; /**< return value of function */
+};
+
+extern struct lcore_config lcore_config[RTE_MAX_LCORE];
/**
* Initialize the memzone subsystem (private to eal).
diff --git a/lib/librte_eal/common/include/rte_lcore.h b/lib/librte_eal/common/include/rte_lcore.h
index c86f72eb12a8..0c683919564e 100644
--- a/lib/librte_eal/common/include/rte_lcore.h
+++ b/lib/librte_eal/common/include/rte_lcore.h
@@ -66,30 +66,6 @@ typedef cpuset_t rte_cpuset_t;
} while (0)
#endif
-/**
- * Structure storing internal configuration (per-lcore)
- */
-struct lcore_config {
- unsigned detected; /**< true if lcore was detected */
- pthread_t thread_id; /**< pthread identifier */
- int pipe_master2slave[2]; /**< communication pipe with master */
- int pipe_slave2master[2]; /**< communication pipe with master */
- lcore_function_t * volatile f; /**< function to call */
- void * volatile arg; /**< argument of function */
- volatile int ret; /**< return value of function */
- volatile enum rte_lcore_state_t state; /**< lcore state */
- unsigned socket_id; /**< physical socket id for this lcore */
- unsigned core_id; /**< core number on socket for this lcore */
- int core_index; /**< relative index, starting from 0 */
- rte_cpuset_t cpuset; /**< cpu set which the lcore affinity to */
- uint8_t core_role; /**< role of core eg: OFF, RTE, SERVICE */
-};
-
-/**
- * Internal configuration (per-lcore)
- */
-extern struct lcore_config lcore_config[RTE_MAX_LCORE];
-
RTE_DECLARE_PER_LCORE(unsigned, _lcore_id); /**< Per thread "lcore id". */
RTE_DECLARE_PER_LCORE(rte_cpuset_t, _cpuset); /**< Per thread "cpuset". */
diff --git a/lib/librte_eal/common/rte_service.c b/lib/librte_eal/common/rte_service.c
index c3653ebae46c..6e21f549051b 100644
--- a/lib/librte_eal/common/rte_service.c
+++ b/lib/librte_eal/common/rte_service.c
@@ -21,6 +21,8 @@
#include <rte_memory.h>
#include <rte_malloc.h>
+#include "eal_private.h"
+
#define RTE_SERVICE_NUM_MAX 64
#define SERVICE_F_REGISTERED (1 << 0)
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index 7cbf82d37b0a..aeedf397764f 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -4,7 +4,6 @@ DPDK_2.0 {
__rte_panic;
eal_parse_sysfs_value;
eal_timer_source;
- lcore_config;
per_lcore__lcore_id;
per_lcore__rte_errno;
rte_calloc;
diff --git a/lib/librte_eal/windows/eal/eal_thread.c b/lib/librte_eal/windows/eal/eal_thread.c
index 906502f90982..0591d4c7fb06 100644
--- a/lib/librte_eal/windows/eal/eal_thread.c
+++ b/lib/librte_eal/windows/eal/eal_thread.c
@@ -12,6 +12,7 @@
#include <rte_common.h>
#include <eal_thread.h>
+#include "eal_private.h"
RTE_DEFINE_PER_LCORE(unsigned int, _lcore_id) = LCORE_ID_ANY;
--
2.20.1
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] [dpdk-techboard] [RFC] Proposals and notes from ABI stability panel @ DPDK Userspace
2019-09-25 14:40 7% ` [dpdk-dev] [dpdk-techboard] " Bruce Richardson
2019-09-25 14:49 4% ` Kevin Traynor
@ 2019-09-25 15:06 4% ` Ray Kinsella
1 sibling, 0 replies; 200+ results
From: Ray Kinsella @ 2019-09-25 15:06 UTC (permalink / raw)
To: Bruce Richardson
Cc: Kevin Traynor, dpdk-dev, O'Driscoll, Tim, Brian, techboard
On 25/09/2019 15:40, Bruce Richardson wrote:
> On Wed, Sep 25, 2019 at 03:29:16PM +0100, Ray Kinsella wrote:
>>
>>> In the short term, based on the feedback at the conference and to give
>>> something concrete to be considered, here is a suggestion,
>>>
>>> ABI freeze starts at 20.02 for 9 months, with a review as planned to see
>>> if 20.11 should be frozen 2 years.
>>>
>>> pros:
>>> + Eliminates any need for delaying 19.11 release
>>>
>>> + Allows maintainers to stick to current deprecation policy if they need
>>> to make changes prior to freeze (Based on comment from Hemmant)
>>>
>>> + Not sure if it's worthy of a new bullet or clear from above but I
>>> would add that changing the release cycle/deprecation policy etc 2 weeks
>>> (I think) before RC1 is late to say the least and there is no notice to
>>> users
>>>
>>> + Means that any changes required prior to freeze are not rushed with
>>> usual big LTS release (19.11). Gives more time and maybe during a saner
>>> release cycle (20.02)
>>>
>>> cons:
>>> - With view for possible 20.11 freeze, gives 2 releases to tease out
>>> process instead of 3
>>>
>>> - Perhaps it is desirable for some users to have the 19.11 LTS ABI
>>> compatible with 20.02/05/08 releases
>>>
>>> I've tried to keep them objective, of course people will have different
>>> opinions about starting a freeze now vs. later etc. too.
>>>
>>> thanks,
>>> Kevin.
>>>
>>
>> *interesting*
>>
>> Another approach, possibly better approach, is to see the LTS as the
>> final act following an ABI declaration/freeze.
>>
>> We we declare the v20 ABI in DPDK 20.02, and hold that ABI until 21.02
>> including the v20.11 LTS. The LTS then becomes the cumulation of the ABI
>> freeze.
>>
>> I didn't go this road, because of the community habit of pushing things
>> in just before the LTS, I thought it would be a bridge too far, and that
>> it would get considerable push back.
>
> I actually think this approach was initially rejected as having an ABI
> break immediately after an LTS makes backporting fixes to the LTS more
> problematic.
>
> /Bruce
>
That too ..
^ permalink raw reply [relevance 4%]
* Re: [dpdk-dev] [dpdk-techboard] [RFC] Proposals and notes from ABI stability panel @ DPDK Userspace
2019-09-25 14:40 7% ` [dpdk-dev] [dpdk-techboard] " Bruce Richardson
@ 2019-09-25 14:49 4% ` Kevin Traynor
2019-09-25 15:06 4% ` Ray Kinsella
1 sibling, 0 replies; 200+ results
From: Kevin Traynor @ 2019-09-25 14:49 UTC (permalink / raw)
To: Bruce Richardson, Ray Kinsella
Cc: dpdk-dev, O'Driscoll, Tim, Brian, techboard
On 25/09/2019 15:40, Bruce Richardson wrote:
> On Wed, Sep 25, 2019 at 03:29:16PM +0100, Ray Kinsella wrote:
>>
>>> In the short term, based on the feedback at the conference and to give
>>> something concrete to be considered, here is a suggestion,
>>>
>>> ABI freeze starts at 20.02 for 9 months, with a review as planned to see
>>> if 20.11 should be frozen 2 years.
>>>
>>> pros:
>>> + Eliminates any need for delaying 19.11 release
>>>
>>> + Allows maintainers to stick to current deprecation policy if they need
>>> to make changes prior to freeze (Based on comment from Hemmant)
>>>
>>> + Not sure if it's worthy of a new bullet or clear from above but I
>>> would add that changing the release cycle/deprecation policy etc 2 weeks
>>> (I think) before RC1 is late to say the least and there is no notice to
>>> users
>>>
>>> + Means that any changes required prior to freeze are not rushed with
>>> usual big LTS release (19.11). Gives more time and maybe during a saner
>>> release cycle (20.02)
>>>
>>> cons:
>>> - With view for possible 20.11 freeze, gives 2 releases to tease out
>>> process instead of 3
>>>
>>> - Perhaps it is desirable for some users to have the 19.11 LTS ABI
>>> compatible with 20.02/05/08 releases
>>>
>>> I've tried to keep them objective, of course people will have different
>>> opinions about starting a freeze now vs. later etc. too.
>>>
>>> thanks,
>>> Kevin.
>>>
>>
>> *interesting*
>>
>> Another approach, possibly better approach, is to see the LTS as the
>> final act following an ABI declaration/freeze.
>>
>> We we declare the v20 ABI in DPDK 20.02, and hold that ABI until 21.02
>> including the v20.11 LTS. The LTS then becomes the cumulation of the ABI
>> freeze.
>>
>> I didn't go this road, because of the community habit of pushing things
>> in just before the LTS, I thought it would be a bridge too far, and that
>> it would get considerable push back.
>
> I actually think this approach was initially rejected as having an ABI
> break immediately after an LTS makes backporting fixes to the LTS more
> problematic.
>
Yeah, it likely would. I guess the freeze cycle or end date of the
freeze trial (if i can call it that) could be discussed further later,
but the start date is the more immediate issue now.
> /Bruce
>
^ permalink raw reply [relevance 4%]
* Re: [dpdk-dev] [PATCH v7 10/10] vhost: add vhost-user-blk example which support inflight
@ 2019-09-25 14:45 3% ` Tiwei Bie
2019-09-26 14:29 0% ` Yu, Jin
0 siblings, 1 reply; 200+ results
From: Tiwei Bie @ 2019-09-25 14:45 UTC (permalink / raw)
To: Jin Yu; +Cc: dev, changpeng.liu, maxime.coquelin, zhihong.wang
On Fri, Sep 20, 2019 at 08:01:02PM +0800, Jin Yu wrote:
> A vhost-user-blk example that support inflight feature. It uses the
> new APIs that introduced in the first patch, so It can show how there
s/It/it/
s/there/these/
> APIs work to support inflight feature.
>
> Signed-off-by: Jin Yu <jin.yu@intel.com>
> ---
> V1 - add the case.
> V2 - add the rte_vhost prefix.
> V3 - add packed ring support
> ---
> examples/vhost_blk/Makefile | 67 ++
> examples/vhost_blk/blk.c | 125 +++
> examples/vhost_blk/blk_spec.h | 95 ++
> examples/vhost_blk/meson.build | 20 +
> examples/vhost_blk/vhost_blk.c | 1313 +++++++++++++++++++++++++
> examples/vhost_blk/vhost_blk.h | 116 +++
> examples/vhost_blk/vhost_blk_compat.c | 195 ++++
> 7 files changed, 1931 insertions(+)
I met some build issues when trying this example.
examples/vhost_blk/vhost_blk.c: In function ‘descriptor_get_next_packed’:
examples/vhost_blk/vhost_blk.c:71:21: error: invalid use of undefined type ‘struct vring_packed_desc’
if (vq->desc_packed[*idx % vq->size].flags & VIRTQ_DESC_F_NEXT) {
^
examples/vhost_blk/vhost_blk.c:71:21: error: dereferencing pointer to incomplete type ‘struct vring_packed_desc’
examples/vhost_blk/vhost_blk.c:73:26: error: invalid use of undefined type ‘struct vring_packed_desc’
return &vq->desc_packed[*idx % vq->size];
^
examples/vhost_blk/vhost_blk.c: In function ‘inflight_submit_completion_packed’:
examples/vhost_blk/vhost_blk.c:165:2: warning: ‘rte_vhost_set_last_inflight_io_packed’ is deprecated: Symbol is not yet part of stable ABI [-Wdeprecated-declarations]
ret = rte_vhost_set_last_inflight_io_packed(ctrlr->bdev->vid, q_idx,
^~~
In file included from examples/vhost_blk/vhost_blk.c:17:0:
x86_64-native-linuxapp-gcc/include/rte_vhost.h:810:1: note: declared here
rte_vhost_set_last_inflight_io_packed(int vid,
^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
^
...
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] [dpdk-techboard] [RFC] Proposals and notes from ABI stability panel @ DPDK Userspace
2019-09-25 14:29 9% ` Ray Kinsella
@ 2019-09-25 14:40 7% ` Bruce Richardson
2019-09-25 14:49 4% ` Kevin Traynor
2019-09-25 15:06 4% ` Ray Kinsella
0 siblings, 2 replies; 200+ results
From: Bruce Richardson @ 2019-09-25 14:40 UTC (permalink / raw)
To: Ray Kinsella
Cc: Kevin Traynor, dpdk-dev, O'Driscoll, Tim, Brian, techboard
On Wed, Sep 25, 2019 at 03:29:16PM +0100, Ray Kinsella wrote:
>
> > In the short term, based on the feedback at the conference and to give
> > something concrete to be considered, here is a suggestion,
> >
> > ABI freeze starts at 20.02 for 9 months, with a review as planned to see
> > if 20.11 should be frozen 2 years.
> >
> > pros:
> > + Eliminates any need for delaying 19.11 release
> >
> > + Allows maintainers to stick to current deprecation policy if they need
> > to make changes prior to freeze (Based on comment from Hemmant)
> >
> > + Not sure if it's worthy of a new bullet or clear from above but I
> > would add that changing the release cycle/deprecation policy etc 2 weeks
> > (I think) before RC1 is late to say the least and there is no notice to
> > users
> >
> > + Means that any changes required prior to freeze are not rushed with
> > usual big LTS release (19.11). Gives more time and maybe during a saner
> > release cycle (20.02)
> >
> > cons:
> > - With view for possible 20.11 freeze, gives 2 releases to tease out
> > process instead of 3
> >
> > - Perhaps it is desirable for some users to have the 19.11 LTS ABI
> > compatible with 20.02/05/08 releases
> >
> > I've tried to keep them objective, of course people will have different
> > opinions about starting a freeze now vs. later etc. too.
> >
> > thanks,
> > Kevin.
> >
>
> *interesting*
>
> Another approach, possibly better approach, is to see the LTS as the
> final act following an ABI declaration/freeze.
>
> We we declare the v20 ABI in DPDK 20.02, and hold that ABI until 21.02
> including the v20.11 LTS. The LTS then becomes the cumulation of the ABI
> freeze.
>
> I didn't go this road, because of the community habit of pushing things
> in just before the LTS, I thought it would be a bridge too far, and that
> it would get considerable push back.
I actually think this approach was initially rejected as having an ABI
break immediately after an LTS makes backporting fixes to the LTS more
problematic.
/Bruce
^ permalink raw reply [relevance 7%]
* Re: [dpdk-dev] [PATCH v4 1/4] doc: separate versioning.rst into version and policy
2019-09-25 13:01 3% ` Ray Kinsella
@ 2019-09-25 14:34 0% ` Neil Horman
0 siblings, 0 replies; 200+ results
From: Neil Horman @ 2019-09-25 14:34 UTC (permalink / raw)
To: Ray Kinsella
Cc: dev, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, maxime.coquelin,
john.mcnamara, marko.kovacevic, hemant.agrawal, ktraynor
On Wed, Sep 25, 2019 at 02:01:01PM +0100, Ray Kinsella wrote:
> Hi Neil,
>
> Thanks for the feedback, other comment below.
>
> On 25/09/2019 13:24, Neil Horman wrote:
> > On Wed, Sep 25, 2019 at 11:23:53AM +0100, Ray Kinsella wrote:
> >> Separate versioning.rst into abi versioning and abi policy guidance, in
> >> preparation for adding more detail to the abi policy.
> >>
> >> Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
> >> ---
> >> doc/guides/contributing/abi_policy.rst | 169 +++++++++
> >> doc/guides/contributing/abi_versioning.rst | 427 +++++++++++++++++++++
> >> doc/guides/contributing/index.rst | 3 +-
> >> doc/guides/contributing/versioning.rst | 591 -----------------------------
> >> 4 files changed, 598 insertions(+), 592 deletions(-)
> >> create mode 100644 doc/guides/contributing/abi_policy.rst
> >> create mode 100644 doc/guides/contributing/abi_versioning.rst
> >> delete mode 100644 doc/guides/contributing/versioning.rst
> >>
> >> diff --git a/doc/guides/contributing/abi_policy.rst b/doc/guides/contributing/abi_policy.rst
> >> new file mode 100644
> >> index 0000000..55bacb4
> >> --- /dev/null
> >> +++ b/doc/guides/contributing/abi_policy.rst
> >> @@ -0,0 +1,169 @@
> >> +.. SPDX-License-Identifier: BSD-3-Clause
> >> + Copyright 2018 The DPDK contributors
> >> +
> >> +.. abi_api_policy:
> >> +
> >> +DPDK ABI/API policy
> >> +===================
> >> +
> >> +Description
> >> +-----------
> >> +
> >> +This document details some methods for handling ABI management in the DPDK.
> >> +
> >> +General Guidelines
> >> +------------------
> >> +
> >> +#. Whenever possible, ABI should be preserved
> >> +#. ABI/API may be changed with a deprecation process
> >> +#. The modification of symbols can generally be managed with versioning
> >> +#. Libraries or APIs marked in ``experimental`` state may change without constraint
> >> +#. New APIs will be marked as ``experimental`` for at least one release to allow
> >> + any issues found by users of the new API to be fixed quickly
> >> +#. The addition of symbols is generally not problematic
> >> +#. The removal of symbols generally is an ABI break and requires bumping of the
> >> + LIBABIVER macro
> >> +#. Updates to the minimum hardware requirements, which drop support for hardware which
> >> + was previously supported, should be treated as an ABI change.
> >> +
> >> +What is an ABI
> >> +~~~~~~~~~~~~~~
> >> +
> >> +An ABI (Application Binary Interface) is the set of runtime interfaces exposed
> >> +by a library. It is similar to an API (Application Programming Interface) but
> >> +is the result of compilation. It is also effectively cloned when applications
> >> +link to dynamic libraries. That is to say when an application is compiled to
> >> +link against dynamic libraries, it is assumed that the ABI remains constant
> >> +between the time the application is compiled/linked, and the time that it runs.
> >> +Therefore, in the case of dynamic linking, it is critical that an ABI is
> >> +preserved, or (when modified), done in such a way that the application is unable
> >> +to behave improperly or in an unexpected fashion.
> >> +
> >> +
> >> +ABI/API Deprecation
> >> +-------------------
> >> +
> >> +The DPDK ABI policy
> >> +~~~~~~~~~~~~~~~~~~~
> >> +
> >> +ABI versions are set at the time of major release labeling, and the ABI may
> >> +change multiple times, without warning, between the last release label and the
> >> +HEAD label of the git tree.
> >> +
> >> +ABI versions, once released, are available until such time as their
> >> +deprecation has been noted in the Release Notes for at least one major release
> >> +cycle. For example consider the case where the ABI for DPDK 2.0 has been
> >> +shipped and then a decision is made to modify it during the development of
> >> +DPDK 2.1. The decision will be recorded in the Release Notes for the DPDK 2.1
> >> +release and the modification will be made available in the DPDK 2.2 release.
> >> +
> > This seems..confusing.
>
> Agreed, this is from the original policy. I updated all the references
> to DPDK 2.0 in the abi_versioning document. Clearly missed these ones,
> thanks for that, the text is confusing I will update it.
>
Thank you!
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [RFC] Proposals and notes from ABI stability panel @ DPDK Userspace
2019-09-25 13:31 8% ` Kevin Traynor
@ 2019-09-25 14:29 9% ` Ray Kinsella
2019-09-25 14:40 7% ` [dpdk-dev] [dpdk-techboard] " Bruce Richardson
0 siblings, 1 reply; 200+ results
From: Ray Kinsella @ 2019-09-25 14:29 UTC (permalink / raw)
To: Kevin Traynor, dpdk-dev, O'Driscoll, Tim, Brian; +Cc: techboard
> In the short term, based on the feedback at the conference and to give
> something concrete to be considered, here is a suggestion,
>
> ABI freeze starts at 20.02 for 9 months, with a review as planned to see
> if 20.11 should be frozen 2 years.
>
> pros:
> + Eliminates any need for delaying 19.11 release
>
> + Allows maintainers to stick to current deprecation policy if they need
> to make changes prior to freeze (Based on comment from Hemmant)
>
> + Not sure if it's worthy of a new bullet or clear from above but I
> would add that changing the release cycle/deprecation policy etc 2 weeks
> (I think) before RC1 is late to say the least and there is no notice to
> users
>
> + Means that any changes required prior to freeze are not rushed with
> usual big LTS release (19.11). Gives more time and maybe during a saner
> release cycle (20.02)
>
> cons:
> - With view for possible 20.11 freeze, gives 2 releases to tease out
> process instead of 3
>
> - Perhaps it is desirable for some users to have the 19.11 LTS ABI
> compatible with 20.02/05/08 releases
>
> I've tried to keep them objective, of course people will have different
> opinions about starting a freeze now vs. later etc. too.
>
> thanks,
> Kevin.
>
*interesting*
Another approach, possibly better approach, is to see the LTS as the
final act following an ABI declaration/freeze.
We we declare the v20 ABI in DPDK 20.02, and hold that ABI until 21.02
including the v20.11 LTS. The LTS then becomes the cumulation of the ABI
freeze.
I didn't go this road, because of the community habit of pushing things
in just before the LTS, I thought it would be a bridge too far, and that
it would get considerable push back.
^ permalink raw reply [relevance 9%]
* [dpdk-dev] [PATCH v2 15/17] net/hinic: add hinic PMD doc files
@ 2019-09-25 14:30 4% ` Xiaoyun wang
2019-09-26 18:51 0% ` Ferruh Yigit
0 siblings, 1 reply; 200+ results
From: Xiaoyun wang @ 2019-09-25 14:30 UTC (permalink / raw)
To: ferruh.yigit
Cc: dev, xuanziyang2, shahar.belkar, luoxianjun, tanya.brokhman,
zhouguoyang, wulike1, Xiaoyun wang
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset="UTF-8", Size: 5042 bytes --]
Add doc files about new features and modification.
Signed-off-by: Xiaoyun wang <cloud.wangxiaoyun@huawei.com>
---
doc/guides/nics/features/hinic.ini | 12 ++++++++-
doc/guides/nics/hinic.rst | 6 +++++
doc/guides/rel_notes/release_19_11.rst | 45 ++++++----------------------------
3 files changed, 25 insertions(+), 38 deletions(-)
diff --git a/doc/guides/nics/features/hinic.ini b/doc/guides/nics/features/hinic.ini
index fe063d6..dc02b4b 100644
--- a/doc/guides/nics/features/hinic.ini
+++ b/doc/guides/nics/features/hinic.ini
@@ -9,16 +9,22 @@ Link status = Y
Link status event = Y
Free Tx mbuf on demand = Y
Queue start/stop = Y
-Jumbo frame = N
+MTU update = Y
+Jumbo frame = Y
Scattered Rx = Y
TSO = Y
+LRO = Y
Promiscuous mode = Y
+Allmulticast mode = Y
Unicast MAC filter = Y
Multicast MAC filter = Y
RSS hash = Y
RSS key update = Y
RSS reta update = Y
Inner RSS = Y
+SR-IOV = Y
+VLAN filter = Y
+VLAN offload = Y
CRC offload = Y
L3 checksum offload = Y
L4 checksum offload = Y
@@ -27,6 +33,10 @@ Inner L4 checksum = Y
Basic stats = Y
Extended stats = Y
Stats per queue = Y
+Flow director = Y
+Flow control = Y
+FW version = Y
+Multiprocess aware = Y
Linux UIO = Y
Linux VFIO = Y
BSD nic_uio = N
diff --git a/doc/guides/nics/hinic.rst b/doc/guides/nics/hinic.rst
index c9329bc..f036fc5 100644
--- a/doc/guides/nics/hinic.rst
+++ b/doc/guides/nics/hinic.rst
@@ -24,6 +24,12 @@ Features
- Link state information
- Link flow control
- Scattered and gather for TX and RX
+- SR¨CIOV - Partially supported at this point, VFIO only
+- Allmulticast mode
+- Unicast MAC filter
+- Multicast MAC filter
+- FW version
+- Flow director
Prerequisites
-------------
diff --git a/doc/guides/rel_notes/release_19_11.rst b/doc/guides/rel_notes/release_19_11.rst
index 65361c4..6c6f27f 100644
--- a/doc/guides/rel_notes/release_19_11.rst
+++ b/doc/guides/rel_notes/release_19_11.rst
@@ -56,11 +56,15 @@ New Features
Also, make sure to start the actual text at the margin.
=========================================================
-* **Updated the Intel ice driver.**
+* **Updated the Huawei hinic driver.**
- Updated the Intel ice driver with new features and improvements, including:
+ Updated the Huawei hinic driver with new features and improvements, including:
- * Added support for device-specific DDP package loading.
+ * Enabled SR-IOV - Partially supported at this point, VFIO only.
+ * Supported VLAN filter and VLAN offload.
+ * Supported Unicast MAC filter and Multicast MAC filter.
+ * Supported FW version get.
+ * Supported Flow director for LACP, VRRP, BGP and so on.
Removed Items
-------------
@@ -99,30 +103,6 @@ API Changes
Also, make sure to start the actual text at the margin.
=========================================================
-* ethdev: changed ``rte_eth_dev_infos_get`` return value from ``void`` to
- ``int`` to provide a way to report various error conditions.
-
-* ethdev: changed ``rte_eth_promiscuous_enable`` and
- ``rte_eth_promiscuous_disable`` return value from ``void`` to ``int`` to
- provide a way to report various error conditions.
-
-* ethdev: changed ``rte_eth_allmulticast_enable`` and
- ``rte_eth_allmulticast_disable`` return value from ``void`` to ``int`` to
- provide a way to report various error conditions.
-
-* ethdev: changed ``rte_eth_dev_xstats_reset`` return value from ``void`` to
- ``int`` to provide a way to report various error conditions.
-
-* ethdev: changed ``rte_eth_link_get`` and ``rte_eth_link_get_nowait``
- return value from ``void`` to ``int`` to provide a way to report various
- error conditions.
-
-* ethdev: changed ``rte_eth_macaddr_get`` return value from ``void`` to
- ``int`` to provide a way to report various error conditions.
-
-* ethdev: changed ``rte_eth_dev_owner_delete`` return value from ``void`` to
- ``int`` to provide a way to report various error conditions.
-
ABI Changes
-----------
@@ -174,7 +154,7 @@ The libraries prepended with a plus sign were incremented in this version.
librte_distributor.so.1
librte_eal.so.11
librte_efd.so.1
- + librte_ethdev.so.13
+ librte_ethdev.so.12
librte_eventdev.so.7
librte_flow_classify.so.1
librte_gro.so.1
@@ -252,12 +232,3 @@ Tested Platforms
Also, make sure to start the actual text at the margin.
=========================================================
-* **Updated Mellanox mlx5 driver.**
-
- Updated Mellanox mlx5 driver with new features and improvements, including:
-
- * Added support for VLAN pop flow offload command.
- * Added support for VLAN push flow offload command.
- * Added support for VLAN set PCP offload command.
- * Added support for VLAN set VID offload command.
-
--
1.8.3.1
^ permalink raw reply [relevance 4%]
* Re: [dpdk-dev] [RFC] Proposals and notes from ABI stability panel @ DPDK Userspace
2019-09-23 17:51 9% [dpdk-dev] [RFC] Proposals and notes from ABI stability panel @ DPDK Userspace Ray Kinsella
@ 2019-09-25 13:31 8% ` Kevin Traynor
2019-09-25 14:29 9% ` Ray Kinsella
0 siblings, 1 reply; 200+ results
From: Kevin Traynor @ 2019-09-25 13:31 UTC (permalink / raw)
To: Ray Kinsella, dpdk-dev, O'Driscoll, Tim, Brian; +Cc: techboard
On 23/09/2019 18:51, Ray Kinsella wrote:
> Folks,
>
> As you may be aware, there was a panel on ABI Stability @ DPDK
> Userspace. There where a number of proposed amendments to the ABI
> stability proposal made, as well as a number of points and comments, you
> will find all these below. The proposals needs further discussion so
> please chime in below.
>
> Thanks to Tim for capturing while I was busy on the stage.
>
Thanks for the notes Tim,
> Thanks,
>
> Ray K
>
>
> Table of Contents
> _________________
>
> 1. Proposals from the panel discussion.
> .. 1. Developer releases (versus User Releases)
> .. 2. Core and non-core packaging
> .. 3. Approach for public data structures
> .. 4. Delaying v19.11
> 2. Other notes from the panel discussion.
> .. 1. Performance as the paramount goal.
> .. 2. Length of ABI Stability.
> .. 3. Testing ABI Stability
> .. 4. Call to action
>
>
> 1 Proposals from the panel discussion.
> ======================================
>
> 1.1 Developer releases (versus User Releases)
> ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
>
> - Summary: Differentiate between "developer" and "user" releases.
> - 1 year is too long to wait to upstream a new feature which breaks
> ABI.
> - Developer releases would be for use by the development community
> only.
> - A proposed compromise was that the .08 release would be the only
> "developer release".
>
>
> 1.2 Core and non-core packaging
> ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
>
> - Summary: OS packaging doesn't include all libraries.
> - This would create a delta between the community ABI, and the OS
> packaging.
> - OS packagers rational is that some libraries are used very rarely.
>
>
> 1.3 Approach for public data structures
> ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
>
> - Summary: Public/exposed data structures are tricky for ABI
> stability.
> - See discussion @
>
> <http://inbox.dpdk.org/dev/980083c6-130a-9658-f82b-0c9ddc7cc0cc@ashroe.eu/>
>
>
> 1.4 Delaying v19.11
> ~~~~~~~~~~~~~~~~~~~
>
> - Summary: push v19.11 to v19.12 to make more time to prepare and wave
> the depreciation process.
> - OVS take Nov LTS release in their Feb release. Delaying 19.11 may
> have an impact on this.
To give some more info about OVS. tldr there is a soft freeze on Jan 1st
but an update that is being discussed/reviewed could go in until Jan
15th when the OVS 2.13 branch would be created.
Thomas is indeed right in that there is a development branch in OVS that
is intended to keep up with DPDK master with a view to finding any
integration issues early.
More info here
http://docs.openvswitch.org/en/latest/internals/release-process/#release-strategy
> - Not easy to change release at this stage as many things depend on
> it (OS distros etc.).
>
In the short term, based on the feedback at the conference and to give
something concrete to be considered, here is a suggestion,
ABI freeze starts at 20.02 for 9 months, with a review as planned to see
if 20.11 should be frozen 2 years.
pros:
+ Eliminates any need for delaying 19.11 release
+ Allows maintainers to stick to current deprecation policy if they need
to make changes prior to freeze (Based on comment from Hemmant)
+ Not sure if it's worthy of a new bullet or clear from above but I
would add that changing the release cycle/deprecation policy etc 2 weeks
(I think) before RC1 is late to say the least and there is no notice to
users
+ Means that any changes required prior to freeze are not rushed with
usual big LTS release (19.11). Gives more time and maybe during a saner
release cycle (20.02)
cons:
- With view for possible 20.11 freeze, gives 2 releases to tease out
process instead of 3
- Perhaps it is desirable for some users to have the 19.11 LTS ABI
compatible with 20.02/05/08 releases
I've tried to keep them objective, of course people will have different
opinions about starting a freeze now vs. later etc. too.
thanks,
Kevin.
>
> 2 Other notes from the panel discussion.
> ========================================
>
> 2.1 Performance as the paramount goal.
> ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
>
> - Some users, would trade performance for better readability and
> debug-ability.
> - Skepticism that micro-benchmarks reflect real world performance.
>
>
> 2.2 Length of ABI Stability.
> ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
>
> - Some users, questioned if 1 year would be long enough.
> - It was clarified that the 1 year period, would be reviewed after the
> first year with the intention of lengthening the period.
>
>
> 2.3 Testing ABI Stability
> ~~~~~~~~~~~~~~~~~~~~~~~~~
>
> - More work for developers and validation teams. Need to validate
> multiple paths for symbol versioning.
>
>
> 2.4 Call to action
> ~~~~~~~~~~~~~~~~~~
>
> - We should do something. So we don’t want to have the same
> conversation again in a year.
>
^ permalink raw reply [relevance 8%]
* Re: [dpdk-dev] [PATCH v4 1/4] doc: separate versioning.rst into version and policy
2019-09-25 12:24 5% ` Neil Horman
@ 2019-09-25 13:01 3% ` Ray Kinsella
2019-09-25 14:34 0% ` Neil Horman
2019-09-27 15:22 4% ` Ray Kinsella
1 sibling, 1 reply; 200+ results
From: Ray Kinsella @ 2019-09-25 13:01 UTC (permalink / raw)
To: Neil Horman
Cc: dev, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, maxime.coquelin,
john.mcnamara, marko.kovacevic, hemant.agrawal, ktraynor
Hi Neil,
Thanks for the feedback, other comment below.
On 25/09/2019 13:24, Neil Horman wrote:
> On Wed, Sep 25, 2019 at 11:23:53AM +0100, Ray Kinsella wrote:
>> Separate versioning.rst into abi versioning and abi policy guidance, in
>> preparation for adding more detail to the abi policy.
>>
>> Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
>> ---
>> doc/guides/contributing/abi_policy.rst | 169 +++++++++
>> doc/guides/contributing/abi_versioning.rst | 427 +++++++++++++++++++++
>> doc/guides/contributing/index.rst | 3 +-
>> doc/guides/contributing/versioning.rst | 591 -----------------------------
>> 4 files changed, 598 insertions(+), 592 deletions(-)
>> create mode 100644 doc/guides/contributing/abi_policy.rst
>> create mode 100644 doc/guides/contributing/abi_versioning.rst
>> delete mode 100644 doc/guides/contributing/versioning.rst
>>
>> diff --git a/doc/guides/contributing/abi_policy.rst b/doc/guides/contributing/abi_policy.rst
>> new file mode 100644
>> index 0000000..55bacb4
>> --- /dev/null
>> +++ b/doc/guides/contributing/abi_policy.rst
>> @@ -0,0 +1,169 @@
>> +.. SPDX-License-Identifier: BSD-3-Clause
>> + Copyright 2018 The DPDK contributors
>> +
>> +.. abi_api_policy:
>> +
>> +DPDK ABI/API policy
>> +===================
>> +
>> +Description
>> +-----------
>> +
>> +This document details some methods for handling ABI management in the DPDK.
>> +
>> +General Guidelines
>> +------------------
>> +
>> +#. Whenever possible, ABI should be preserved
>> +#. ABI/API may be changed with a deprecation process
>> +#. The modification of symbols can generally be managed with versioning
>> +#. Libraries or APIs marked in ``experimental`` state may change without constraint
>> +#. New APIs will be marked as ``experimental`` for at least one release to allow
>> + any issues found by users of the new API to be fixed quickly
>> +#. The addition of symbols is generally not problematic
>> +#. The removal of symbols generally is an ABI break and requires bumping of the
>> + LIBABIVER macro
>> +#. Updates to the minimum hardware requirements, which drop support for hardware which
>> + was previously supported, should be treated as an ABI change.
>> +
>> +What is an ABI
>> +~~~~~~~~~~~~~~
>> +
>> +An ABI (Application Binary Interface) is the set of runtime interfaces exposed
>> +by a library. It is similar to an API (Application Programming Interface) but
>> +is the result of compilation. It is also effectively cloned when applications
>> +link to dynamic libraries. That is to say when an application is compiled to
>> +link against dynamic libraries, it is assumed that the ABI remains constant
>> +between the time the application is compiled/linked, and the time that it runs.
>> +Therefore, in the case of dynamic linking, it is critical that an ABI is
>> +preserved, or (when modified), done in such a way that the application is unable
>> +to behave improperly or in an unexpected fashion.
>> +
>> +
>> +ABI/API Deprecation
>> +-------------------
>> +
>> +The DPDK ABI policy
>> +~~~~~~~~~~~~~~~~~~~
>> +
>> +ABI versions are set at the time of major release labeling, and the ABI may
>> +change multiple times, without warning, between the last release label and the
>> +HEAD label of the git tree.
>> +
>> +ABI versions, once released, are available until such time as their
>> +deprecation has been noted in the Release Notes for at least one major release
>> +cycle. For example consider the case where the ABI for DPDK 2.0 has been
>> +shipped and then a decision is made to modify it during the development of
>> +DPDK 2.1. The decision will be recorded in the Release Notes for the DPDK 2.1
>> +release and the modification will be made available in the DPDK 2.2 release.
>> +
> This seems..confusing.
Agreed, this is from the original policy. I updated all the references
to DPDK 2.0 in the abi_versioning document. Clearly missed these ones,
thanks for that, the text is confusing I will update it.
> In patch 0:
> =================================================================
> * DPDK v20 is declared as the supported ABI version for one year, aligned with
> the DPDK v19.11 (LTS) release. All library sonames are updated to reflect the
> new ABI version, e.g. librte_eal.so.20, librte_acl.so.20...
> * DPDK v20.02 .. v20.08 releases are ABI compatible with the DPDK v20 ABI. ABI
> changes are permitted from DPDK v20.02 onwards, with the condition that ABI
> compatibility with DPDK v20 is preserved.
> * DPDK v21 is declared as the new supported ABI version for two years, aligned
> with the DPDK v20.11 (LTS) release. The DPDK v20 ABI is now depreciated,
> library sonames are updated to v21 and ABI compatibility breaking changes may
> be introduced.
> ===================================================================
>
> Issues I see:
> 1) We have lots of version numbers floating around here:
> v20 (referencing an ABI version I think),
yes
> DPDK 19.11 (an LTS release that maps to ABI v20), dpdk 20.02..
yes
> dpdk 20.08 which can modify the ABI as long as they maintain backwards compatibility (I think)
yes
> dpdk v21 (referecing a new ABI that will be supported at a later release),
yes
> dpdk 20.11 which guarantees ABI v21,
yes, and depreciates v20
> dpdk 2.0 which maps to
> abi v20, dpdk 2.1 (a minor release which decides to break ABI), and dpdk 2.2
> (a subsequent minor release which adheres to a new abi)
references to 2.x should be removed.
> 2) Conflicts as to when ABI can be modified in breaking and compatible ways.
> Are we allowed to break abi after 1 year, or only in a new major release
ABI breaking is permitted at the declaration of a new major release, at
the moment, aligned with the LTS. Other than, aligned with the quarterly
releases, only in compatible ways.
>
> I think you need a taxonomy, to clearly deliniate your syntax for noting abi
> versions, vs dpdk release major versions, and minor versions, so we are more
> clear as to what the docs are referring to, as well as perhaps a timeline to
> more clearly illustrate when compatible and incompatible ABI changes are
> allowed.
I agree - I think a diagram would even better.
I have a good starting point for this.
>
>
>> +ABI versions may be deprecated in whole or in part as needed by a given
>> +update.
>> +
>> +Some ABI changes may be too significant to reasonably maintain multiple
>> +versions. In those cases ABI's may be updated without backward compatibility
>> +being provided. The requirements for doing so are:
>> +
>> +#. At least 3 acknowledgments of the need to do so must be made on the
>> + dpdk.org mailing list.
>> +
>> + - The acknowledgment of the maintainer of the component is mandatory, or if
>> + no maintainer is available for the component, the tree/sub-tree maintainer
>> + for that component must acknowledge the ABI change instead.
>> +
>> + - It is also recommended that acknowledgments from different "areas of
>> + interest" be sought for each deprecation, for example: from NIC vendors,
>> + CPU vendors, end-users, etc.
>> +
>> +#. The changes (including an alternative map file) can be included with
>> + deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option,
>> + to provide more details about oncoming changes.
>> + ``RTE_NEXT_ABI`` wrapper will be removed when it become the default ABI.
>> + More preferred way to provide this information is sending the feature
>> + as a separate patch and reference it in deprecation notice.
>> +
>> +#. A full deprecation cycle, as explained above, must be made to offer
>> + downstream consumers sufficient warning of the change.
>> +
>> +Note that the above process for ABI deprecation should not be undertaken
>> +lightly. ABI stability is extremely important for downstream consumers of the
>> +DPDK, especially when distributed in shared object form. Every effort should
>> +be made to preserve the ABI whenever possible. The ABI should only be changed
>> +for significant reasons, such as performance enhancements. ABI breakage due to
>> +changes such as reorganizing public structure fields for aesthetic or
>> +readability purposes should be avoided.
>> +
>> +.. note::
>> +
>> + Updates to the minimum hardware requirements, which drop support for hardware
>> + which was previously supported, should be treated as an ABI change, and
>> + follow the relevant deprecation policy procedures as above: 3 acks and
>> + announcement at least one release in advance.
>> +
>> +Examples of Deprecation Notices
>> +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
>> +
>> +The following are some examples of ABI deprecation notices which would be
>> +added to the Release Notes:
>> +
>> +* The Macro ``#RTE_FOO`` is deprecated and will be removed with version 2.0,
>> + to be replaced with the inline function ``rte_foo()``.
>> +
>> +* The function ``rte_mbuf_grok()`` has been updated to include a new parameter
>> + in version 2.0. Backwards compatibility will be maintained for this function
>> + until the release of version 2.1
>> +
>> +* The members of ``struct rte_foo`` have been reorganized in release 2.0 for
>> + performance reasons. Existing binary applications will have backwards
>> + compatibility in release 2.0, while newly built binaries will need to
>> + reference the new structure variant ``struct rte_foo2``. Compatibility will
>> + be removed in release 2.2, and all applications will require updating and
>> + rebuilding to the new structure at that time, which will be renamed to the
>> + original ``struct rte_foo``.
>> +
>> +* Significant ABI changes are planned for the ``librte_dostuff`` library. The
>> + upcoming release 2.0 will not contain these changes, but release 2.1 will,
>> + and no backwards compatibility is planned due to the extensive nature of
>> + these changes. Binaries using this library built prior to version 2.1 will
>> + require updating and recompilation.
>> +
>> +New API replacing previous one
>> +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
>> +
>> +If a new API proposed functionally replaces an existing one, when the new API
>> +becomes non-experimental then the old one is marked with ``__rte_deprecated``.
>> +Deprecated APIs are removed completely just after the next LTS.
>> +
>> +Reminder that old API should follow deprecation process to be removed.
>> +
>> +
>> +Experimental APIs
>> +-----------------
>> +
>> +APIs marked as ``experimental`` are not considered part of the ABI and may
>> +change without warning at any time. Since changes to APIs are most likely
>> +immediately after their introduction, as users begin to take advantage of
>> +those new APIs and start finding issues with them, new DPDK APIs will be
>> +automatically marked as ``experimental`` to allow for a period of stabilization
>> +before they become part of a tracked ABI.
>> +
>> +Note that marking an API as experimental is a multi step process.
>> +To mark an API as experimental, the symbols which are desired to be exported
>> +must be placed in an EXPERIMENTAL version block in the corresponding libraries'
>> +version map script.
>> +Secondly, the corresponding prototypes of those exported functions (in the
>> +development header files), must be marked with the ``__rte_experimental`` tag
>> +(see ``rte_compat.h``).
>> +The DPDK build makefiles perform a check to ensure that the map file and the
>> +C code reflect the same list of symbols.
>> +This check can be circumvented by defining ``ALLOW_EXPERIMENTAL_API``
>> +during compilation in the corresponding library Makefile.
>> +
>> +In addition to tagging the code with ``__rte_experimental``,
>> +the doxygen markup must also contain the EXPERIMENTAL string,
>> +and the MAINTAINERS file should note the EXPERIMENTAL libraries.
>> +
>> +For removing the experimental tag associated with an API, deprecation notice
>> +is not required. Though, an API should remain in experimental state for at least
>> +one release. Thereafter, normal process of posting patch for review to mailing
>> +list can be followed.
>> diff --git a/doc/guides/contributing/abi_versioning.rst b/doc/guides/contributing/abi_versioning.rst
>> new file mode 100644
>> index 0000000..53e6ac0
>> --- /dev/null
>> +++ b/doc/guides/contributing/abi_versioning.rst
>> @@ -0,0 +1,427 @@
>> +.. SPDX-License-Identifier: BSD-3-Clause
>> + Copyright 2018 The DPDK contributors
>> +
>> +.. library_versioning:
>> +
>> +Library versioning
>> +------------------
>> +
>> +Downstreams might want to provide different DPDK releases at the same time to
>> +support multiple consumers of DPDK linked against older and newer sonames.
>> +
>> +Also due to the interdependencies that DPDK libraries can have applications
>> +might end up with an executable space in which multiple versions of a library
>> +are mapped by ld.so.
>> +
>> +Think of LibA that got an ABI bump and LibB that did not get an ABI bump but is
>> +depending on LibA.
>> +
>> +.. note::
>> +
>> + Application
>> + \-> LibA.old
>> + \-> LibB.new -> LibA.new
>> +
>> +That is a conflict which can be avoided by setting ``CONFIG_RTE_MAJOR_ABI``.
>> +If set, the value of ``CONFIG_RTE_MAJOR_ABI`` overwrites all - otherwise per
>> +library - versions defined in the libraries ``LIBABIVER``.
>> +An example might be ``CONFIG_RTE_MAJOR_ABI=16.11`` which will make all libraries
>> +``librte<?>.so.16.11`` instead of ``librte<?>.so.<LIBABIVER>``.
>> +
>> +
>> +ABI versioning
>> +--------------
>> +
>> +Versioning Macros
>> +~~~~~~~~~~~~~~~~~
>> +
>> +When a symbol is exported from a library to provide an API, it also provides a
>> +calling convention (ABI) that is embodied in its name, return type and
>> +arguments. Occasionally that function may need to change to accommodate new
>> +functionality or behavior. When that occurs, it is desirable to allow for
>> +backward compatibility for a time with older binaries that are dynamically
>> +linked to the DPDK.
>> +
>> +To support backward compatibility the ``rte_compat.h``
>> +header file provides macros to use when updating exported functions. These
>> +macros are used in conjunction with the ``rte_<library>_version.map`` file for
>> +a given library to allow multiple versions of a symbol to exist in a shared
>> +library so that older binaries need not be immediately recompiled.
>> +
>> +The macros exported are:
>> +
>> +* ``VERSION_SYMBOL(b, e, n)``: Creates a symbol version table entry binding
>> + versioned symbol ``b@DPDK_n`` to the internal function ``b_e``.
>> +
>> +* ``BIND_DEFAULT_SYMBOL(b, e, n)``: Creates a symbol version entry instructing
>> + the linker to bind references to symbol ``b`` to the internal symbol
>> + ``b_e``.
>> +
>> +* ``MAP_STATIC_SYMBOL(f, p)``: Declare the prototype ``f``, and map it to the
>> + fully qualified function ``p``, so that if a symbol becomes versioned, it
>> + can still be mapped back to the public symbol name.
>> +
>> +Examples of ABI Macro use
>> +^^^^^^^^^^^^^^^^^^^^^^^^^
>> +
>> +Updating a public API
>> +_____________________
>> +
>> +Assume we have a function as follows
>> +
>> +.. code-block:: c
>> +
>> + /*
>> + * Create an acl context object for apps to
>> + * manipulate
>> + */
>> + struct rte_acl_ctx *
>> + rte_acl_create(const struct rte_acl_param *param)
>> + {
>> + ...
>> + }
>> +
>> +
>> +Assume that struct rte_acl_ctx is a private structure, and that a developer
>> +wishes to enhance the acl api so that a debugging flag can be enabled on a
>> +per-context basis. This requires an addition to the structure (which, being
>> +private, is safe), but it also requires modifying the code as follows
>> +
>> +.. code-block:: c
>> +
>> + /*
>> + * Create an acl context object for apps to
>> + * manipulate
>> + */
>> + struct rte_acl_ctx *
>> + rte_acl_create(const struct rte_acl_param *param, int debug)
>> + {
>> + ...
>> + }
>> +
>> +
>> +Note also that, being a public function, the header file prototype must also be
>> +changed, as must all the call sites, to reflect the new ABI footprint. We will
>> +maintain previous ABI versions that are accessible only to previously compiled
>> +binaries
>> +
>> +The addition of a parameter to the function is ABI breaking as the function is
>> +public, and existing application may use it in its current form. However, the
>> +compatibility macros in DPDK allow a developer to use symbol versioning so that
>> +multiple functions can be mapped to the same public symbol based on when an
>> +application was linked to it. To see how this is done, we start with the
>> +requisite libraries version map file. Initially the version map file for the
>> +acl library looks like this
>> +
>> +.. code-block:: none
>> +
>> + DPDK_2.0 {
>> + global:
>> +
>> + rte_acl_add_rules;
>> + rte_acl_build;
>> + rte_acl_classify;
>> + rte_acl_classify_alg;
>> + rte_acl_classify_scalar;
>> + rte_acl_create;
>> + rte_acl_dump;
>> + rte_acl_find_existing;
>> + rte_acl_free;
>> + rte_acl_ipv4vlan_add_rules;
>> + rte_acl_ipv4vlan_build;
>> + rte_acl_list_dump;
>> + rte_acl_reset;
>> + rte_acl_reset_rules;
>> + rte_acl_set_ctx_classify;
>> +
>> + local: *;
>> + };
>> +
>> +This file needs to be modified as follows
>> +
>> +.. code-block:: none
>> +
>> + DPDK_2.0 {
>> + global:
>> +
>> + rte_acl_add_rules;
>> + rte_acl_build;
>> + rte_acl_classify;
>> + rte_acl_classify_alg;
>> + rte_acl_classify_scalar;
>> + rte_acl_create;
>> + rte_acl_dump;
>> + rte_acl_find_existing;
>> + rte_acl_free;
>> + rte_acl_ipv4vlan_add_rules;
>> + rte_acl_ipv4vlan_build;
>> + rte_acl_list_dump;
>> + rte_acl_reset;
>> + rte_acl_reset_rules;
>> + rte_acl_set_ctx_classify;
>> +
>> + local: *;
>> + };
>> +
>> + DPDK_2.1 {
>> + global:
>> + rte_acl_create;
>> +
>> + } DPDK_2.0;
>> +
>> +The addition of the new block tells the linker that a new version node is
>> +available (DPDK_2.1), which contains the symbol rte_acl_create, and inherits the
>> +symbols from the DPDK_2.0 node. This list is directly translated into a list of
>> +exported symbols when DPDK is compiled as a shared library
>> +
>> +Next, we need to specify in the code which function map to the rte_acl_create
>> +symbol at which versions. First, at the site of the initial symbol definition,
>> +we need to update the function so that it is uniquely named, and not in conflict
>> +with the public symbol name
>> +
>> +.. code-block:: c
>> +
>> + struct rte_acl_ctx *
>> + -rte_acl_create(const struct rte_acl_param *param)
>> + +rte_acl_create_v20(const struct rte_acl_param *param)
>> + {
>> + size_t sz;
>> + struct rte_acl_ctx *ctx;
>> + ...
>> +
>> +Note that the base name of the symbol was kept intact, as this is conducive to
>> +the macros used for versioning symbols. That is our next step, mapping this new
>> +symbol name to the initial symbol name at version node 2.0. Immediately after
>> +the function, we add this line of code
>> +
>> +.. code-block:: c
>> +
>> + VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
>> +
>> +Remembering to also add the rte_compat.h header to the requisite c file where
>> +these changes are being made. The above macro instructs the linker to create a
>> +new symbol ``rte_acl_create@DPDK_2.0``, which matches the symbol created in older
>> +builds, but now points to the above newly named function. We have now mapped
>> +the original rte_acl_create symbol to the original function (but with a new
>> +name)
>> +
>> +Next, we need to create the 2.1 version of the symbol. We create a new function
>> +name, with a different suffix, and implement it appropriately
>> +
>> +.. code-block:: c
>> +
>> + struct rte_acl_ctx *
>> + rte_acl_create_v21(const struct rte_acl_param *param, int debug);
>> + {
>> + struct rte_acl_ctx *ctx = rte_acl_create_v20(param);
>> +
>> + ctx->debug = debug;
>> +
>> + return ctx;
>> + }
>> +
>> +This code serves as our new API call. Its the same as our old call, but adds
>> +the new parameter in place. Next we need to map this function to the symbol
>> +``rte_acl_create@DPDK_2.1``. To do this, we modify the public prototype of the call
>> +in the header file, adding the macro there to inform all including applications,
>> +that on re-link, the default rte_acl_create symbol should point to this
>> +function. Note that we could do this by simply naming the function above
>> +rte_acl_create, and the linker would chose the most recent version tag to apply
>> +in the version script, but we can also do this in the header file
>> +
>> +.. code-block:: c
>> +
>> + struct rte_acl_ctx *
>> + -rte_acl_create(const struct rte_acl_param *param);
>> + +rte_acl_create(const struct rte_acl_param *param, int debug);
>> + +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
>> +
>> +The BIND_DEFAULT_SYMBOL macro explicitly tells applications that include this
>> +header, to link to the rte_acl_create_v21 function and apply the DPDK_2.1
>> +version node to it. This method is more explicit and flexible than just
>> +re-implementing the exact symbol name, and allows for other features (such as
>> +linking to the old symbol version by default, when the new ABI is to be opt-in
>> +for a period.
>> +
>> +One last thing we need to do. Note that we've taken what was a public symbol,
>> +and duplicated it into two uniquely and differently named symbols. We've then
>> +mapped each of those back to the public symbol ``rte_acl_create`` with different
>> +version tags. This only applies to dynamic linking, as static linking has no
>> +notion of versioning. That leaves this code in a position of no longer having a
>> +symbol simply named ``rte_acl_create`` and a static build will fail on that
>> +missing symbol.
>> +
>> +To correct this, we can simply map a function of our choosing back to the public
>> +symbol in the static build with the ``MAP_STATIC_SYMBOL`` macro. Generally the
>> +assumption is that the most recent version of the symbol is the one you want to
>> +map. So, back in the C file where, immediately after ``rte_acl_create_v21`` is
>> +defined, we add this
>> +
>> +.. code-block:: c
>> +
>> + struct rte_acl_ctx *
>> + rte_acl_create_v21(const struct rte_acl_param *param, int debug)
>> + {
>> + ...
>> + }
>> + MAP_STATIC_SYMBOL(struct rte_acl_ctx *rte_acl_create(const struct rte_acl_param *param, int debug), rte_acl_create_v21);
>> +
>> +That tells the compiler that, when building a static library, any calls to the
>> +symbol ``rte_acl_create`` should be linked to ``rte_acl_create_v21``
>> +
>> +That's it, on the next shared library rebuild, there will be two versions of
>> +rte_acl_create, an old DPDK_2.0 version, used by previously built applications,
>> +and a new DPDK_2.1 version, used by future built applications.
>> +
>> +
>> +Deprecating part of a public API
>> +________________________________
>> +
>> +Lets assume that you've done the above update, and after a few releases have
>> +passed you decide you would like to retire the old version of the function.
>> +After having gone through the ABI deprecation announcement process, removal is
>> +easy. Start by removing the symbol from the requisite version map file:
>> +
>> +.. code-block:: none
>> +
>> + DPDK_2.0 {
>> + global:
>> +
>> + rte_acl_add_rules;
>> + rte_acl_build;
>> + rte_acl_classify;
>> + rte_acl_classify_alg;
>> + rte_acl_classify_scalar;
>> + rte_acl_dump;
>> + - rte_acl_create
>> + rte_acl_find_existing;
>> + rte_acl_free;
>> + rte_acl_ipv4vlan_add_rules;
>> + rte_acl_ipv4vlan_build;
>> + rte_acl_list_dump;
>> + rte_acl_reset;
>> + rte_acl_reset_rules;
>> + rte_acl_set_ctx_classify;
>> +
>> + local: *;
>> + };
>> +
>> + DPDK_2.1 {
>> + global:
>> + rte_acl_create;
>> + } DPDK_2.0;
>> +
>> +
>> +Next remove the corresponding versioned export.
>> +
>> +.. code-block:: c
>> +
>> + -VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
>> +
>> +
>> +Note that the internal function definition could also be removed, but its used
>> +in our example by the newer version _v21, so we leave it in place. This is a
>> +coding style choice.
>> +
>> +Lastly, we need to bump the LIBABIVER number for this library in the Makefile to
>> +indicate to applications doing dynamic linking that this is a later, and
>> +possibly incompatible library version:
>> +
>> +.. code-block:: c
>> +
>> + -LIBABIVER := 1
>> + +LIBABIVER := 2
>> +
>> +Deprecating an entire ABI version
>> +_________________________________
>> +
>> +While removing a symbol from and ABI may be useful, it is often more practical
>> +to remove an entire version node at once. If a version node completely
>> +specifies an API, then removing part of it, typically makes it incomplete. In
>> +those cases it is better to remove the entire node
>> +
>> +To do this, start by modifying the version map file, such that all symbols from
>> +the node to be removed are merged into the next node in the map
>> +
>> +In the case of our map above, it would transform to look as follows
>> +
>> +.. code-block:: none
>> +
>> + DPDK_2.1 {
>> + global:
>> +
>> + rte_acl_add_rules;
>> + rte_acl_build;
>> + rte_acl_classify;
>> + rte_acl_classify_alg;
>> + rte_acl_classify_scalar;
>> + rte_acl_dump;
>> + rte_acl_create
>> + rte_acl_find_existing;
>> + rte_acl_free;
>> + rte_acl_ipv4vlan_add_rules;
>> + rte_acl_ipv4vlan_build;
>> + rte_acl_list_dump;
>> + rte_acl_reset;
>> + rte_acl_reset_rules;
>> + rte_acl_set_ctx_classify;
>> +
>> + local: *;
>> + };
>> +
>> +Then any uses of BIND_DEFAULT_SYMBOL that pointed to the old node should be
>> +updated to point to the new version node in any header files for all affected
>> +symbols.
>> +
>> +.. code-block:: c
>> +
>> + -BIND_DEFAULT_SYMBOL(rte_acl_create, _v20, 2.0);
>> + +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
>> +
>> +Lastly, any VERSION_SYMBOL macros that point to the old version node should be
>> +removed, taking care to keep, where need old code in place to support newer
>> +versions of the symbol.
>> +
>> +
>> +Running the ABI Validator
>> +-------------------------
>> +
>> +The ``devtools`` directory in the DPDK source tree contains a utility program,
>> +``validate-abi.sh``, for validating the DPDK ABI based on the Linux `ABI
>> +Compliance Checker
>> +<http://ispras.linuxbase.org/index.php/ABI_compliance_checker>`_.
>> +
>> +This has a dependency on the ``abi-compliance-checker`` and ``and abi-dumper``
>> +utilities which can be installed via a package manager. For example::
>> +
>> + sudo yum install abi-compliance-checker
>> + sudo yum install abi-dumper
>> +
>> +The syntax of the ``validate-abi.sh`` utility is::
>> +
>> + ./devtools/validate-abi.sh <REV1> <REV2>
>> +
>> +Where ``REV1`` and ``REV2`` are valid gitrevisions(7)
>> +https://www.kernel.org/pub/software/scm/git/docs/gitrevisions.html
>> +on the local repo.
>> +
>> +For example::
>> +
>> + # Check between the previous and latest commit:
>> + ./devtools/validate-abi.sh HEAD~1 HEAD
>> +
>> + # Check on a specific compilation target:
>> + ./devtools/validate-abi.sh -t x86_64-native-linux-gcc HEAD~1 HEAD
>> +
>> + # Check between two tags:
>> + ./devtools/validate-abi.sh v2.0.0 v2.1.0
>> +
>> + # Check between git master and local topic-branch "vhost-hacking":
>> + ./devtools/validate-abi.sh master vhost-hacking
>> +
>> +After the validation script completes (it can take a while since it need to
>> +compile both tags) it will create compatibility reports in the
>> +``./abi-check/compat_report`` directory. Listed incompatibilities can be found
>> +as follows::
>> +
>> + grep -lr Incompatible abi-check/compat_reports/
>> diff --git a/doc/guides/contributing/index.rst b/doc/guides/contributing/index.rst
>> index e2608d3..2fefd91 100644
>> --- a/doc/guides/contributing/index.rst
>> +++ b/doc/guides/contributing/index.rst
>> @@ -10,7 +10,8 @@ Contributor's Guidelines
>>
>> coding_style
>> design
>> - versioning
>> + abi_policy
>> + abi_versioning
>> documentation
>> patches
>> vulnerability
>> diff --git a/doc/guides/contributing/versioning.rst b/doc/guides/contributing/versioning.rst
>> deleted file mode 100644
>> index 3ab2c43..0000000
>> --- a/doc/guides/contributing/versioning.rst
>> +++ /dev/null
>> @@ -1,591 +0,0 @@
>> -.. SPDX-License-Identifier: BSD-3-Clause
>> - Copyright 2018 The DPDK contributors
>> -
>> -DPDK ABI/API policy
>> -===================
>> -
>> -Description
>> ------------
>> -
>> -This document details some methods for handling ABI management in the DPDK.
>> -
>> -General Guidelines
>> -------------------
>> -
>> -#. Whenever possible, ABI should be preserved
>> -#. ABI/API may be changed with a deprecation process
>> -#. The modification of symbols can generally be managed with versioning
>> -#. Libraries or APIs marked in ``experimental`` state may change without constraint
>> -#. New APIs will be marked as ``experimental`` for at least one release to allow
>> - any issues found by users of the new API to be fixed quickly
>> -#. The addition of symbols is generally not problematic
>> -#. The removal of symbols generally is an ABI break and requires bumping of the
>> - LIBABIVER macro
>> -#. Updates to the minimum hardware requirements, which drop support for hardware which
>> - was previously supported, should be treated as an ABI change.
>> -
>> -What is an ABI
>> -~~~~~~~~~~~~~~
>> -
>> -An ABI (Application Binary Interface) is the set of runtime interfaces exposed
>> -by a library. It is similar to an API (Application Programming Interface) but
>> -is the result of compilation. It is also effectively cloned when applications
>> -link to dynamic libraries. That is to say when an application is compiled to
>> -link against dynamic libraries, it is assumed that the ABI remains constant
>> -between the time the application is compiled/linked, and the time that it runs.
>> -Therefore, in the case of dynamic linking, it is critical that an ABI is
>> -preserved, or (when modified), done in such a way that the application is unable
>> -to behave improperly or in an unexpected fashion.
>> -
>> -
>> -ABI/API Deprecation
>> --------------------
>> -
>> -The DPDK ABI policy
>> -~~~~~~~~~~~~~~~~~~~
>> -
>> -ABI versions are set at the time of major release labeling, and the ABI may
>> -change multiple times, without warning, between the last release label and the
>> -HEAD label of the git tree.
>> -
>> -ABI versions, once released, are available until such time as their
>> -deprecation has been noted in the Release Notes for at least one major release
>> -cycle. For example consider the case where the ABI for DPDK 2.0 has been
>> -shipped and then a decision is made to modify it during the development of
>> -DPDK 2.1. The decision will be recorded in the Release Notes for the DPDK 2.1
>> -release and the modification will be made available in the DPDK 2.2 release.
>> -
>> -ABI versions may be deprecated in whole or in part as needed by a given
>> -update.
>> -
>> -Some ABI changes may be too significant to reasonably maintain multiple
>> -versions. In those cases ABI's may be updated without backward compatibility
>> -being provided. The requirements for doing so are:
>> -
>> -#. At least 3 acknowledgments of the need to do so must be made on the
>> - dpdk.org mailing list.
>> -
>> - - The acknowledgment of the maintainer of the component is mandatory, or if
>> - no maintainer is available for the component, the tree/sub-tree maintainer
>> - for that component must acknowledge the ABI change instead.
>> -
>> - - It is also recommended that acknowledgments from different "areas of
>> - interest" be sought for each deprecation, for example: from NIC vendors,
>> - CPU vendors, end-users, etc.
>> -
>> -#. The changes (including an alternative map file) can be included with
>> - deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option,
>> - to provide more details about oncoming changes.
>> - ``RTE_NEXT_ABI`` wrapper will be removed when it become the default ABI.
>> - More preferred way to provide this information is sending the feature
>> - as a separate patch and reference it in deprecation notice.
>> -
>> -#. A full deprecation cycle, as explained above, must be made to offer
>> - downstream consumers sufficient warning of the change.
>> -
>> -Note that the above process for ABI deprecation should not be undertaken
>> -lightly. ABI stability is extremely important for downstream consumers of the
>> -DPDK, especially when distributed in shared object form. Every effort should
>> -be made to preserve the ABI whenever possible. The ABI should only be changed
>> -for significant reasons, such as performance enhancements. ABI breakage due to
>> -changes such as reorganizing public structure fields for aesthetic or
>> -readability purposes should be avoided.
>> -
>> -.. note::
>> -
>> - Updates to the minimum hardware requirements, which drop support for hardware
>> - which was previously supported, should be treated as an ABI change, and
>> - follow the relevant deprecation policy procedures as above: 3 acks and
>> - announcement at least one release in advance.
>> -
>> -Examples of Deprecation Notices
>> -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
>> -
>> -The following are some examples of ABI deprecation notices which would be
>> -added to the Release Notes:
>> -
>> -* The Macro ``#RTE_FOO`` is deprecated and will be removed with version 2.0,
>> - to be replaced with the inline function ``rte_foo()``.
>> -
>> -* The function ``rte_mbuf_grok()`` has been updated to include a new parameter
>> - in version 2.0. Backwards compatibility will be maintained for this function
>> - until the release of version 2.1
>> -
>> -* The members of ``struct rte_foo`` have been reorganized in release 2.0 for
>> - performance reasons. Existing binary applications will have backwards
>> - compatibility in release 2.0, while newly built binaries will need to
>> - reference the new structure variant ``struct rte_foo2``. Compatibility will
>> - be removed in release 2.2, and all applications will require updating and
>> - rebuilding to the new structure at that time, which will be renamed to the
>> - original ``struct rte_foo``.
>> -
>> -* Significant ABI changes are planned for the ``librte_dostuff`` library. The
>> - upcoming release 2.0 will not contain these changes, but release 2.1 will,
>> - and no backwards compatibility is planned due to the extensive nature of
>> - these changes. Binaries using this library built prior to version 2.1 will
>> - require updating and recompilation.
>> -
>> -New API replacing previous one
>> -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
>> -
>> -If a new API proposed functionally replaces an existing one, when the new API
>> -becomes non-experimental then the old one is marked with ``__rte_deprecated``.
>> -Deprecated APIs are removed completely just after the next LTS.
>> -
>> -Reminder that old API should follow deprecation process to be removed.
>> -
>> -
>> -Experimental APIs
>> ------------------
>> -
>> -APIs marked as ``experimental`` are not considered part of the ABI and may
>> -change without warning at any time. Since changes to APIs are most likely
>> -immediately after their introduction, as users begin to take advantage of
>> -those new APIs and start finding issues with them, new DPDK APIs will be
>> -automatically marked as ``experimental`` to allow for a period of stabilization
>> -before they become part of a tracked ABI.
>> -
>> -Note that marking an API as experimental is a multi step process.
>> -To mark an API as experimental, the symbols which are desired to be exported
>> -must be placed in an EXPERIMENTAL version block in the corresponding libraries'
>> -version map script.
>> -Secondly, the corresponding prototypes of those exported functions (in the
>> -development header files), must be marked with the ``__rte_experimental`` tag
>> -(see ``rte_compat.h``).
>> -The DPDK build makefiles perform a check to ensure that the map file and the
>> -C code reflect the same list of symbols.
>> -This check can be circumvented by defining ``ALLOW_EXPERIMENTAL_API``
>> -during compilation in the corresponding library Makefile.
>> -
>> -In addition to tagging the code with ``__rte_experimental``,
>> -the doxygen markup must also contain the EXPERIMENTAL string,
>> -and the MAINTAINERS file should note the EXPERIMENTAL libraries.
>> -
>> -For removing the experimental tag associated with an API, deprecation notice
>> -is not required. Though, an API should remain in experimental state for at least
>> -one release. Thereafter, normal process of posting patch for review to mailing
>> -list can be followed.
>> -
>> -
>> -Library versioning
>> -------------------
>> -
>> -Downstreams might want to provide different DPDK releases at the same time to
>> -support multiple consumers of DPDK linked against older and newer sonames.
>> -
>> -Also due to the interdependencies that DPDK libraries can have applications
>> -might end up with an executable space in which multiple versions of a library
>> -are mapped by ld.so.
>> -
>> -Think of LibA that got an ABI bump and LibB that did not get an ABI bump but is
>> -depending on LibA.
>> -
>> -.. note::
>> -
>> - Application
>> - \-> LibA.old
>> - \-> LibB.new -> LibA.new
>> -
>> -That is a conflict which can be avoided by setting ``CONFIG_RTE_MAJOR_ABI``.
>> -If set, the value of ``CONFIG_RTE_MAJOR_ABI`` overwrites all - otherwise per
>> -library - versions defined in the libraries ``LIBABIVER``.
>> -An example might be ``CONFIG_RTE_MAJOR_ABI=16.11`` which will make all libraries
>> -``librte<?>.so.16.11`` instead of ``librte<?>.so.<LIBABIVER>``.
>> -
>> -
>> -ABI versioning
>> ---------------
>> -
>> -Versioning Macros
>> -~~~~~~~~~~~~~~~~~
>> -
>> -When a symbol is exported from a library to provide an API, it also provides a
>> -calling convention (ABI) that is embodied in its name, return type and
>> -arguments. Occasionally that function may need to change to accommodate new
>> -functionality or behavior. When that occurs, it is desirable to allow for
>> -backward compatibility for a time with older binaries that are dynamically
>> -linked to the DPDK.
>> -
>> -To support backward compatibility the ``rte_compat.h``
>> -header file provides macros to use when updating exported functions. These
>> -macros are used in conjunction with the ``rte_<library>_version.map`` file for
>> -a given library to allow multiple versions of a symbol to exist in a shared
>> -library so that older binaries need not be immediately recompiled.
>> -
>> -The macros exported are:
>> -
>> -* ``VERSION_SYMBOL(b, e, n)``: Creates a symbol version table entry binding
>> - versioned symbol ``b@DPDK_n`` to the internal function ``b_e``.
>> -
>> -* ``BIND_DEFAULT_SYMBOL(b, e, n)``: Creates a symbol version entry instructing
>> - the linker to bind references to symbol ``b`` to the internal symbol
>> - ``b_e``.
>> -
>> -* ``MAP_STATIC_SYMBOL(f, p)``: Declare the prototype ``f``, and map it to the
>> - fully qualified function ``p``, so that if a symbol becomes versioned, it
>> - can still be mapped back to the public symbol name.
>> -
>> -Examples of ABI Macro use
>> -^^^^^^^^^^^^^^^^^^^^^^^^^
>> -
>> -Updating a public API
>> -_____________________
>> -
>> -Assume we have a function as follows
>> -
>> -.. code-block:: c
>> -
>> - /*
>> - * Create an acl context object for apps to
>> - * manipulate
>> - */
>> - struct rte_acl_ctx *
>> - rte_acl_create(const struct rte_acl_param *param)
>> - {
>> - ...
>> - }
>> -
>> -
>> -Assume that struct rte_acl_ctx is a private structure, and that a developer
>> -wishes to enhance the acl api so that a debugging flag can be enabled on a
>> -per-context basis. This requires an addition to the structure (which, being
>> -private, is safe), but it also requires modifying the code as follows
>> -
>> -.. code-block:: c
>> -
>> - /*
>> - * Create an acl context object for apps to
>> - * manipulate
>> - */
>> - struct rte_acl_ctx *
>> - rte_acl_create(const struct rte_acl_param *param, int debug)
>> - {
>> - ...
>> - }
>> -
>> -
>> -Note also that, being a public function, the header file prototype must also be
>> -changed, as must all the call sites, to reflect the new ABI footprint. We will
>> -maintain previous ABI versions that are accessible only to previously compiled
>> -binaries
>> -
>> -The addition of a parameter to the function is ABI breaking as the function is
>> -public, and existing application may use it in its current form. However, the
>> -compatibility macros in DPDK allow a developer to use symbol versioning so that
>> -multiple functions can be mapped to the same public symbol based on when an
>> -application was linked to it. To see how this is done, we start with the
>> -requisite libraries version map file. Initially the version map file for the
>> -acl library looks like this
>> -
>> -.. code-block:: none
>> -
>> - DPDK_2.0 {
>> - global:
>> -
>> - rte_acl_add_rules;
>> - rte_acl_build;
>> - rte_acl_classify;
>> - rte_acl_classify_alg;
>> - rte_acl_classify_scalar;
>> - rte_acl_create;
>> - rte_acl_dump;
>> - rte_acl_find_existing;
>> - rte_acl_free;
>> - rte_acl_ipv4vlan_add_rules;
>> - rte_acl_ipv4vlan_build;
>> - rte_acl_list_dump;
>> - rte_acl_reset;
>> - rte_acl_reset_rules;
>> - rte_acl_set_ctx_classify;
>> -
>> - local: *;
>> - };
>> -
>> -This file needs to be modified as follows
>> -
>> -.. code-block:: none
>> -
>> - DPDK_2.0 {
>> - global:
>> -
>> - rte_acl_add_rules;
>> - rte_acl_build;
>> - rte_acl_classify;
>> - rte_acl_classify_alg;
>> - rte_acl_classify_scalar;
>> - rte_acl_create;
>> - rte_acl_dump;
>> - rte_acl_find_existing;
>> - rte_acl_free;
>> - rte_acl_ipv4vlan_add_rules;
>> - rte_acl_ipv4vlan_build;
>> - rte_acl_list_dump;
>> - rte_acl_reset;
>> - rte_acl_reset_rules;
>> - rte_acl_set_ctx_classify;
>> -
>> - local: *;
>> - };
>> -
>> - DPDK_2.1 {
>> - global:
>> - rte_acl_create;
>> -
>> - } DPDK_2.0;
>> -
>> -The addition of the new block tells the linker that a new version node is
>> -available (DPDK_2.1), which contains the symbol rte_acl_create, and inherits the
>> -symbols from the DPDK_2.0 node. This list is directly translated into a list of
>> -exported symbols when DPDK is compiled as a shared library
>> -
>> -Next, we need to specify in the code which function map to the rte_acl_create
>> -symbol at which versions. First, at the site of the initial symbol definition,
>> -we need to update the function so that it is uniquely named, and not in conflict
>> -with the public symbol name
>> -
>> -.. code-block:: c
>> -
>> - struct rte_acl_ctx *
>> - -rte_acl_create(const struct rte_acl_param *param)
>> - +rte_acl_create_v20(const struct rte_acl_param *param)
>> - {
>> - size_t sz;
>> - struct rte_acl_ctx *ctx;
>> - ...
>> -
>> -Note that the base name of the symbol was kept intact, as this is conducive to
>> -the macros used for versioning symbols. That is our next step, mapping this new
>> -symbol name to the initial symbol name at version node 2.0. Immediately after
>> -the function, we add this line of code
>> -
>> -.. code-block:: c
>> -
>> - VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
>> -
>> -Remembering to also add the rte_compat.h header to the requisite c file where
>> -these changes are being made. The above macro instructs the linker to create a
>> -new symbol ``rte_acl_create@DPDK_2.0``, which matches the symbol created in older
>> -builds, but now points to the above newly named function. We have now mapped
>> -the original rte_acl_create symbol to the original function (but with a new
>> -name)
>> -
>> -Next, we need to create the 2.1 version of the symbol. We create a new function
>> -name, with a different suffix, and implement it appropriately
>> -
>> -.. code-block:: c
>> -
>> - struct rte_acl_ctx *
>> - rte_acl_create_v21(const struct rte_acl_param *param, int debug);
>> - {
>> - struct rte_acl_ctx *ctx = rte_acl_create_v20(param);
>> -
>> - ctx->debug = debug;
>> -
>> - return ctx;
>> - }
>> -
>> -This code serves as our new API call. Its the same as our old call, but adds
>> -the new parameter in place. Next we need to map this function to the symbol
>> -``rte_acl_create@DPDK_2.1``. To do this, we modify the public prototype of the call
>> -in the header file, adding the macro there to inform all including applications,
>> -that on re-link, the default rte_acl_create symbol should point to this
>> -function. Note that we could do this by simply naming the function above
>> -rte_acl_create, and the linker would chose the most recent version tag to apply
>> -in the version script, but we can also do this in the header file
>> -
>> -.. code-block:: c
>> -
>> - struct rte_acl_ctx *
>> - -rte_acl_create(const struct rte_acl_param *param);
>> - +rte_acl_create(const struct rte_acl_param *param, int debug);
>> - +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
>> -
>> -The BIND_DEFAULT_SYMBOL macro explicitly tells applications that include this
>> -header, to link to the rte_acl_create_v21 function and apply the DPDK_2.1
>> -version node to it. This method is more explicit and flexible than just
>> -re-implementing the exact symbol name, and allows for other features (such as
>> -linking to the old symbol version by default, when the new ABI is to be opt-in
>> -for a period.
>> -
>> -One last thing we need to do. Note that we've taken what was a public symbol,
>> -and duplicated it into two uniquely and differently named symbols. We've then
>> -mapped each of those back to the public symbol ``rte_acl_create`` with different
>> -version tags. This only applies to dynamic linking, as static linking has no
>> -notion of versioning. That leaves this code in a position of no longer having a
>> -symbol simply named ``rte_acl_create`` and a static build will fail on that
>> -missing symbol.
>> -
>> -To correct this, we can simply map a function of our choosing back to the public
>> -symbol in the static build with the ``MAP_STATIC_SYMBOL`` macro. Generally the
>> -assumption is that the most recent version of the symbol is the one you want to
>> -map. So, back in the C file where, immediately after ``rte_acl_create_v21`` is
>> -defined, we add this
>> -
>> -.. code-block:: c
>> -
>> - struct rte_acl_ctx *
>> - rte_acl_create_v21(const struct rte_acl_param *param, int debug)
>> - {
>> - ...
>> - }
>> - MAP_STATIC_SYMBOL(struct rte_acl_ctx *rte_acl_create(const struct rte_acl_param *param, int debug), rte_acl_create_v21);
>> -
>> -That tells the compiler that, when building a static library, any calls to the
>> -symbol ``rte_acl_create`` should be linked to ``rte_acl_create_v21``
>> -
>> -That's it, on the next shared library rebuild, there will be two versions of
>> -rte_acl_create, an old DPDK_2.0 version, used by previously built applications,
>> -and a new DPDK_2.1 version, used by future built applications.
>> -
>> -
>> -Deprecating part of a public API
>> -________________________________
>> -
>> -Lets assume that you've done the above update, and after a few releases have
>> -passed you decide you would like to retire the old version of the function.
>> -After having gone through the ABI deprecation announcement process, removal is
>> -easy. Start by removing the symbol from the requisite version map file:
>> -
>> -.. code-block:: none
>> -
>> - DPDK_2.0 {
>> - global:
>> -
>> - rte_acl_add_rules;
>> - rte_acl_build;
>> - rte_acl_classify;
>> - rte_acl_classify_alg;
>> - rte_acl_classify_scalar;
>> - rte_acl_dump;
>> - - rte_acl_create
>> - rte_acl_find_existing;
>> - rte_acl_free;
>> - rte_acl_ipv4vlan_add_rules;
>> - rte_acl_ipv4vlan_build;
>> - rte_acl_list_dump;
>> - rte_acl_reset;
>> - rte_acl_reset_rules;
>> - rte_acl_set_ctx_classify;
>> -
>> - local: *;
>> - };
>> -
>> - DPDK_2.1 {
>> - global:
>> - rte_acl_create;
>> - } DPDK_2.0;
>> -
>> -
>> -Next remove the corresponding versioned export.
>> -
>> -.. code-block:: c
>> -
>> - -VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
>> -
>> -
>> -Note that the internal function definition could also be removed, but its used
>> -in our example by the newer version _v21, so we leave it in place. This is a
>> -coding style choice.
>> -
>> -Lastly, we need to bump the LIBABIVER number for this library in the Makefile to
>> -indicate to applications doing dynamic linking that this is a later, and
>> -possibly incompatible library version:
>> -
>> -.. code-block:: c
>> -
>> - -LIBABIVER := 1
>> - +LIBABIVER := 2
>> -
>> -Deprecating an entire ABI version
>> -_________________________________
>> -
>> -While removing a symbol from and ABI may be useful, it is often more practical
>> -to remove an entire version node at once. If a version node completely
>> -specifies an API, then removing part of it, typically makes it incomplete. In
>> -those cases it is better to remove the entire node
>> -
>> -To do this, start by modifying the version map file, such that all symbols from
>> -the node to be removed are merged into the next node in the map
>> -
>> -In the case of our map above, it would transform to look as follows
>> -
>> -.. code-block:: none
>> -
>> - DPDK_2.1 {
>> - global:
>> -
>> - rte_acl_add_rules;
>> - rte_acl_build;
>> - rte_acl_classify;
>> - rte_acl_classify_alg;
>> - rte_acl_classify_scalar;
>> - rte_acl_dump;
>> - rte_acl_create
>> - rte_acl_find_existing;
>> - rte_acl_free;
>> - rte_acl_ipv4vlan_add_rules;
>> - rte_acl_ipv4vlan_build;
>> - rte_acl_list_dump;
>> - rte_acl_reset;
>> - rte_acl_reset_rules;
>> - rte_acl_set_ctx_classify;
>> -
>> - local: *;
>> - };
>> -
>> -Then any uses of BIND_DEFAULT_SYMBOL that pointed to the old node should be
>> -updated to point to the new version node in any header files for all affected
>> -symbols.
>> -
>> -.. code-block:: c
>> -
>> - -BIND_DEFAULT_SYMBOL(rte_acl_create, _v20, 2.0);
>> - +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
>> -
>> -Lastly, any VERSION_SYMBOL macros that point to the old version node should be
>> -removed, taking care to keep, where need old code in place to support newer
>> -versions of the symbol.
>> -
>> -
>> -Running the ABI Validator
>> --------------------------
>> -
>> -The ``devtools`` directory in the DPDK source tree contains a utility program,
>> -``validate-abi.sh``, for validating the DPDK ABI based on the Linux `ABI
>> -Compliance Checker
>> -<http://ispras.linuxbase.org/index.php/ABI_compliance_checker>`_.
>> -
>> -This has a dependency on the ``abi-compliance-checker`` and ``and abi-dumper``
>> -utilities which can be installed via a package manager. For example::
>> -
>> - sudo yum install abi-compliance-checker
>> - sudo yum install abi-dumper
>> -
>> -The syntax of the ``validate-abi.sh`` utility is::
>> -
>> - ./devtools/validate-abi.sh <REV1> <REV2>
>> -
>> -Where ``REV1`` and ``REV2`` are valid gitrevisions(7)
>> -https://www.kernel.org/pub/software/scm/git/docs/gitrevisions.html
>> -on the local repo.
>> -
>> -For example::
>> -
>> - # Check between the previous and latest commit:
>> - ./devtools/validate-abi.sh HEAD~1 HEAD
>> -
>> - # Check on a specific compilation target:
>> - ./devtools/validate-abi.sh -t x86_64-native-linux-gcc HEAD~1 HEAD
>> -
>> - # Check between two tags:
>> - ./devtools/validate-abi.sh v2.0.0 v2.1.0
>> -
>> - # Check between git master and local topic-branch "vhost-hacking":
>> - ./devtools/validate-abi.sh master vhost-hacking
>> -
>> -After the validation script completes (it can take a while since it need to
>> -compile both tags) it will create compatibility reports in the
>> -``./abi-check/compat_report`` directory. Listed incompatibilities can be found
>> -as follows::
>> -
>> - grep -lr Incompatible abi-check/compat_reports/
>> --
>> 2.7.4
>>
>>
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] [PATCH v3 2/4] doc: changes to abi policy introducing major abi versions
2019-09-24 11:32 10% ` Ray Kinsella
@ 2019-09-25 12:29 5% ` Kevin Traynor
0 siblings, 0 replies; 200+ results
From: Kevin Traynor @ 2019-09-25 12:29 UTC (permalink / raw)
To: Ray Kinsella, dev
Cc: thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, nhorman,
maxime.coquelin, john.mcnamara, marko.kovacevic, hemant.agrawal,
Luca Boccassi, David Marchand
On 24/09/2019 12:32, Ray Kinsella wrote:
>
> Thanks Kevin for working through all this.
> Other comments are inline.
>
> On 30/08/2019 17:20, Kevin Traynor wrote:
>> Hi Ray,
>>
>> On 15/08/2019 11:23, Ray Kinsella wrote:
>>> This policy change introduces major ABI versions, these are
>>> declared every year, typically aligned with the LTS release
>>> and are supported by subsequent releases in the following year.
>>> This change is intended to improve ABI stabilty for those projects
>>> consuming DPDK.
>>>
>>> Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
>>> ---
>>> doc/guides/contributing/abi_policy.rst | 308 ++++++++++++++++++++++++---------
>>> doc/guides/contributing/stable.rst | 38 ++--
>>> 2 files changed, 245 insertions(+), 101 deletions(-)
>>>
>>> diff --git a/doc/guides/contributing/abi_policy.rst b/doc/guides/contributing/abi_policy.rst
>>> index 55bacb4..6190bdc 100644
>>> --- a/doc/guides/contributing/abi_policy.rst
>>> +++ b/doc/guides/contributing/abi_policy.rst
>>> @@ -1,33 +1,46 @@
>>> .. SPDX-License-Identifier: BSD-3-Clause
>>> - Copyright 2018 The DPDK contributors
>>> + Copyright 2019 The DPDK contributors
>>>
>>> -.. abi_api_policy:
>>> +.. _abi_policy:
>>>
>>> -DPDK ABI/API policy
>>> -===================
>>> +ABI Policy
>>> +==========
>>>
>>> Description
>>> -----------
>>>
>>> -This document details some methods for handling ABI management in the DPDK.
>>> +This document details the management policy that ensures the long-term stability
>>> +of the DPDK ABI and API.
>>>
>>> General Guidelines
>>> ------------------
>>>
>>> -#. Whenever possible, ABI should be preserved
>>> -#. ABI/API may be changed with a deprecation process
>>> -#. The modification of symbols can generally be managed with versioning
>>> -#. Libraries or APIs marked in ``experimental`` state may change without constraint
>>> -#. New APIs will be marked as ``experimental`` for at least one release to allow
>>> - any issues found by users of the new API to be fixed quickly
>>> -#. The addition of symbols is generally not problematic
>>> -#. The removal of symbols generally is an ABI break and requires bumping of the
>>> - LIBABIVER macro
>>> -#. Updates to the minimum hardware requirements, which drop support for hardware which
>>> - was previously supported, should be treated as an ABI change.
>>> -
>>> -What is an ABI
>>> -~~~~~~~~~~~~~~
>>> +#. Major ABI versions are declared every **year** and are then supported for one
>>> + year, typically aligned with the :ref:`LTS release <stable_lts_releases>`.
>>> +#. The ABI version is managed at a project level in DPDK, with the ABI version
>>> + reflected in all :ref:`library's soname <what_is_soname>`.
>>> +#. The ABI should be preserved and not changed lightly. ABI changes must follow
>>> + the outlined :ref:`deprecation process <abi_changes>`.
>>> +#. The addition of symbols is generally not problematic. The modification of
>>> + symbols is managed with :ref:`ABI Versioning <abi_versioning>`.
>>> +#. The removal of symbols is considered an :ref:`ABI breakage <abi_breakages>`,
>>> + once approved these will form part of the next ABI version.
>>> +#. Libraries or APIs marked as :ref:`Experimental <experimental_apis>` are not
>>> + considered part of an ABI version and may change without constraint.
>>> +#. Updates to the :ref:`minimum hardware requirements <hw_rqmts>`, which drop
>>> + support for hardware which was previously supported, should be treated as an
>>> + ABI change.
>>> +
>>> +.. note::
>>> +
>>> + In 2019, the DPDK community stated it's intention to move to ABI stable
>>> + releases, over a number of release cycles. Beginning with maintaining ABI
>>> + stability through one year of DPDK releases starting from DPDK 19.11. This
>>> + policy will be reviewed in 2020, with intention of lengthening the stability
>>> + period.
>>> +
>>> +What is an ABI?
>>> +~~~~~~~~~~~~~~~
>>>
>>> An ABI (Application Binary Interface) is the set of runtime interfaces exposed
>>> by a library. It is similar to an API (Application Programming Interface) but
>>> @@ -39,30 +52,67 @@ Therefore, in the case of dynamic linking, it is critical that an ABI is
>>> preserved, or (when modified), done in such a way that the application is unable
>>> to behave improperly or in an unexpected fashion.
>>>
>>> +What is an ABI version?
>>> +~~~~~~~~~~~~~~~~~~~~~~~
>>>
>>> -ABI/API Deprecation
>>> --------------------
>>> +An ABI version is an instance of a library's ABI at a specific release. Certain
>>> +releases are considered by the community to be milestone releases, the yearly
>>> +LTS for example. Supporting those milestone release's ABI for some number of
>>> +subsequent releases is desirable to facilitate application upgrade. Those ABI
>>> +version's aligned with milestones release are therefore called 'ABI major
>>> +versions' and are supported for some number of releases.
>>> +
>>> +More details on major ABI version can be found in the :ref:`ABI versioning
>>> +<major_abi_versions>` guide.
>>>
>>> The DPDK ABI policy
>>> -~~~~~~~~~~~~~~~~~~~
>>> +-------------------
>>> +
>>> +A major ABI version is declared every year, aligned with that year's LTS
>>> +release, e.g. v19.11. This ABI version is then supported for one year by all
>>> +subsequent releases within that time period, until the next LTS release, e.g.
>>> +v20.11.
>>> +
>>> +At the declaration of a major ABI version, major version numbers encoded in
>>> +libraries soname's are bumped to indicate the new version, with the minor
>>> +version reset to ``0``. An example would be ``librte_eal.so.20.3`` would become
>>> +``librte_eal.so.21.0``.
>>> +
>>> +The ABI may then change multiple times, without warning, between the last major
>>> +ABI version increment and the HEAD label of the git tree, with the condition
>>> +that ABI compatibility with the major ABI version is preserved and therefore
>>> +soname's do not change.
>>> +
>>> +Minor versions are incremented to indicate the release of a new ABI compatible
>>> +DPDK release, typically the DPDK quarterly releases. An example of this, might
>>> +be that ``librte_eal.so.20.1`` would indicate the first ABI compatible DPDK
>>> +release, following the declaration of the new major ABI version ``20``.
>>> +
>>> +ABI versions, are supported by each release until such time as the next major
>>> +ABI version is declared. At that time, the deprecation of the previous major ABI
>>> +version will be noted in the Release Notes with guidance on individual symbol
>>> +depreciation and upgrade notes provided.
>>>
>>> -ABI versions are set at the time of major release labeling, and the ABI may
>>> -change multiple times, without warning, between the last release label and the
>>> -HEAD label of the git tree.
>>> +.. _abi_changes:
>>>
>>> -ABI versions, once released, are available until such time as their
>>> -deprecation has been noted in the Release Notes for at least one major release
>>> -cycle. For example consider the case where the ABI for DPDK 2.0 has been
>>> -shipped and then a decision is made to modify it during the development of
>>> -DPDK 2.1. The decision will be recorded in the Release Notes for the DPDK 2.1
>>> -release and the modification will be made available in the DPDK 2.2 release.
>>> +ABI Changes
>>> +~~~~~~~~~~~
>>>
>>> -ABI versions may be deprecated in whole or in part as needed by a given
>>> -update.
>>> +The ABI may still change after the declaration of a major ABI version, that is
>>> +new APIs may be still added or existing APIs may be modified.
>>>
>>> -Some ABI changes may be too significant to reasonably maintain multiple
>>> -versions. In those cases ABI's may be updated without backward compatibility
>>> -being provided. The requirements for doing so are:
>>> +.. Warning::
>>> +
>>> + Note that, the process for ABI deprecation should not be undertaken lightly.
>>> + ABI stability is extremely important for downstream consumers of the DPDK,
>>
>>> + especially when distributed in shared object form. Every effort should be
>>> + made to preserve the ABI whenever possible. The ABI should only be changed
>>> + for significant reasons, such as performance enhancements. ABI breakage due
>>> + to changes such as reorganizing public structure fields for aesthetic or
>>> + readability purposes should be avoided.
>>> +
>>
>> This text is not changed and it reads like *any* performance enhancement
>> is a good enough reason for an ABI break. Can't obviously quantify it,
>> but maybe "major performance enhancement" is closer to the intended
>> tone? Sorry for nit-picking over one word!
>
> I agree, I was in two minds about whether to clarify this section or if
> it was fine as-is. I left it there as a general warning to stop and
> think before you ask to change the ABI. A performance gain alone doesn't
> absolve the contributor from an obligation to preserve ABI compatibility.
>
> Perhaps reword as follows?
>
> .. Warning::
>
> Note that, this policy details the method by which the ABI may be
> changed, with due regard to preserving compatibility and observing
> depreciation notices. This process however should not be undertaken
> lightly, as a general rule ABI stability is extremely important for
> downstream consumers of DPDK. The ABI should only be changed for
> significant reasons, such as performance enhancements. ABI breakages due
> to changes such as reorganizing public structure fields for aesthetic or
> readability purposes should be avoided.
>
Hi Ray,
ok, thanks for checking it.
>>
>>> +
>>> +The requirements for changing the ABI are:
>>>
>>> #. At least 3 acknowledgments of the need to do so must be made on the
>>> dpdk.org mailing list.
>>> @@ -71,34 +121,119 @@ being provided. The requirements for doing so are:
>>> no maintainer is available for the component, the tree/sub-tree maintainer
>>> for that component must acknowledge the ABI change instead.
>>>
>>> + - The acknowledgment of a member of the technical board, as a delegate of the
>>> + `technical board <https://core.dpdk.org/techboard/>`_ acknowledging the
>>> + need for the ABI change, is also mandatory.
>>> +
>>> - It is also recommended that acknowledgments from different "areas of
>>> interest" be sought for each deprecation, for example: from NIC vendors,
>>> CPU vendors, end-users, etc.
>>>
>>> -#. The changes (including an alternative map file) can be included with
>>> - deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option,
>>> - to provide more details about oncoming changes.
>>> - ``RTE_NEXT_ABI`` wrapper will be removed when it become the default ABI.
>>> - More preferred way to provide this information is sending the feature
>>> - as a separate patch and reference it in deprecation notice.
>>> +#. Backward compatibly with the major ABI version must be maintained through
>>
>> s/compatibly/compatibility/
>
> ACK
>
>>
>>> + :ref:`abi_versioning`, with :ref:`forward-only <forward-only>` compatibility
>>> + offered for any ABI changes that are indicated to be part of the next ABI
>>> + version.
>>>
>>> -#. A full deprecation cycle, as explained above, must be made to offer
>>> - downstream consumers sufficient warning of the change.
>>> + - In situations were backward compatibility is not possible, read the
>>
>> s/were/where/
>
> ACK
>
>>
>>> + section on :ref:`abi_breakages`.
>>>
>>> -Note that the above process for ABI deprecation should not be undertaken
>>> -lightly. ABI stability is extremely important for downstream consumers of the
>>> -DPDK, especially when distributed in shared object form. Every effort should
>>> -be made to preserve the ABI whenever possible. The ABI should only be changed
>>> -for significant reasons, such as performance enhancements. ABI breakage due to
>>> -changes such as reorganizing public structure fields for aesthetic or
>>> -readability purposes should be avoided.
>>> + - No backward or forward compatibility is offered for API changes marked as
>>> + ``experimental``, as described in the section on :ref:`Experimental APIs
>>> + and Libraries <experimental_apis>`.
>>>
>>> -.. note::
>>> +#. If a newly proposed API functionally replaces an existing one, when the new
>>> + API becomes non-experimental, then the old one is marked with
>>> + ``__rte_deprecated``.
>>> +
>>> + - The depreciated API should follow the notification process to be removed,
>>> + see :ref:`deprecation_notices`.
>>> +
>>> + - At the declaration of the next major ABI version, those ABI changes then
>>> + become a formal part of the new ABI and the requirement to preserve ABI
>>> + compatibility with the last major ABI version is then dropped.
>>> +
>>> + - The responsibility for removing redundant ABI compatibility code rests
>>> + with the original contributor of the ABI changes, failing that, then with
>>> + the contributor's company and then finally with the maintainer.
>>> +
>>> +.. _forward-only:
>>> +
>>> +.. Note::
>>> +
>>> + Note that forward-only compatibility is offered for those changes made
>>> + between major ABI versions. As a library's soname can only describe
>>> + compatibility with the last major ABI version, until the next major ABI
>>> + version is declared, these changes therefore cannot be resolved as a runtime
>>> + dependency through the soname. Therefore any application wishing to make use
>>> + of these ABI changes can only ensure that it's runtime dependencies are met
>>> + through Operating System package versioning.
>>> +
>>> +.. _hw_rqmts:
>>> +
>>> +.. Note::
>>>
>>> Updates to the minimum hardware requirements, which drop support for hardware
>>> which was previously supported, should be treated as an ABI change, and
>>> - follow the relevant deprecation policy procedures as above: 3 acks and
>>> - announcement at least one release in advance.
>>> + follow the relevant deprecation policy procedures as above: 3 acks, technical
>>> + board approval and announcement at least one release in advance.
>>> +
>>> +.. _abi_breakages:
>>> +
>>> +ABI Breakages
>>> +~~~~~~~~~~~~~
>>> +
>>> +For those ABI changes that are too significant to reasonably maintain multiple
>>> +symbol versions, there is an amended process. In these cases, ABIs may be
>>> +updated without the requirement of backward compatibility being provided. These
>>> +changes must follow the `same process :ref:`described above <abi_changes>` as non-breaking
>>> +changes, however with the following additional requirements:
>>> +
>>> +#. ABI breaking changes (including an alternative map file) can be included with
>>> + deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option, to provide
>>> + more details about oncoming changes. ``RTE_NEXT_ABI`` wrapper will be removed
>>> + at the declaration of the next major ABI version.
>>> +
>>> +#. Once approved, and after the depreciation notice has been observed these
>>> + changes will form part of the next declared major ABI version.
>>> +
>>> +Examples of ABI Changes
>>> +~~~~~~~~~~~~~~~~~~~~~~~
>>> +
>>> +The following are examples of allowable ABI changes occurring between
>>> +declarations of major ABI versions.
>>> +
>>> +* DPDK 19.11 release, defines the function ``rte_foo()``, and ``rte_foo()``
>>> + as part of the major ABI version ``20``.
>>> +
>>> +* DPDK 20.02 release defines a new function ``rte_foo(uint8_t bar)``, and
>>> + this is not a problem as long as the symbol ``rte_foo@DPDK20`` is
>>> + preserved through :ref:`abi_versioning`.
>>> +
>>> + - The new function may be marked with the ``__rte_experimental`` tag for a
>>> + number of releases, as described in the section :ref:`experimental_apis`.
>>> +
>>> + - Once ``rte_foo(uint8_t bar)`` becomes non-experimental ``rte_foo()`` is then
>>> + declared as ``__rte_depreciated``, with an associated deprecation notice
>>> + provided.
>>> +
>>> +* DPDK 19.11 is not re-released to include ``rte_foo(uint8_t bar)``, the new
>>> + version of ``rte_foo`` only exists from DPDK 20.02 onwards as described in the
>>> + :ref:`note on forward-only compatibility<forward-only>`.
>>> +
>>> +* DPDK 20.02 release defines the experimental function ``__rte_experimental
>>> + rte_baz()``. This function may or may not exist in the DPDK 20.05 release.
>>> +
>>> +* An application ``dPacket`` wishes to use ``rte_foo(uint8_t bar)``, before the
>>> + declaration of the DPDK ``21`` major API version. The application can only
>>> + ensure it's runtime dependencies are met by specifying ``DPDK (>= 20.2)`` as
>>> + an explicit package dependency, as the soname only may only indicate the
>>> + supported major ABI version.
>>> +
>>> +* At the release of DPDK 20.11, the function ``rte_foo(uint8_t bar)`` becomes
>>> + formally part of then new major ABI version DPDK 21.0 and ``rte_foo()`` may be
>>> + removed.
>>> +
>>> +.. _deprecation_notices:
>>>
>>> Examples of Deprecation Notices
>>> ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
>>> @@ -106,46 +241,42 @@ Examples of Deprecation Notices
>>> The following are some examples of ABI deprecation notices which would be
>>> added to the Release Notes:
>>>
>>> -* The Macro ``#RTE_FOO`` is deprecated and will be removed with version 2.0,
>>> - to be replaced with the inline function ``rte_foo()``.
>>> +* The Macro ``#RTE_FOO`` is deprecated and will be removed with ABI version
>>> + 21, to be replaced with the inline function ``rte_foo()``.
>>>
>>> * The function ``rte_mbuf_grok()`` has been updated to include a new parameter
>>> - in version 2.0. Backwards compatibility will be maintained for this function
>>> - until the release of version 2.1
>>> + in version 20.2. Backwards compatibility will be maintained for this function
>>> + until the release of the new DPDK major ABI version 21, in DPDK version
>>> + 20.11.
>>>
>>> -* The members of ``struct rte_foo`` have been reorganized in release 2.0 for
>>> +* The members of ``struct rte_foo`` have been reorganized in DPDK 20.02 for
>>> performance reasons. Existing binary applications will have backwards
>>> - compatibility in release 2.0, while newly built binaries will need to
>>> - reference the new structure variant ``struct rte_foo2``. Compatibility will
>>> - be removed in release 2.2, and all applications will require updating and
>>> + compatibility in release 20.02, while newly built binaries will need to
>>> + reference the new structure variant ``struct rte_foo2``. Compatibility will be
>>> + removed in release 20.11, and all applications will require updating and
>>> rebuilding to the new structure at that time, which will be renamed to the
>>> original ``struct rte_foo``.
>>>
>>> * Significant ABI changes are planned for the ``librte_dostuff`` library. The
>>> - upcoming release 2.0 will not contain these changes, but release 2.1 will,
>>> + upcoming release 20.02 will not contain these changes, but release 20.11 will,
>>> and no backwards compatibility is planned due to the extensive nature of
>>> - these changes. Binaries using this library built prior to version 2.1 will
>>> + these changes. Binaries using this library built prior to ABI version 21 will
>>> require updating and recompilation.
>>>
>>> -New API replacing previous one
>>> -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
>>> -
>>> -If a new API proposed functionally replaces an existing one, when the new API
>>> -becomes non-experimental then the old one is marked with ``__rte_deprecated``.
>>> -Deprecated APIs are removed completely just after the next LTS.
>>> -
>>> -Reminder that old API should follow deprecation process to be removed.
>>> +.. _experimental_apis:
>>>
>>> +Experimental
>>> +------------
>>>
>>> -Experimental APIs
>>> ------------------
>>> +APIs
>>> +~~~~
>>>
>>> -APIs marked as ``experimental`` are not considered part of the ABI and may
>>> -change without warning at any time. Since changes to APIs are most likely
>>> -immediately after their introduction, as users begin to take advantage of
>>> -those new APIs and start finding issues with them, new DPDK APIs will be
>>> -automatically marked as ``experimental`` to allow for a period of stabilization
>>> -before they become part of a tracked ABI.
>>> +APIs marked as ``experimental`` are not considered part of an ABI version and
>>> +may change without warning at any time. Since changes to APIs are most likely
>>> +immediately after their introduction, as users begin to take advantage of those
>>> +new APIs and start finding issues with them, new DPDK APIs will be automatically
>>> +marked as ``experimental`` to allow for a period of stabilization before they
>>> +become part of a tracked ABI version.
>>>
>>> Note that marking an API as experimental is a multi step process.
>>> To mark an API as experimental, the symbols which are desired to be exported
>>> @@ -163,7 +294,16 @@ In addition to tagging the code with ``__rte_experimental``,
>>> the doxygen markup must also contain the EXPERIMENTAL string,
>>> and the MAINTAINERS file should note the EXPERIMENTAL libraries.
>>>
>>> -For removing the experimental tag associated with an API, deprecation notice
>>> -is not required. Though, an API should remain in experimental state for at least
>>> -one release. Thereafter, normal process of posting patch for review to mailing
>>> -list can be followed.
>>> +For removing the experimental tag associated with an API, deprecation notice is
>>> +not required. Though, an API should remain in experimental state for at least
>>> +one release. Thereafter, the normal process of posting patch for review to
>>> +mailing list can be followed.
>>> +
>>> +Libraries
>>> +~~~~~~~~~
>>> +
>>> +Libraries marked as ``experimental`` are entirely not considered part of an ABI
>>> +version, and may change without warning at any time. Experimental libraries
>>> +always have a major version of ``0`` to indicate they exist outside of
>>> +:ref:`abi_versioning` , with the minor version incremented with each ABI change
>>> +to library.
>>> diff --git a/doc/guides/contributing/stable.rst b/doc/guides/contributing/stable.rst
>>> index 6a5eee9..d95c200 100644
>>> --- a/doc/guides/contributing/stable.rst
>>> +++ b/doc/guides/contributing/stable.rst
>>> @@ -1,7 +1,7 @@
>>> .. SPDX-License-Identifier: BSD-3-Clause
>>> Copyright 2018 The DPDK contributors
>>>
>>> -.. stable_lts_releases:
>>> +.. _stable_lts_releases:
>>>
>>> DPDK Stable Releases and Long Term Support
>>> ==========================================
>>> @@ -53,6 +53,9 @@ year's November (X.11) release will be maintained as an LTS for 2 years.
>>> After the X.11 release, an LTS branch will be created for it at
>>> http://git.dpdk.org/dpdk-stable where bugfixes will be backported to.
>>>
>>> +A LTS release may align with the declaration of a new major ABI version,
>>> +please read the :ref:`abi_policy` for more information.
>>> +
>>
>> Above is worth to mention, but as discussed on call earlier today, the
>> changes below should be dropped from this patchset. At present each LTS
>> minor release (e.g. 18.11.2) maintains the API/ABI of the original LTS
>> release (e.g. 18.11) and that is not changing.
>
> ACK, I will remove
>
>>
>> What type of non-ABI breaking things are backported to LTS branches can
>> be discussed during the LTS presentation in DPDK userspace.
>
> Do you anticipate any updates here?
>
I should probably update it to add some more info, similar to one of the
slides from Bordeaux re examples etc.
thanks,
Kevin.
> Thanks,
>
> Ray K
>
>>
>> thanks,
>> Kevin.
>>
>>> It is anticipated that there will be at least 4 releases per year of the LTS
>>> or approximately 1 every 3 months. However, the cadence can be shorter or
>>> longer depending on the number and criticality of the backported
>>> @@ -68,10 +71,13 @@ point the LTS branch will no longer be maintained with no further releases.
>>> What changes should be backported
>>> ---------------------------------
>>>
>>> -Backporting should be limited to bug fixes. All patches accepted on the master
>>> -branch with a Fixes: tag should be backported to the relevant stable/LTS
>>> -branches, unless the submitter indicates otherwise. If there are exceptions,
>>> -they will be discussed on the mailing lists.
>>> +Backporting is a naturally conservative activity, and therefore should only
>>> +include bug fixes and support for new hardware, were adding support does not
>>> +necessitate DPDK ABI/API changes.
>>> +
>>> +All patches accepted on the master branch with a Fixes: tag should be backported
>>> +to the relevant stable/LTS branches, unless the submitter indicates otherwise.
>>> +If there are exceptions, they will be discussed on the mailing lists.
>>>
>>> Fixes suitable for backport should have a ``Cc: stable@dpdk.org`` tag in the
>>> commit message body as follows::
>>> @@ -86,13 +92,18 @@ commit message body as follows::
>>> Signed-off-by: Alex Smith <alex.smith@example.com>
>>>
>>>
>>> -Fixes not suitable for backport should not include the ``Cc: stable@dpdk.org`` tag.
>>> +Fixes not suitable for backport should not include the ``Cc: stable@dpdk.org``
>>> +tag.
>>>
>>> -Features should not be backported to stable releases. It may be acceptable, in
>>> -limited cases, to back port features for the LTS release where:
>>> +New features, with the exception of new hardware support, should not be
>>> +backported to stable releases. In the case of new hardware support or any other
>>> +exceptional circumstances limited backporting maybe permitted to the LTS release
>>> +where:
>>>
>>> -* There is a justifiable use case (for example a new PMD).
>>> -* The change is non-invasive.
>>> +* There is a justifiable use case, for example the change is required to support
>>> + a new platform or device (for example a new PMD).
>>> +* The change is ABI/API preserving, it does not present an obvious "new feature"
>>> + to end consumer.
>>> * The work of preparing the backport is done by the proposer.
>>> * There is support within the community.
>>>
>>> @@ -119,10 +130,3 @@ A Stable Release will be released by:
>>> list.
>>>
>>> Stable releases are available on the `dpdk.org download page <http://core.dpdk.org/download/>`_.
>>> -
>>> -
>>> -ABI
>>> ----
>>> -
>>> -The Stable Release should not be seen as a way of breaking or circumventing
>>> -the DPDK ABI policy.
>>>
^ permalink raw reply [relevance 5%]
* Re: [dpdk-dev] [PATCH v4 1/4] doc: separate versioning.rst into version and policy
2019-09-25 10:23 13% ` [dpdk-dev] [PATCH v4 1/4] doc: separate versioning.rst into version and policy Ray Kinsella
@ 2019-09-25 12:24 5% ` Neil Horman
2019-09-25 13:01 3% ` Ray Kinsella
2019-09-27 15:22 4% ` Ray Kinsella
2019-09-27 15:43 4% ` Aaron Conole
1 sibling, 2 replies; 200+ results
From: Neil Horman @ 2019-09-25 12:24 UTC (permalink / raw)
To: Ray Kinsella
Cc: dev, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, maxime.coquelin,
john.mcnamara, marko.kovacevic, hemant.agrawal, ktraynor
On Wed, Sep 25, 2019 at 11:23:53AM +0100, Ray Kinsella wrote:
> Separate versioning.rst into abi versioning and abi policy guidance, in
> preparation for adding more detail to the abi policy.
>
> Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
> ---
> doc/guides/contributing/abi_policy.rst | 169 +++++++++
> doc/guides/contributing/abi_versioning.rst | 427 +++++++++++++++++++++
> doc/guides/contributing/index.rst | 3 +-
> doc/guides/contributing/versioning.rst | 591 -----------------------------
> 4 files changed, 598 insertions(+), 592 deletions(-)
> create mode 100644 doc/guides/contributing/abi_policy.rst
> create mode 100644 doc/guides/contributing/abi_versioning.rst
> delete mode 100644 doc/guides/contributing/versioning.rst
>
> diff --git a/doc/guides/contributing/abi_policy.rst b/doc/guides/contributing/abi_policy.rst
> new file mode 100644
> index 0000000..55bacb4
> --- /dev/null
> +++ b/doc/guides/contributing/abi_policy.rst
> @@ -0,0 +1,169 @@
> +.. SPDX-License-Identifier: BSD-3-Clause
> + Copyright 2018 The DPDK contributors
> +
> +.. abi_api_policy:
> +
> +DPDK ABI/API policy
> +===================
> +
> +Description
> +-----------
> +
> +This document details some methods for handling ABI management in the DPDK.
> +
> +General Guidelines
> +------------------
> +
> +#. Whenever possible, ABI should be preserved
> +#. ABI/API may be changed with a deprecation process
> +#. The modification of symbols can generally be managed with versioning
> +#. Libraries or APIs marked in ``experimental`` state may change without constraint
> +#. New APIs will be marked as ``experimental`` for at least one release to allow
> + any issues found by users of the new API to be fixed quickly
> +#. The addition of symbols is generally not problematic
> +#. The removal of symbols generally is an ABI break and requires bumping of the
> + LIBABIVER macro
> +#. Updates to the minimum hardware requirements, which drop support for hardware which
> + was previously supported, should be treated as an ABI change.
> +
> +What is an ABI
> +~~~~~~~~~~~~~~
> +
> +An ABI (Application Binary Interface) is the set of runtime interfaces exposed
> +by a library. It is similar to an API (Application Programming Interface) but
> +is the result of compilation. It is also effectively cloned when applications
> +link to dynamic libraries. That is to say when an application is compiled to
> +link against dynamic libraries, it is assumed that the ABI remains constant
> +between the time the application is compiled/linked, and the time that it runs.
> +Therefore, in the case of dynamic linking, it is critical that an ABI is
> +preserved, or (when modified), done in such a way that the application is unable
> +to behave improperly or in an unexpected fashion.
> +
> +
> +ABI/API Deprecation
> +-------------------
> +
> +The DPDK ABI policy
> +~~~~~~~~~~~~~~~~~~~
> +
> +ABI versions are set at the time of major release labeling, and the ABI may
> +change multiple times, without warning, between the last release label and the
> +HEAD label of the git tree.
> +
> +ABI versions, once released, are available until such time as their
> +deprecation has been noted in the Release Notes for at least one major release
> +cycle. For example consider the case where the ABI for DPDK 2.0 has been
> +shipped and then a decision is made to modify it during the development of
> +DPDK 2.1. The decision will be recorded in the Release Notes for the DPDK 2.1
> +release and the modification will be made available in the DPDK 2.2 release.
> +
This seems..confusing. In patch 0:
=================================================================
* DPDK v20 is declared as the supported ABI version for one year, aligned with
the DPDK v19.11 (LTS) release. All library sonames are updated to reflect the
new ABI version, e.g. librte_eal.so.20, librte_acl.so.20...
* DPDK v20.02 .. v20.08 releases are ABI compatible with the DPDK v20 ABI. ABI
changes are permitted from DPDK v20.02 onwards, with the condition that ABI
compatibility with DPDK v20 is preserved.
* DPDK v21 is declared as the new supported ABI version for two years, aligned
with the DPDK v20.11 (LTS) release. The DPDK v20 ABI is now depreciated,
library sonames are updated to v21 and ABI compatibility breaking changes may
be introduced.
===================================================================
Issues I see:
1) We have lots of version numbers floating around here: v20 (referencing an ABI
version I think), DPDK 19.11 (an LTS release that maps to ABI v20), dpdk 20.02..
dpdk 20.08 which can modify the ABI as long as they maintain backwards
compatibility (I think), dpdk v21 (referecing a new ABI that will be supported
at a later release), dpdk 20.11 which guarantees ABI v21, dpdk 2.0 which maps to
abi v20, dpdk 2.1 (a minor release which decides to break ABI), and dpdk 2.2
(a subsequent minor release which adheres to a new abi)
2) Conflicts as to when ABI can be modified in breaking and compatible ways.
Are we allowed to break abi after 1 year, or only in a new major release
I think you need a taxonomy, to clearly deliniate your syntax for noting abi
versions, vs dpdk release major versions, and minor versions, so we are more
clear as to what the docs are referring to, as well as perhaps a timeline to
more clearly illustrate when compatible and incompatible ABI changes are
allowed.
> +ABI versions may be deprecated in whole or in part as needed by a given
> +update.
> +
> +Some ABI changes may be too significant to reasonably maintain multiple
> +versions. In those cases ABI's may be updated without backward compatibility
> +being provided. The requirements for doing so are:
> +
> +#. At least 3 acknowledgments of the need to do so must be made on the
> + dpdk.org mailing list.
> +
> + - The acknowledgment of the maintainer of the component is mandatory, or if
> + no maintainer is available for the component, the tree/sub-tree maintainer
> + for that component must acknowledge the ABI change instead.
> +
> + - It is also recommended that acknowledgments from different "areas of
> + interest" be sought for each deprecation, for example: from NIC vendors,
> + CPU vendors, end-users, etc.
> +
> +#. The changes (including an alternative map file) can be included with
> + deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option,
> + to provide more details about oncoming changes.
> + ``RTE_NEXT_ABI`` wrapper will be removed when it become the default ABI.
> + More preferred way to provide this information is sending the feature
> + as a separate patch and reference it in deprecation notice.
> +
> +#. A full deprecation cycle, as explained above, must be made to offer
> + downstream consumers sufficient warning of the change.
> +
> +Note that the above process for ABI deprecation should not be undertaken
> +lightly. ABI stability is extremely important for downstream consumers of the
> +DPDK, especially when distributed in shared object form. Every effort should
> +be made to preserve the ABI whenever possible. The ABI should only be changed
> +for significant reasons, such as performance enhancements. ABI breakage due to
> +changes such as reorganizing public structure fields for aesthetic or
> +readability purposes should be avoided.
> +
> +.. note::
> +
> + Updates to the minimum hardware requirements, which drop support for hardware
> + which was previously supported, should be treated as an ABI change, and
> + follow the relevant deprecation policy procedures as above: 3 acks and
> + announcement at least one release in advance.
> +
> +Examples of Deprecation Notices
> +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
> +
> +The following are some examples of ABI deprecation notices which would be
> +added to the Release Notes:
> +
> +* The Macro ``#RTE_FOO`` is deprecated and will be removed with version 2.0,
> + to be replaced with the inline function ``rte_foo()``.
> +
> +* The function ``rte_mbuf_grok()`` has been updated to include a new parameter
> + in version 2.0. Backwards compatibility will be maintained for this function
> + until the release of version 2.1
> +
> +* The members of ``struct rte_foo`` have been reorganized in release 2.0 for
> + performance reasons. Existing binary applications will have backwards
> + compatibility in release 2.0, while newly built binaries will need to
> + reference the new structure variant ``struct rte_foo2``. Compatibility will
> + be removed in release 2.2, and all applications will require updating and
> + rebuilding to the new structure at that time, which will be renamed to the
> + original ``struct rte_foo``.
> +
> +* Significant ABI changes are planned for the ``librte_dostuff`` library. The
> + upcoming release 2.0 will not contain these changes, but release 2.1 will,
> + and no backwards compatibility is planned due to the extensive nature of
> + these changes. Binaries using this library built prior to version 2.1 will
> + require updating and recompilation.
> +
> +New API replacing previous one
> +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
> +
> +If a new API proposed functionally replaces an existing one, when the new API
> +becomes non-experimental then the old one is marked with ``__rte_deprecated``.
> +Deprecated APIs are removed completely just after the next LTS.
> +
> +Reminder that old API should follow deprecation process to be removed.
> +
> +
> +Experimental APIs
> +-----------------
> +
> +APIs marked as ``experimental`` are not considered part of the ABI and may
> +change without warning at any time. Since changes to APIs are most likely
> +immediately after their introduction, as users begin to take advantage of
> +those new APIs and start finding issues with them, new DPDK APIs will be
> +automatically marked as ``experimental`` to allow for a period of stabilization
> +before they become part of a tracked ABI.
> +
> +Note that marking an API as experimental is a multi step process.
> +To mark an API as experimental, the symbols which are desired to be exported
> +must be placed in an EXPERIMENTAL version block in the corresponding libraries'
> +version map script.
> +Secondly, the corresponding prototypes of those exported functions (in the
> +development header files), must be marked with the ``__rte_experimental`` tag
> +(see ``rte_compat.h``).
> +The DPDK build makefiles perform a check to ensure that the map file and the
> +C code reflect the same list of symbols.
> +This check can be circumvented by defining ``ALLOW_EXPERIMENTAL_API``
> +during compilation in the corresponding library Makefile.
> +
> +In addition to tagging the code with ``__rte_experimental``,
> +the doxygen markup must also contain the EXPERIMENTAL string,
> +and the MAINTAINERS file should note the EXPERIMENTAL libraries.
> +
> +For removing the experimental tag associated with an API, deprecation notice
> +is not required. Though, an API should remain in experimental state for at least
> +one release. Thereafter, normal process of posting patch for review to mailing
> +list can be followed.
> diff --git a/doc/guides/contributing/abi_versioning.rst b/doc/guides/contributing/abi_versioning.rst
> new file mode 100644
> index 0000000..53e6ac0
> --- /dev/null
> +++ b/doc/guides/contributing/abi_versioning.rst
> @@ -0,0 +1,427 @@
> +.. SPDX-License-Identifier: BSD-3-Clause
> + Copyright 2018 The DPDK contributors
> +
> +.. library_versioning:
> +
> +Library versioning
> +------------------
> +
> +Downstreams might want to provide different DPDK releases at the same time to
> +support multiple consumers of DPDK linked against older and newer sonames.
> +
> +Also due to the interdependencies that DPDK libraries can have applications
> +might end up with an executable space in which multiple versions of a library
> +are mapped by ld.so.
> +
> +Think of LibA that got an ABI bump and LibB that did not get an ABI bump but is
> +depending on LibA.
> +
> +.. note::
> +
> + Application
> + \-> LibA.old
> + \-> LibB.new -> LibA.new
> +
> +That is a conflict which can be avoided by setting ``CONFIG_RTE_MAJOR_ABI``.
> +If set, the value of ``CONFIG_RTE_MAJOR_ABI`` overwrites all - otherwise per
> +library - versions defined in the libraries ``LIBABIVER``.
> +An example might be ``CONFIG_RTE_MAJOR_ABI=16.11`` which will make all libraries
> +``librte<?>.so.16.11`` instead of ``librte<?>.so.<LIBABIVER>``.
> +
> +
> +ABI versioning
> +--------------
> +
> +Versioning Macros
> +~~~~~~~~~~~~~~~~~
> +
> +When a symbol is exported from a library to provide an API, it also provides a
> +calling convention (ABI) that is embodied in its name, return type and
> +arguments. Occasionally that function may need to change to accommodate new
> +functionality or behavior. When that occurs, it is desirable to allow for
> +backward compatibility for a time with older binaries that are dynamically
> +linked to the DPDK.
> +
> +To support backward compatibility the ``rte_compat.h``
> +header file provides macros to use when updating exported functions. These
> +macros are used in conjunction with the ``rte_<library>_version.map`` file for
> +a given library to allow multiple versions of a symbol to exist in a shared
> +library so that older binaries need not be immediately recompiled.
> +
> +The macros exported are:
> +
> +* ``VERSION_SYMBOL(b, e, n)``: Creates a symbol version table entry binding
> + versioned symbol ``b@DPDK_n`` to the internal function ``b_e``.
> +
> +* ``BIND_DEFAULT_SYMBOL(b, e, n)``: Creates a symbol version entry instructing
> + the linker to bind references to symbol ``b`` to the internal symbol
> + ``b_e``.
> +
> +* ``MAP_STATIC_SYMBOL(f, p)``: Declare the prototype ``f``, and map it to the
> + fully qualified function ``p``, so that if a symbol becomes versioned, it
> + can still be mapped back to the public symbol name.
> +
> +Examples of ABI Macro use
> +^^^^^^^^^^^^^^^^^^^^^^^^^
> +
> +Updating a public API
> +_____________________
> +
> +Assume we have a function as follows
> +
> +.. code-block:: c
> +
> + /*
> + * Create an acl context object for apps to
> + * manipulate
> + */
> + struct rte_acl_ctx *
> + rte_acl_create(const struct rte_acl_param *param)
> + {
> + ...
> + }
> +
> +
> +Assume that struct rte_acl_ctx is a private structure, and that a developer
> +wishes to enhance the acl api so that a debugging flag can be enabled on a
> +per-context basis. This requires an addition to the structure (which, being
> +private, is safe), but it also requires modifying the code as follows
> +
> +.. code-block:: c
> +
> + /*
> + * Create an acl context object for apps to
> + * manipulate
> + */
> + struct rte_acl_ctx *
> + rte_acl_create(const struct rte_acl_param *param, int debug)
> + {
> + ...
> + }
> +
> +
> +Note also that, being a public function, the header file prototype must also be
> +changed, as must all the call sites, to reflect the new ABI footprint. We will
> +maintain previous ABI versions that are accessible only to previously compiled
> +binaries
> +
> +The addition of a parameter to the function is ABI breaking as the function is
> +public, and existing application may use it in its current form. However, the
> +compatibility macros in DPDK allow a developer to use symbol versioning so that
> +multiple functions can be mapped to the same public symbol based on when an
> +application was linked to it. To see how this is done, we start with the
> +requisite libraries version map file. Initially the version map file for the
> +acl library looks like this
> +
> +.. code-block:: none
> +
> + DPDK_2.0 {
> + global:
> +
> + rte_acl_add_rules;
> + rte_acl_build;
> + rte_acl_classify;
> + rte_acl_classify_alg;
> + rte_acl_classify_scalar;
> + rte_acl_create;
> + rte_acl_dump;
> + rte_acl_find_existing;
> + rte_acl_free;
> + rte_acl_ipv4vlan_add_rules;
> + rte_acl_ipv4vlan_build;
> + rte_acl_list_dump;
> + rte_acl_reset;
> + rte_acl_reset_rules;
> + rte_acl_set_ctx_classify;
> +
> + local: *;
> + };
> +
> +This file needs to be modified as follows
> +
> +.. code-block:: none
> +
> + DPDK_2.0 {
> + global:
> +
> + rte_acl_add_rules;
> + rte_acl_build;
> + rte_acl_classify;
> + rte_acl_classify_alg;
> + rte_acl_classify_scalar;
> + rte_acl_create;
> + rte_acl_dump;
> + rte_acl_find_existing;
> + rte_acl_free;
> + rte_acl_ipv4vlan_add_rules;
> + rte_acl_ipv4vlan_build;
> + rte_acl_list_dump;
> + rte_acl_reset;
> + rte_acl_reset_rules;
> + rte_acl_set_ctx_classify;
> +
> + local: *;
> + };
> +
> + DPDK_2.1 {
> + global:
> + rte_acl_create;
> +
> + } DPDK_2.0;
> +
> +The addition of the new block tells the linker that a new version node is
> +available (DPDK_2.1), which contains the symbol rte_acl_create, and inherits the
> +symbols from the DPDK_2.0 node. This list is directly translated into a list of
> +exported symbols when DPDK is compiled as a shared library
> +
> +Next, we need to specify in the code which function map to the rte_acl_create
> +symbol at which versions. First, at the site of the initial symbol definition,
> +we need to update the function so that it is uniquely named, and not in conflict
> +with the public symbol name
> +
> +.. code-block:: c
> +
> + struct rte_acl_ctx *
> + -rte_acl_create(const struct rte_acl_param *param)
> + +rte_acl_create_v20(const struct rte_acl_param *param)
> + {
> + size_t sz;
> + struct rte_acl_ctx *ctx;
> + ...
> +
> +Note that the base name of the symbol was kept intact, as this is conducive to
> +the macros used for versioning symbols. That is our next step, mapping this new
> +symbol name to the initial symbol name at version node 2.0. Immediately after
> +the function, we add this line of code
> +
> +.. code-block:: c
> +
> + VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
> +
> +Remembering to also add the rte_compat.h header to the requisite c file where
> +these changes are being made. The above macro instructs the linker to create a
> +new symbol ``rte_acl_create@DPDK_2.0``, which matches the symbol created in older
> +builds, but now points to the above newly named function. We have now mapped
> +the original rte_acl_create symbol to the original function (but with a new
> +name)
> +
> +Next, we need to create the 2.1 version of the symbol. We create a new function
> +name, with a different suffix, and implement it appropriately
> +
> +.. code-block:: c
> +
> + struct rte_acl_ctx *
> + rte_acl_create_v21(const struct rte_acl_param *param, int debug);
> + {
> + struct rte_acl_ctx *ctx = rte_acl_create_v20(param);
> +
> + ctx->debug = debug;
> +
> + return ctx;
> + }
> +
> +This code serves as our new API call. Its the same as our old call, but adds
> +the new parameter in place. Next we need to map this function to the symbol
> +``rte_acl_create@DPDK_2.1``. To do this, we modify the public prototype of the call
> +in the header file, adding the macro there to inform all including applications,
> +that on re-link, the default rte_acl_create symbol should point to this
> +function. Note that we could do this by simply naming the function above
> +rte_acl_create, and the linker would chose the most recent version tag to apply
> +in the version script, but we can also do this in the header file
> +
> +.. code-block:: c
> +
> + struct rte_acl_ctx *
> + -rte_acl_create(const struct rte_acl_param *param);
> + +rte_acl_create(const struct rte_acl_param *param, int debug);
> + +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
> +
> +The BIND_DEFAULT_SYMBOL macro explicitly tells applications that include this
> +header, to link to the rte_acl_create_v21 function and apply the DPDK_2.1
> +version node to it. This method is more explicit and flexible than just
> +re-implementing the exact symbol name, and allows for other features (such as
> +linking to the old symbol version by default, when the new ABI is to be opt-in
> +for a period.
> +
> +One last thing we need to do. Note that we've taken what was a public symbol,
> +and duplicated it into two uniquely and differently named symbols. We've then
> +mapped each of those back to the public symbol ``rte_acl_create`` with different
> +version tags. This only applies to dynamic linking, as static linking has no
> +notion of versioning. That leaves this code in a position of no longer having a
> +symbol simply named ``rte_acl_create`` and a static build will fail on that
> +missing symbol.
> +
> +To correct this, we can simply map a function of our choosing back to the public
> +symbol in the static build with the ``MAP_STATIC_SYMBOL`` macro. Generally the
> +assumption is that the most recent version of the symbol is the one you want to
> +map. So, back in the C file where, immediately after ``rte_acl_create_v21`` is
> +defined, we add this
> +
> +.. code-block:: c
> +
> + struct rte_acl_ctx *
> + rte_acl_create_v21(const struct rte_acl_param *param, int debug)
> + {
> + ...
> + }
> + MAP_STATIC_SYMBOL(struct rte_acl_ctx *rte_acl_create(const struct rte_acl_param *param, int debug), rte_acl_create_v21);
> +
> +That tells the compiler that, when building a static library, any calls to the
> +symbol ``rte_acl_create`` should be linked to ``rte_acl_create_v21``
> +
> +That's it, on the next shared library rebuild, there will be two versions of
> +rte_acl_create, an old DPDK_2.0 version, used by previously built applications,
> +and a new DPDK_2.1 version, used by future built applications.
> +
> +
> +Deprecating part of a public API
> +________________________________
> +
> +Lets assume that you've done the above update, and after a few releases have
> +passed you decide you would like to retire the old version of the function.
> +After having gone through the ABI deprecation announcement process, removal is
> +easy. Start by removing the symbol from the requisite version map file:
> +
> +.. code-block:: none
> +
> + DPDK_2.0 {
> + global:
> +
> + rte_acl_add_rules;
> + rte_acl_build;
> + rte_acl_classify;
> + rte_acl_classify_alg;
> + rte_acl_classify_scalar;
> + rte_acl_dump;
> + - rte_acl_create
> + rte_acl_find_existing;
> + rte_acl_free;
> + rte_acl_ipv4vlan_add_rules;
> + rte_acl_ipv4vlan_build;
> + rte_acl_list_dump;
> + rte_acl_reset;
> + rte_acl_reset_rules;
> + rte_acl_set_ctx_classify;
> +
> + local: *;
> + };
> +
> + DPDK_2.1 {
> + global:
> + rte_acl_create;
> + } DPDK_2.0;
> +
> +
> +Next remove the corresponding versioned export.
> +
> +.. code-block:: c
> +
> + -VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
> +
> +
> +Note that the internal function definition could also be removed, but its used
> +in our example by the newer version _v21, so we leave it in place. This is a
> +coding style choice.
> +
> +Lastly, we need to bump the LIBABIVER number for this library in the Makefile to
> +indicate to applications doing dynamic linking that this is a later, and
> +possibly incompatible library version:
> +
> +.. code-block:: c
> +
> + -LIBABIVER := 1
> + +LIBABIVER := 2
> +
> +Deprecating an entire ABI version
> +_________________________________
> +
> +While removing a symbol from and ABI may be useful, it is often more practical
> +to remove an entire version node at once. If a version node completely
> +specifies an API, then removing part of it, typically makes it incomplete. In
> +those cases it is better to remove the entire node
> +
> +To do this, start by modifying the version map file, such that all symbols from
> +the node to be removed are merged into the next node in the map
> +
> +In the case of our map above, it would transform to look as follows
> +
> +.. code-block:: none
> +
> + DPDK_2.1 {
> + global:
> +
> + rte_acl_add_rules;
> + rte_acl_build;
> + rte_acl_classify;
> + rte_acl_classify_alg;
> + rte_acl_classify_scalar;
> + rte_acl_dump;
> + rte_acl_create
> + rte_acl_find_existing;
> + rte_acl_free;
> + rte_acl_ipv4vlan_add_rules;
> + rte_acl_ipv4vlan_build;
> + rte_acl_list_dump;
> + rte_acl_reset;
> + rte_acl_reset_rules;
> + rte_acl_set_ctx_classify;
> +
> + local: *;
> + };
> +
> +Then any uses of BIND_DEFAULT_SYMBOL that pointed to the old node should be
> +updated to point to the new version node in any header files for all affected
> +symbols.
> +
> +.. code-block:: c
> +
> + -BIND_DEFAULT_SYMBOL(rte_acl_create, _v20, 2.0);
> + +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
> +
> +Lastly, any VERSION_SYMBOL macros that point to the old version node should be
> +removed, taking care to keep, where need old code in place to support newer
> +versions of the symbol.
> +
> +
> +Running the ABI Validator
> +-------------------------
> +
> +The ``devtools`` directory in the DPDK source tree contains a utility program,
> +``validate-abi.sh``, for validating the DPDK ABI based on the Linux `ABI
> +Compliance Checker
> +<http://ispras.linuxbase.org/index.php/ABI_compliance_checker>`_.
> +
> +This has a dependency on the ``abi-compliance-checker`` and ``and abi-dumper``
> +utilities which can be installed via a package manager. For example::
> +
> + sudo yum install abi-compliance-checker
> + sudo yum install abi-dumper
> +
> +The syntax of the ``validate-abi.sh`` utility is::
> +
> + ./devtools/validate-abi.sh <REV1> <REV2>
> +
> +Where ``REV1`` and ``REV2`` are valid gitrevisions(7)
> +https://www.kernel.org/pub/software/scm/git/docs/gitrevisions.html
> +on the local repo.
> +
> +For example::
> +
> + # Check between the previous and latest commit:
> + ./devtools/validate-abi.sh HEAD~1 HEAD
> +
> + # Check on a specific compilation target:
> + ./devtools/validate-abi.sh -t x86_64-native-linux-gcc HEAD~1 HEAD
> +
> + # Check between two tags:
> + ./devtools/validate-abi.sh v2.0.0 v2.1.0
> +
> + # Check between git master and local topic-branch "vhost-hacking":
> + ./devtools/validate-abi.sh master vhost-hacking
> +
> +After the validation script completes (it can take a while since it need to
> +compile both tags) it will create compatibility reports in the
> +``./abi-check/compat_report`` directory. Listed incompatibilities can be found
> +as follows::
> +
> + grep -lr Incompatible abi-check/compat_reports/
> diff --git a/doc/guides/contributing/index.rst b/doc/guides/contributing/index.rst
> index e2608d3..2fefd91 100644
> --- a/doc/guides/contributing/index.rst
> +++ b/doc/guides/contributing/index.rst
> @@ -10,7 +10,8 @@ Contributor's Guidelines
>
> coding_style
> design
> - versioning
> + abi_policy
> + abi_versioning
> documentation
> patches
> vulnerability
> diff --git a/doc/guides/contributing/versioning.rst b/doc/guides/contributing/versioning.rst
> deleted file mode 100644
> index 3ab2c43..0000000
> --- a/doc/guides/contributing/versioning.rst
> +++ /dev/null
> @@ -1,591 +0,0 @@
> -.. SPDX-License-Identifier: BSD-3-Clause
> - Copyright 2018 The DPDK contributors
> -
> -DPDK ABI/API policy
> -===================
> -
> -Description
> ------------
> -
> -This document details some methods for handling ABI management in the DPDK.
> -
> -General Guidelines
> -------------------
> -
> -#. Whenever possible, ABI should be preserved
> -#. ABI/API may be changed with a deprecation process
> -#. The modification of symbols can generally be managed with versioning
> -#. Libraries or APIs marked in ``experimental`` state may change without constraint
> -#. New APIs will be marked as ``experimental`` for at least one release to allow
> - any issues found by users of the new API to be fixed quickly
> -#. The addition of symbols is generally not problematic
> -#. The removal of symbols generally is an ABI break and requires bumping of the
> - LIBABIVER macro
> -#. Updates to the minimum hardware requirements, which drop support for hardware which
> - was previously supported, should be treated as an ABI change.
> -
> -What is an ABI
> -~~~~~~~~~~~~~~
> -
> -An ABI (Application Binary Interface) is the set of runtime interfaces exposed
> -by a library. It is similar to an API (Application Programming Interface) but
> -is the result of compilation. It is also effectively cloned when applications
> -link to dynamic libraries. That is to say when an application is compiled to
> -link against dynamic libraries, it is assumed that the ABI remains constant
> -between the time the application is compiled/linked, and the time that it runs.
> -Therefore, in the case of dynamic linking, it is critical that an ABI is
> -preserved, or (when modified), done in such a way that the application is unable
> -to behave improperly or in an unexpected fashion.
> -
> -
> -ABI/API Deprecation
> --------------------
> -
> -The DPDK ABI policy
> -~~~~~~~~~~~~~~~~~~~
> -
> -ABI versions are set at the time of major release labeling, and the ABI may
> -change multiple times, without warning, between the last release label and the
> -HEAD label of the git tree.
> -
> -ABI versions, once released, are available until such time as their
> -deprecation has been noted in the Release Notes for at least one major release
> -cycle. For example consider the case where the ABI for DPDK 2.0 has been
> -shipped and then a decision is made to modify it during the development of
> -DPDK 2.1. The decision will be recorded in the Release Notes for the DPDK 2.1
> -release and the modification will be made available in the DPDK 2.2 release.
> -
> -ABI versions may be deprecated in whole or in part as needed by a given
> -update.
> -
> -Some ABI changes may be too significant to reasonably maintain multiple
> -versions. In those cases ABI's may be updated without backward compatibility
> -being provided. The requirements for doing so are:
> -
> -#. At least 3 acknowledgments of the need to do so must be made on the
> - dpdk.org mailing list.
> -
> - - The acknowledgment of the maintainer of the component is mandatory, or if
> - no maintainer is available for the component, the tree/sub-tree maintainer
> - for that component must acknowledge the ABI change instead.
> -
> - - It is also recommended that acknowledgments from different "areas of
> - interest" be sought for each deprecation, for example: from NIC vendors,
> - CPU vendors, end-users, etc.
> -
> -#. The changes (including an alternative map file) can be included with
> - deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option,
> - to provide more details about oncoming changes.
> - ``RTE_NEXT_ABI`` wrapper will be removed when it become the default ABI.
> - More preferred way to provide this information is sending the feature
> - as a separate patch and reference it in deprecation notice.
> -
> -#. A full deprecation cycle, as explained above, must be made to offer
> - downstream consumers sufficient warning of the change.
> -
> -Note that the above process for ABI deprecation should not be undertaken
> -lightly. ABI stability is extremely important for downstream consumers of the
> -DPDK, especially when distributed in shared object form. Every effort should
> -be made to preserve the ABI whenever possible. The ABI should only be changed
> -for significant reasons, such as performance enhancements. ABI breakage due to
> -changes such as reorganizing public structure fields for aesthetic or
> -readability purposes should be avoided.
> -
> -.. note::
> -
> - Updates to the minimum hardware requirements, which drop support for hardware
> - which was previously supported, should be treated as an ABI change, and
> - follow the relevant deprecation policy procedures as above: 3 acks and
> - announcement at least one release in advance.
> -
> -Examples of Deprecation Notices
> -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
> -
> -The following are some examples of ABI deprecation notices which would be
> -added to the Release Notes:
> -
> -* The Macro ``#RTE_FOO`` is deprecated and will be removed with version 2.0,
> - to be replaced with the inline function ``rte_foo()``.
> -
> -* The function ``rte_mbuf_grok()`` has been updated to include a new parameter
> - in version 2.0. Backwards compatibility will be maintained for this function
> - until the release of version 2.1
> -
> -* The members of ``struct rte_foo`` have been reorganized in release 2.0 for
> - performance reasons. Existing binary applications will have backwards
> - compatibility in release 2.0, while newly built binaries will need to
> - reference the new structure variant ``struct rte_foo2``. Compatibility will
> - be removed in release 2.2, and all applications will require updating and
> - rebuilding to the new structure at that time, which will be renamed to the
> - original ``struct rte_foo``.
> -
> -* Significant ABI changes are planned for the ``librte_dostuff`` library. The
> - upcoming release 2.0 will not contain these changes, but release 2.1 will,
> - and no backwards compatibility is planned due to the extensive nature of
> - these changes. Binaries using this library built prior to version 2.1 will
> - require updating and recompilation.
> -
> -New API replacing previous one
> -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
> -
> -If a new API proposed functionally replaces an existing one, when the new API
> -becomes non-experimental then the old one is marked with ``__rte_deprecated``.
> -Deprecated APIs are removed completely just after the next LTS.
> -
> -Reminder that old API should follow deprecation process to be removed.
> -
> -
> -Experimental APIs
> ------------------
> -
> -APIs marked as ``experimental`` are not considered part of the ABI and may
> -change without warning at any time. Since changes to APIs are most likely
> -immediately after their introduction, as users begin to take advantage of
> -those new APIs and start finding issues with them, new DPDK APIs will be
> -automatically marked as ``experimental`` to allow for a period of stabilization
> -before they become part of a tracked ABI.
> -
> -Note that marking an API as experimental is a multi step process.
> -To mark an API as experimental, the symbols which are desired to be exported
> -must be placed in an EXPERIMENTAL version block in the corresponding libraries'
> -version map script.
> -Secondly, the corresponding prototypes of those exported functions (in the
> -development header files), must be marked with the ``__rte_experimental`` tag
> -(see ``rte_compat.h``).
> -The DPDK build makefiles perform a check to ensure that the map file and the
> -C code reflect the same list of symbols.
> -This check can be circumvented by defining ``ALLOW_EXPERIMENTAL_API``
> -during compilation in the corresponding library Makefile.
> -
> -In addition to tagging the code with ``__rte_experimental``,
> -the doxygen markup must also contain the EXPERIMENTAL string,
> -and the MAINTAINERS file should note the EXPERIMENTAL libraries.
> -
> -For removing the experimental tag associated with an API, deprecation notice
> -is not required. Though, an API should remain in experimental state for at least
> -one release. Thereafter, normal process of posting patch for review to mailing
> -list can be followed.
> -
> -
> -Library versioning
> -------------------
> -
> -Downstreams might want to provide different DPDK releases at the same time to
> -support multiple consumers of DPDK linked against older and newer sonames.
> -
> -Also due to the interdependencies that DPDK libraries can have applications
> -might end up with an executable space in which multiple versions of a library
> -are mapped by ld.so.
> -
> -Think of LibA that got an ABI bump and LibB that did not get an ABI bump but is
> -depending on LibA.
> -
> -.. note::
> -
> - Application
> - \-> LibA.old
> - \-> LibB.new -> LibA.new
> -
> -That is a conflict which can be avoided by setting ``CONFIG_RTE_MAJOR_ABI``.
> -If set, the value of ``CONFIG_RTE_MAJOR_ABI`` overwrites all - otherwise per
> -library - versions defined in the libraries ``LIBABIVER``.
> -An example might be ``CONFIG_RTE_MAJOR_ABI=16.11`` which will make all libraries
> -``librte<?>.so.16.11`` instead of ``librte<?>.so.<LIBABIVER>``.
> -
> -
> -ABI versioning
> ---------------
> -
> -Versioning Macros
> -~~~~~~~~~~~~~~~~~
> -
> -When a symbol is exported from a library to provide an API, it also provides a
> -calling convention (ABI) that is embodied in its name, return type and
> -arguments. Occasionally that function may need to change to accommodate new
> -functionality or behavior. When that occurs, it is desirable to allow for
> -backward compatibility for a time with older binaries that are dynamically
> -linked to the DPDK.
> -
> -To support backward compatibility the ``rte_compat.h``
> -header file provides macros to use when updating exported functions. These
> -macros are used in conjunction with the ``rte_<library>_version.map`` file for
> -a given library to allow multiple versions of a symbol to exist in a shared
> -library so that older binaries need not be immediately recompiled.
> -
> -The macros exported are:
> -
> -* ``VERSION_SYMBOL(b, e, n)``: Creates a symbol version table entry binding
> - versioned symbol ``b@DPDK_n`` to the internal function ``b_e``.
> -
> -* ``BIND_DEFAULT_SYMBOL(b, e, n)``: Creates a symbol version entry instructing
> - the linker to bind references to symbol ``b`` to the internal symbol
> - ``b_e``.
> -
> -* ``MAP_STATIC_SYMBOL(f, p)``: Declare the prototype ``f``, and map it to the
> - fully qualified function ``p``, so that if a symbol becomes versioned, it
> - can still be mapped back to the public symbol name.
> -
> -Examples of ABI Macro use
> -^^^^^^^^^^^^^^^^^^^^^^^^^
> -
> -Updating a public API
> -_____________________
> -
> -Assume we have a function as follows
> -
> -.. code-block:: c
> -
> - /*
> - * Create an acl context object for apps to
> - * manipulate
> - */
> - struct rte_acl_ctx *
> - rte_acl_create(const struct rte_acl_param *param)
> - {
> - ...
> - }
> -
> -
> -Assume that struct rte_acl_ctx is a private structure, and that a developer
> -wishes to enhance the acl api so that a debugging flag can be enabled on a
> -per-context basis. This requires an addition to the structure (which, being
> -private, is safe), but it also requires modifying the code as follows
> -
> -.. code-block:: c
> -
> - /*
> - * Create an acl context object for apps to
> - * manipulate
> - */
> - struct rte_acl_ctx *
> - rte_acl_create(const struct rte_acl_param *param, int debug)
> - {
> - ...
> - }
> -
> -
> -Note also that, being a public function, the header file prototype must also be
> -changed, as must all the call sites, to reflect the new ABI footprint. We will
> -maintain previous ABI versions that are accessible only to previously compiled
> -binaries
> -
> -The addition of a parameter to the function is ABI breaking as the function is
> -public, and existing application may use it in its current form. However, the
> -compatibility macros in DPDK allow a developer to use symbol versioning so that
> -multiple functions can be mapped to the same public symbol based on when an
> -application was linked to it. To see how this is done, we start with the
> -requisite libraries version map file. Initially the version map file for the
> -acl library looks like this
> -
> -.. code-block:: none
> -
> - DPDK_2.0 {
> - global:
> -
> - rte_acl_add_rules;
> - rte_acl_build;
> - rte_acl_classify;
> - rte_acl_classify_alg;
> - rte_acl_classify_scalar;
> - rte_acl_create;
> - rte_acl_dump;
> - rte_acl_find_existing;
> - rte_acl_free;
> - rte_acl_ipv4vlan_add_rules;
> - rte_acl_ipv4vlan_build;
> - rte_acl_list_dump;
> - rte_acl_reset;
> - rte_acl_reset_rules;
> - rte_acl_set_ctx_classify;
> -
> - local: *;
> - };
> -
> -This file needs to be modified as follows
> -
> -.. code-block:: none
> -
> - DPDK_2.0 {
> - global:
> -
> - rte_acl_add_rules;
> - rte_acl_build;
> - rte_acl_classify;
> - rte_acl_classify_alg;
> - rte_acl_classify_scalar;
> - rte_acl_create;
> - rte_acl_dump;
> - rte_acl_find_existing;
> - rte_acl_free;
> - rte_acl_ipv4vlan_add_rules;
> - rte_acl_ipv4vlan_build;
> - rte_acl_list_dump;
> - rte_acl_reset;
> - rte_acl_reset_rules;
> - rte_acl_set_ctx_classify;
> -
> - local: *;
> - };
> -
> - DPDK_2.1 {
> - global:
> - rte_acl_create;
> -
> - } DPDK_2.0;
> -
> -The addition of the new block tells the linker that a new version node is
> -available (DPDK_2.1), which contains the symbol rte_acl_create, and inherits the
> -symbols from the DPDK_2.0 node. This list is directly translated into a list of
> -exported symbols when DPDK is compiled as a shared library
> -
> -Next, we need to specify in the code which function map to the rte_acl_create
> -symbol at which versions. First, at the site of the initial symbol definition,
> -we need to update the function so that it is uniquely named, and not in conflict
> -with the public symbol name
> -
> -.. code-block:: c
> -
> - struct rte_acl_ctx *
> - -rte_acl_create(const struct rte_acl_param *param)
> - +rte_acl_create_v20(const struct rte_acl_param *param)
> - {
> - size_t sz;
> - struct rte_acl_ctx *ctx;
> - ...
> -
> -Note that the base name of the symbol was kept intact, as this is conducive to
> -the macros used for versioning symbols. That is our next step, mapping this new
> -symbol name to the initial symbol name at version node 2.0. Immediately after
> -the function, we add this line of code
> -
> -.. code-block:: c
> -
> - VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
> -
> -Remembering to also add the rte_compat.h header to the requisite c file where
> -these changes are being made. The above macro instructs the linker to create a
> -new symbol ``rte_acl_create@DPDK_2.0``, which matches the symbol created in older
> -builds, but now points to the above newly named function. We have now mapped
> -the original rte_acl_create symbol to the original function (but with a new
> -name)
> -
> -Next, we need to create the 2.1 version of the symbol. We create a new function
> -name, with a different suffix, and implement it appropriately
> -
> -.. code-block:: c
> -
> - struct rte_acl_ctx *
> - rte_acl_create_v21(const struct rte_acl_param *param, int debug);
> - {
> - struct rte_acl_ctx *ctx = rte_acl_create_v20(param);
> -
> - ctx->debug = debug;
> -
> - return ctx;
> - }
> -
> -This code serves as our new API call. Its the same as our old call, but adds
> -the new parameter in place. Next we need to map this function to the symbol
> -``rte_acl_create@DPDK_2.1``. To do this, we modify the public prototype of the call
> -in the header file, adding the macro there to inform all including applications,
> -that on re-link, the default rte_acl_create symbol should point to this
> -function. Note that we could do this by simply naming the function above
> -rte_acl_create, and the linker would chose the most recent version tag to apply
> -in the version script, but we can also do this in the header file
> -
> -.. code-block:: c
> -
> - struct rte_acl_ctx *
> - -rte_acl_create(const struct rte_acl_param *param);
> - +rte_acl_create(const struct rte_acl_param *param, int debug);
> - +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
> -
> -The BIND_DEFAULT_SYMBOL macro explicitly tells applications that include this
> -header, to link to the rte_acl_create_v21 function and apply the DPDK_2.1
> -version node to it. This method is more explicit and flexible than just
> -re-implementing the exact symbol name, and allows for other features (such as
> -linking to the old symbol version by default, when the new ABI is to be opt-in
> -for a period.
> -
> -One last thing we need to do. Note that we've taken what was a public symbol,
> -and duplicated it into two uniquely and differently named symbols. We've then
> -mapped each of those back to the public symbol ``rte_acl_create`` with different
> -version tags. This only applies to dynamic linking, as static linking has no
> -notion of versioning. That leaves this code in a position of no longer having a
> -symbol simply named ``rte_acl_create`` and a static build will fail on that
> -missing symbol.
> -
> -To correct this, we can simply map a function of our choosing back to the public
> -symbol in the static build with the ``MAP_STATIC_SYMBOL`` macro. Generally the
> -assumption is that the most recent version of the symbol is the one you want to
> -map. So, back in the C file where, immediately after ``rte_acl_create_v21`` is
> -defined, we add this
> -
> -.. code-block:: c
> -
> - struct rte_acl_ctx *
> - rte_acl_create_v21(const struct rte_acl_param *param, int debug)
> - {
> - ...
> - }
> - MAP_STATIC_SYMBOL(struct rte_acl_ctx *rte_acl_create(const struct rte_acl_param *param, int debug), rte_acl_create_v21);
> -
> -That tells the compiler that, when building a static library, any calls to the
> -symbol ``rte_acl_create`` should be linked to ``rte_acl_create_v21``
> -
> -That's it, on the next shared library rebuild, there will be two versions of
> -rte_acl_create, an old DPDK_2.0 version, used by previously built applications,
> -and a new DPDK_2.1 version, used by future built applications.
> -
> -
> -Deprecating part of a public API
> -________________________________
> -
> -Lets assume that you've done the above update, and after a few releases have
> -passed you decide you would like to retire the old version of the function.
> -After having gone through the ABI deprecation announcement process, removal is
> -easy. Start by removing the symbol from the requisite version map file:
> -
> -.. code-block:: none
> -
> - DPDK_2.0 {
> - global:
> -
> - rte_acl_add_rules;
> - rte_acl_build;
> - rte_acl_classify;
> - rte_acl_classify_alg;
> - rte_acl_classify_scalar;
> - rte_acl_dump;
> - - rte_acl_create
> - rte_acl_find_existing;
> - rte_acl_free;
> - rte_acl_ipv4vlan_add_rules;
> - rte_acl_ipv4vlan_build;
> - rte_acl_list_dump;
> - rte_acl_reset;
> - rte_acl_reset_rules;
> - rte_acl_set_ctx_classify;
> -
> - local: *;
> - };
> -
> - DPDK_2.1 {
> - global:
> - rte_acl_create;
> - } DPDK_2.0;
> -
> -
> -Next remove the corresponding versioned export.
> -
> -.. code-block:: c
> -
> - -VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
> -
> -
> -Note that the internal function definition could also be removed, but its used
> -in our example by the newer version _v21, so we leave it in place. This is a
> -coding style choice.
> -
> -Lastly, we need to bump the LIBABIVER number for this library in the Makefile to
> -indicate to applications doing dynamic linking that this is a later, and
> -possibly incompatible library version:
> -
> -.. code-block:: c
> -
> - -LIBABIVER := 1
> - +LIBABIVER := 2
> -
> -Deprecating an entire ABI version
> -_________________________________
> -
> -While removing a symbol from and ABI may be useful, it is often more practical
> -to remove an entire version node at once. If a version node completely
> -specifies an API, then removing part of it, typically makes it incomplete. In
> -those cases it is better to remove the entire node
> -
> -To do this, start by modifying the version map file, such that all symbols from
> -the node to be removed are merged into the next node in the map
> -
> -In the case of our map above, it would transform to look as follows
> -
> -.. code-block:: none
> -
> - DPDK_2.1 {
> - global:
> -
> - rte_acl_add_rules;
> - rte_acl_build;
> - rte_acl_classify;
> - rte_acl_classify_alg;
> - rte_acl_classify_scalar;
> - rte_acl_dump;
> - rte_acl_create
> - rte_acl_find_existing;
> - rte_acl_free;
> - rte_acl_ipv4vlan_add_rules;
> - rte_acl_ipv4vlan_build;
> - rte_acl_list_dump;
> - rte_acl_reset;
> - rte_acl_reset_rules;
> - rte_acl_set_ctx_classify;
> -
> - local: *;
> - };
> -
> -Then any uses of BIND_DEFAULT_SYMBOL that pointed to the old node should be
> -updated to point to the new version node in any header files for all affected
> -symbols.
> -
> -.. code-block:: c
> -
> - -BIND_DEFAULT_SYMBOL(rte_acl_create, _v20, 2.0);
> - +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
> -
> -Lastly, any VERSION_SYMBOL macros that point to the old version node should be
> -removed, taking care to keep, where need old code in place to support newer
> -versions of the symbol.
> -
> -
> -Running the ABI Validator
> --------------------------
> -
> -The ``devtools`` directory in the DPDK source tree contains a utility program,
> -``validate-abi.sh``, for validating the DPDK ABI based on the Linux `ABI
> -Compliance Checker
> -<http://ispras.linuxbase.org/index.php/ABI_compliance_checker>`_.
> -
> -This has a dependency on the ``abi-compliance-checker`` and ``and abi-dumper``
> -utilities which can be installed via a package manager. For example::
> -
> - sudo yum install abi-compliance-checker
> - sudo yum install abi-dumper
> -
> -The syntax of the ``validate-abi.sh`` utility is::
> -
> - ./devtools/validate-abi.sh <REV1> <REV2>
> -
> -Where ``REV1`` and ``REV2`` are valid gitrevisions(7)
> -https://www.kernel.org/pub/software/scm/git/docs/gitrevisions.html
> -on the local repo.
> -
> -For example::
> -
> - # Check between the previous and latest commit:
> - ./devtools/validate-abi.sh HEAD~1 HEAD
> -
> - # Check on a specific compilation target:
> - ./devtools/validate-abi.sh -t x86_64-native-linux-gcc HEAD~1 HEAD
> -
> - # Check between two tags:
> - ./devtools/validate-abi.sh v2.0.0 v2.1.0
> -
> - # Check between git master and local topic-branch "vhost-hacking":
> - ./devtools/validate-abi.sh master vhost-hacking
> -
> -After the validation script completes (it can take a while since it need to
> -compile both tags) it will create compatibility reports in the
> -``./abi-check/compat_report`` directory. Listed incompatibilities can be found
> -as follows::
> -
> - grep -lr Incompatible abi-check/compat_reports/
> --
> 2.7.4
>
>
^ permalink raw reply [relevance 5%]
* [dpdk-dev] [PATCH v4 4/4] doc: add maintainer for abi policy
2019-09-25 10:23 10% [dpdk-dev] [PATCH v4 0/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
` (2 preceding siblings ...)
2019-09-25 10:23 30% ` [dpdk-dev] [PATCH v4 3/4] doc: updates to versioning guide for " Ray Kinsella
@ 2019-09-25 10:23 13% ` Ray Kinsella
3 siblings, 0 replies; 200+ results
From: Ray Kinsella @ 2019-09-25 10:23 UTC (permalink / raw)
To: dev
Cc: mdr, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, nhorman,
maxime.coquelin, john.mcnamara, marko.kovacevic, hemant.agrawal,
ktraynor
Add an entry to the maintainer file for the abi policy.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
MAINTAINERS | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/MAINTAINERS b/MAINTAINERS
index b3d9aad..d231f03 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -80,6 +80,10 @@ M: Marko Kovacevic <marko.kovacevic@intel.com>
F: README
F: doc/
+ABI Policy
+M: Ray Kinsella <mdr@ashroe.eu>
+F: doc/guides/contributing/abi_*.rst
+
Developers and Maintainers Tools
M: Thomas Monjalon <thomas@monjalon.net>
F: MAINTAINERS
--
2.7.4
^ permalink raw reply [relevance 13%]
* [dpdk-dev] [PATCH v4 3/4] doc: updates to versioning guide for abi versions
2019-09-25 10:23 10% [dpdk-dev] [PATCH v4 0/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
2019-09-25 10:23 13% ` [dpdk-dev] [PATCH v4 1/4] doc: separate versioning.rst into version and policy Ray Kinsella
2019-09-25 10:23 31% ` [dpdk-dev] [PATCH v4 2/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
@ 2019-09-25 10:23 30% ` Ray Kinsella
2019-09-25 10:23 13% ` [dpdk-dev] [PATCH v4 4/4] doc: add maintainer for abi policy Ray Kinsella
3 siblings, 0 replies; 200+ results
From: Ray Kinsella @ 2019-09-25 10:23 UTC (permalink / raw)
To: dev
Cc: mdr, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, nhorman,
maxime.coquelin, john.mcnamara, marko.kovacevic, hemant.agrawal,
ktraynor
Updates to the ABI versioning guide, to account for the changes to the DPDK
ABI/API policy. Fixes for references to abi versioning and policy guides.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
doc/guides/contributing/abi_versioning.rst | 248 +++++++++++++++++++----------
doc/guides/contributing/patches.rst | 6 +-
doc/guides/rel_notes/deprecation.rst | 2 +-
3 files changed, 172 insertions(+), 84 deletions(-)
diff --git a/doc/guides/contributing/abi_versioning.rst b/doc/guides/contributing/abi_versioning.rst
index 53e6ac0..76c63c8 100644
--- a/doc/guides/contributing/abi_versioning.rst
+++ b/doc/guides/contributing/abi_versioning.rst
@@ -1,44 +1,134 @@
.. SPDX-License-Identifier: BSD-3-Clause
Copyright 2018 The DPDK contributors
-.. library_versioning:
+.. _abi_versioning:
-Library versioning
+ABI Versioning
+==============
+
+This document details the mechanics of ABI version management in DPDK.
+
+.. _what_is_soname:
+
+What is a library's soname?
+---------------------------
+
+System libraries usually adopt the familiar major and minor version naming
+convention, where major versions (e.g. ``librte_eal 20.x, 21.x``) are presumed
+to be ABI incompatible with each other and minor versions (e.g. ``librte_eal
+20.1, 20.2``) are presumed to be ABI compatible. A library's `soname
+<https://en.wikipedia.org/wiki/Soname>`_. is typically used to provide backward
+compatibility information about a given library, describing the lowest common
+denominator ABI supported by the library. The soname or logical name for the
+library, is typically comprised of the library's name and major version e.g.
+``librte_eal.so.20``.
+
+During an application's build process, a library's soname is noted as a runtime
+dependency of the application. This information is then used by the `dynamic
+linker <https://en.wikipedia.org/wiki/Dynamic_linker>`_ when resolving the
+applications dependencies at runtime, to load a library supporting the correct
+ABI version. The library loaded at runtime therefore, may be a minor revision
+supporting the same major ABI version (e.g. ``librte_eal.20.2``), as the library
+used to link the application (e.g ``librte_eal.20.0``).
+
+.. _major_abi_versions:
+
+Major ABI versions
------------------
-Downstreams might want to provide different DPDK releases at the same time to
-support multiple consumers of DPDK linked against older and newer sonames.
+An ABI version change to a given library, especially in core libraries such as
+``librte_mbuf``, may cause an implicit ripple effect on the ABI of it's
+consuming libraries, causing ABI breakages. There may however be no explicit
+reason to bump a dependent library's ABI version, as there may have been no
+obvious change to the dependent library's API, even though the library's ABI
+compatibility will have been broken.
+
+This interdependence of DPDK libraries, means that ABI versioning of libraries
+is more manageable at a project level, with all project libraries sharing a
+**single ABI version**. In addition, the need to maintain a stable ABI for some
+number of releases as described in the section :ref:`abi_policy`, means
+that ABI version increments need to carefully planned and managed at a project
+level.
+
+Major ABI versions are therefore declared typically aligned with an LTS release
+and is then supported some number of subsequent releases, shared across all
+libraries. This means that a single project level ABI version, reflected in all
+individual library's soname, library filenames and associated version maps
+persists over multiple releases.
+
+.. code-block:: none
+
+ $ head ./lib/librte_acl/rte_acl_version.map
+ DPDK_20 {
+ global:
+ ...
-Also due to the interdependencies that DPDK libraries can have applications
-might end up with an executable space in which multiple versions of a library
-are mapped by ld.so.
+ $ head ./lib/librte_eal/rte_eal_version.map
+ DPDK_20 {
+ global:
+ ...
-Think of LibA that got an ABI bump and LibB that did not get an ABI bump but is
-depending on LibA.
+When an ABI change is made between major ABI versions to a given library, a new
+section is added to that library's version map describing the impending new ABI
+version, as described in the section :ref:`example_abi_macro_usage`. The
+library's soname and filename however do not change, e.g. ``libacl.so.20``, as
+ABI compatibility with the last major ABI version continues to be preserved for
+that library.
-.. note::
+.. code-block:: none
- Application
- \-> LibA.old
- \-> LibB.new -> LibA.new
+ $ head ./lib/librte_acl/rte_acl_version.map
+ DPDK_20 {
+ global:
+ ...
-That is a conflict which can be avoided by setting ``CONFIG_RTE_MAJOR_ABI``.
-If set, the value of ``CONFIG_RTE_MAJOR_ABI`` overwrites all - otherwise per
-library - versions defined in the libraries ``LIBABIVER``.
-An example might be ``CONFIG_RTE_MAJOR_ABI=16.11`` which will make all libraries
-``librte<?>.so.16.11`` instead of ``librte<?>.so.<LIBABIVER>``.
+ DPDK_21 {
+ global:
+
+ } DPDK_20;
+ ...
+ $ head ./lib/librte_eal/rte_eal_version.map
+ DPDK_20 {
+ global:
+ ...
+
+However when a new ABI version is declared, for example DPDK ``21``, old
+depreciated functions may be safely removed at this point and the entire old
+major ABI version removed, see the section :ref:`deprecating_entire_abi` on
+how this may be done.
+
+.. code-block:: none
+
+ $ head ./lib/librte_acl/rte_acl_version.map
+ DPDK_21 {
+ global:
+ ...
+
+ $ head ./lib/librte_eal/rte_eal_version.map
+ DPDK_21 {
+ global:
+ ...
+
+At the same time, the major ABI version is changed atomically across all
+libraries by incrementing the major version in individual library's soname, e.g.
+``libacl.so.21``. This is done by bumping the LIBABIVER number in the libraries
+Makefile to indicate to dynamic linking applications that this is a later, and
+possibly incompatible library version:
+
+.. code-block:: c
+
+ -LIBABIVER := 20
+ +LIBABIVER := 21
-ABI versioning
---------------
Versioning Macros
-~~~~~~~~~~~~~~~~~
+-----------------
When a symbol is exported from a library to provide an API, it also provides a
calling convention (ABI) that is embodied in its name, return type and
arguments. Occasionally that function may need to change to accommodate new
-functionality or behavior. When that occurs, it is desirable to allow for
+functionality or behavior. When that occurs, it is may be required to allow for
backward compatibility for a time with older binaries that are dynamically
linked to the DPDK.
@@ -61,8 +151,10 @@ The macros exported are:
fully qualified function ``p``, so that if a symbol becomes versioned, it
can still be mapped back to the public symbol name.
+.. _example_abi_macro_usage:
+
Examples of ABI Macro use
-^^^^^^^^^^^^^^^^^^^^^^^^^
+~~~~~~~~~~~~~~~~~~~~~~~~~
Updating a public API
_____________________
@@ -106,16 +198,16 @@ maintain previous ABI versions that are accessible only to previously compiled
binaries
The addition of a parameter to the function is ABI breaking as the function is
-public, and existing application may use it in its current form. However, the
+public, and existing application may use it in its current form. However, the
compatibility macros in DPDK allow a developer to use symbol versioning so that
multiple functions can be mapped to the same public symbol based on when an
-application was linked to it. To see how this is done, we start with the
-requisite libraries version map file. Initially the version map file for the
-acl library looks like this
+application was linked to it. To see how this is done, we start with the
+requisite libraries version map file. Initially the version map file for the acl
+library looks like this
.. code-block:: none
- DPDK_2.0 {
+ DPDK_20 {
global:
rte_acl_add_rules;
@@ -141,7 +233,7 @@ This file needs to be modified as follows
.. code-block:: none
- DPDK_2.0 {
+ DPDK_20 {
global:
rte_acl_add_rules;
@@ -163,16 +255,16 @@ This file needs to be modified as follows
local: *;
};
- DPDK_2.1 {
+ DPDK_21 {
global:
rte_acl_create;
- } DPDK_2.0;
+ } DPDK_20;
The addition of the new block tells the linker that a new version node is
-available (DPDK_2.1), which contains the symbol rte_acl_create, and inherits the
-symbols from the DPDK_2.0 node. This list is directly translated into a list of
-exported symbols when DPDK is compiled as a shared library
+available (DPDK_21), which contains the symbol rte_acl_create, and inherits
+the symbols from the DPDK_20 node. This list is directly translated into a
+list of exported symbols when DPDK is compiled as a shared library
Next, we need to specify in the code which function map to the rte_acl_create
symbol at which versions. First, at the site of the initial symbol definition,
@@ -191,22 +283,22 @@ with the public symbol name
Note that the base name of the symbol was kept intact, as this is conducive to
the macros used for versioning symbols. That is our next step, mapping this new
-symbol name to the initial symbol name at version node 2.0. Immediately after
+symbol name to the initial symbol name at version node 20. Immediately after
the function, we add this line of code
.. code-block:: c
- VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
+ VERSION_SYMBOL(rte_acl_create, _v20, 20);
Remembering to also add the rte_compat.h header to the requisite c file where
-these changes are being made. The above macro instructs the linker to create a
-new symbol ``rte_acl_create@DPDK_2.0``, which matches the symbol created in older
-builds, but now points to the above newly named function. We have now mapped
-the original rte_acl_create symbol to the original function (but with a new
-name)
+these changes are being made. The above macro instructs the linker to create a
+new symbol ``rte_acl_create@DPDK_20``, which matches the symbol created in
+older builds, but now points to the above newly named function. We have now
+mapped the original rte_acl_create symbol to the original function (but with a
+new name)
-Next, we need to create the 2.1 version of the symbol. We create a new function
-name, with a different suffix, and implement it appropriately
+Next, we need to create the 21 version of the symbol. We create a new function
+name, with a different suffix, and implement it appropriately
.. code-block:: c
@@ -220,12 +312,12 @@ name, with a different suffix, and implement it appropriately
return ctx;
}
-This code serves as our new API call. Its the same as our old call, but adds
-the new parameter in place. Next we need to map this function to the symbol
-``rte_acl_create@DPDK_2.1``. To do this, we modify the public prototype of the call
-in the header file, adding the macro there to inform all including applications,
-that on re-link, the default rte_acl_create symbol should point to this
-function. Note that we could do this by simply naming the function above
+This code serves as our new API call. Its the same as our old call, but adds the
+new parameter in place. Next we need to map this function to the symbol
+``rte_acl_create@DPDK_21``. To do this, we modify the public prototype of the
+call in the header file, adding the macro there to inform all including
+applications, that on re-link, the default rte_acl_create symbol should point to
+this function. Note that we could do this by simply naming the function above
rte_acl_create, and the linker would chose the most recent version tag to apply
in the version script, but we can also do this in the header file
@@ -233,11 +325,11 @@ in the version script, but we can also do this in the header file
struct rte_acl_ctx *
-rte_acl_create(const struct rte_acl_param *param);
- +rte_acl_create(const struct rte_acl_param *param, int debug);
- +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
+ +rte_acl_create_v21(const struct rte_acl_param *param, int debug);
+ +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 21);
The BIND_DEFAULT_SYMBOL macro explicitly tells applications that include this
-header, to link to the rte_acl_create_v21 function and apply the DPDK_2.1
+header, to link to the rte_acl_create_v21 function and apply the DPDK_21
version node to it. This method is more explicit and flexible than just
re-implementing the exact symbol name, and allows for other features (such as
linking to the old symbol version by default, when the new ABI is to be opt-in
@@ -257,6 +349,7 @@ assumption is that the most recent version of the symbol is the one you want to
map. So, back in the C file where, immediately after ``rte_acl_create_v21`` is
defined, we add this
+
.. code-block:: c
struct rte_acl_ctx *
@@ -270,21 +363,22 @@ That tells the compiler that, when building a static library, any calls to the
symbol ``rte_acl_create`` should be linked to ``rte_acl_create_v21``
That's it, on the next shared library rebuild, there will be two versions of
-rte_acl_create, an old DPDK_2.0 version, used by previously built applications,
-and a new DPDK_2.1 version, used by future built applications.
+rte_acl_create, an old DPDK_20 version, used by previously built applications,
+and a new DPDK_21 version, used by future built applications.
Deprecating part of a public API
________________________________
-Lets assume that you've done the above update, and after a few releases have
-passed you decide you would like to retire the old version of the function.
-After having gone through the ABI deprecation announcement process, removal is
-easy. Start by removing the symbol from the requisite version map file:
+Lets assume that you've done the above update, and in preparation for the next
+major ABI version you decide you would like to retire the old version of the
+function. After having gone through the ABI deprecation announcement process,
+removal is easy. Start by removing the symbol from the requisite version map
+file:
.. code-block:: none
- DPDK_2.0 {
+ DPDK_20 {
global:
rte_acl_add_rules;
@@ -306,48 +400,42 @@ easy. Start by removing the symbol from the requisite version map file:
local: *;
};
- DPDK_2.1 {
+ DPDK_21 {
global:
rte_acl_create;
- } DPDK_2.0;
+ } DPDK_20;
Next remove the corresponding versioned export.
.. code-block:: c
- -VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
+ -VERSION_SYMBOL(rte_acl_create, _v20, 20);
Note that the internal function definition could also be removed, but its used
-in our example by the newer version _v21, so we leave it in place. This is a
-coding style choice.
-
-Lastly, we need to bump the LIBABIVER number for this library in the Makefile to
-indicate to applications doing dynamic linking that this is a later, and
-possibly incompatible library version:
-
-.. code-block:: c
+in our example by the newer version v21, so we leave it in place and declare it
+as static. This is a coding style choice.
- -LIBABIVER := 1
- +LIBABIVER := 2
+.. _deprecating_entire_abi:
Deprecating an entire ABI version
_________________________________
-While removing a symbol from and ABI may be useful, it is often more practical
-to remove an entire version node at once. If a version node completely
-specifies an API, then removing part of it, typically makes it incomplete. In
-those cases it is better to remove the entire node
+While removing a symbol from an ABI may be useful, it is more practical to
+remove an entire version node at once, as is typically done at the declaration
+of a major ABI version. If a version node completely specifies an API, then
+removing part of it, typically makes it incomplete. In those cases it is better
+to remove the entire node.
To do this, start by modifying the version map file, such that all symbols from
-the node to be removed are merged into the next node in the map
+the node to be removed are merged into the next node in the map.
In the case of our map above, it would transform to look as follows
.. code-block:: none
- DPDK_2.1 {
+ DPDK_21 {
global:
rte_acl_add_rules;
@@ -375,8 +463,8 @@ symbols.
.. code-block:: c
- -BIND_DEFAULT_SYMBOL(rte_acl_create, _v20, 2.0);
- +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
+ -BIND_DEFAULT_SYMBOL(rte_acl_create, _v20, 20);
+ +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 21);
Lastly, any VERSION_SYMBOL macros that point to the old version node should be
removed, taking care to keep, where need old code in place to support newer
diff --git a/doc/guides/contributing/patches.rst b/doc/guides/contributing/patches.rst
index 9e1013b..d63c9f5 100644
--- a/doc/guides/contributing/patches.rst
+++ b/doc/guides/contributing/patches.rst
@@ -156,9 +156,9 @@ Make your planned changes in the cloned ``dpdk`` repo. Here are some guidelines
* For other PMDs and more info, refer to the ``MAINTAINERS`` file.
-* New external functions should be added to the local ``version.map`` file.
- See the :doc:`Guidelines for ABI policy and versioning </contributing/versioning>`.
- New external functions should also be added in alphabetical order.
+* New external functions should be added to the local ``version.map`` file. See
+ the :ref:`ABI policy <abi_policy>` and :ref:`ABI versioning <abi_versioning>`
+ guides. New external functions should also be added in alphabetical order.
* Important changes will require an addition to the release notes in ``doc/guides/rel_notes/``.
See the :ref:`Release Notes section of the Documentation Guidelines <doc_guidelines>` for details.
diff --git a/doc/guides/rel_notes/deprecation.rst b/doc/guides/rel_notes/deprecation.rst
index 0ee8533..2e163a5 100644
--- a/doc/guides/rel_notes/deprecation.rst
+++ b/doc/guides/rel_notes/deprecation.rst
@@ -4,7 +4,7 @@
ABI and API Deprecation
=======================
-See the :doc:`guidelines document for details of the ABI policy </contributing/versioning>`.
+See the :ref:`guidelines document for details of the ABI policy <abi_policy>`.
API and ABI deprecation notices are to be posted here.
--
2.7.4
^ permalink raw reply [relevance 30%]
* [dpdk-dev] [PATCH v4 1/4] doc: separate versioning.rst into version and policy
2019-09-25 10:23 10% [dpdk-dev] [PATCH v4 0/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
@ 2019-09-25 10:23 13% ` Ray Kinsella
2019-09-25 12:24 5% ` Neil Horman
2019-09-27 15:43 4% ` Aaron Conole
2019-09-25 10:23 31% ` [dpdk-dev] [PATCH v4 2/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
` (2 subsequent siblings)
3 siblings, 2 replies; 200+ results
From: Ray Kinsella @ 2019-09-25 10:23 UTC (permalink / raw)
To: dev
Cc: mdr, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, nhorman,
maxime.coquelin, john.mcnamara, marko.kovacevic, hemant.agrawal,
ktraynor
Separate versioning.rst into abi versioning and abi policy guidance, in
preparation for adding more detail to the abi policy.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
doc/guides/contributing/abi_policy.rst | 169 +++++++++
doc/guides/contributing/abi_versioning.rst | 427 +++++++++++++++++++++
doc/guides/contributing/index.rst | 3 +-
doc/guides/contributing/versioning.rst | 591 -----------------------------
4 files changed, 598 insertions(+), 592 deletions(-)
create mode 100644 doc/guides/contributing/abi_policy.rst
create mode 100644 doc/guides/contributing/abi_versioning.rst
delete mode 100644 doc/guides/contributing/versioning.rst
diff --git a/doc/guides/contributing/abi_policy.rst b/doc/guides/contributing/abi_policy.rst
new file mode 100644
index 0000000..55bacb4
--- /dev/null
+++ b/doc/guides/contributing/abi_policy.rst
@@ -0,0 +1,169 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+ Copyright 2018 The DPDK contributors
+
+.. abi_api_policy:
+
+DPDK ABI/API policy
+===================
+
+Description
+-----------
+
+This document details some methods for handling ABI management in the DPDK.
+
+General Guidelines
+------------------
+
+#. Whenever possible, ABI should be preserved
+#. ABI/API may be changed with a deprecation process
+#. The modification of symbols can generally be managed with versioning
+#. Libraries or APIs marked in ``experimental`` state may change without constraint
+#. New APIs will be marked as ``experimental`` for at least one release to allow
+ any issues found by users of the new API to be fixed quickly
+#. The addition of symbols is generally not problematic
+#. The removal of symbols generally is an ABI break and requires bumping of the
+ LIBABIVER macro
+#. Updates to the minimum hardware requirements, which drop support for hardware which
+ was previously supported, should be treated as an ABI change.
+
+What is an ABI
+~~~~~~~~~~~~~~
+
+An ABI (Application Binary Interface) is the set of runtime interfaces exposed
+by a library. It is similar to an API (Application Programming Interface) but
+is the result of compilation. It is also effectively cloned when applications
+link to dynamic libraries. That is to say when an application is compiled to
+link against dynamic libraries, it is assumed that the ABI remains constant
+between the time the application is compiled/linked, and the time that it runs.
+Therefore, in the case of dynamic linking, it is critical that an ABI is
+preserved, or (when modified), done in such a way that the application is unable
+to behave improperly or in an unexpected fashion.
+
+
+ABI/API Deprecation
+-------------------
+
+The DPDK ABI policy
+~~~~~~~~~~~~~~~~~~~
+
+ABI versions are set at the time of major release labeling, and the ABI may
+change multiple times, without warning, between the last release label and the
+HEAD label of the git tree.
+
+ABI versions, once released, are available until such time as their
+deprecation has been noted in the Release Notes for at least one major release
+cycle. For example consider the case where the ABI for DPDK 2.0 has been
+shipped and then a decision is made to modify it during the development of
+DPDK 2.1. The decision will be recorded in the Release Notes for the DPDK 2.1
+release and the modification will be made available in the DPDK 2.2 release.
+
+ABI versions may be deprecated in whole or in part as needed by a given
+update.
+
+Some ABI changes may be too significant to reasonably maintain multiple
+versions. In those cases ABI's may be updated without backward compatibility
+being provided. The requirements for doing so are:
+
+#. At least 3 acknowledgments of the need to do so must be made on the
+ dpdk.org mailing list.
+
+ - The acknowledgment of the maintainer of the component is mandatory, or if
+ no maintainer is available for the component, the tree/sub-tree maintainer
+ for that component must acknowledge the ABI change instead.
+
+ - It is also recommended that acknowledgments from different "areas of
+ interest" be sought for each deprecation, for example: from NIC vendors,
+ CPU vendors, end-users, etc.
+
+#. The changes (including an alternative map file) can be included with
+ deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option,
+ to provide more details about oncoming changes.
+ ``RTE_NEXT_ABI`` wrapper will be removed when it become the default ABI.
+ More preferred way to provide this information is sending the feature
+ as a separate patch and reference it in deprecation notice.
+
+#. A full deprecation cycle, as explained above, must be made to offer
+ downstream consumers sufficient warning of the change.
+
+Note that the above process for ABI deprecation should not be undertaken
+lightly. ABI stability is extremely important for downstream consumers of the
+DPDK, especially when distributed in shared object form. Every effort should
+be made to preserve the ABI whenever possible. The ABI should only be changed
+for significant reasons, such as performance enhancements. ABI breakage due to
+changes such as reorganizing public structure fields for aesthetic or
+readability purposes should be avoided.
+
+.. note::
+
+ Updates to the minimum hardware requirements, which drop support for hardware
+ which was previously supported, should be treated as an ABI change, and
+ follow the relevant deprecation policy procedures as above: 3 acks and
+ announcement at least one release in advance.
+
+Examples of Deprecation Notices
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The following are some examples of ABI deprecation notices which would be
+added to the Release Notes:
+
+* The Macro ``#RTE_FOO`` is deprecated and will be removed with version 2.0,
+ to be replaced with the inline function ``rte_foo()``.
+
+* The function ``rte_mbuf_grok()`` has been updated to include a new parameter
+ in version 2.0. Backwards compatibility will be maintained for this function
+ until the release of version 2.1
+
+* The members of ``struct rte_foo`` have been reorganized in release 2.0 for
+ performance reasons. Existing binary applications will have backwards
+ compatibility in release 2.0, while newly built binaries will need to
+ reference the new structure variant ``struct rte_foo2``. Compatibility will
+ be removed in release 2.2, and all applications will require updating and
+ rebuilding to the new structure at that time, which will be renamed to the
+ original ``struct rte_foo``.
+
+* Significant ABI changes are planned for the ``librte_dostuff`` library. The
+ upcoming release 2.0 will not contain these changes, but release 2.1 will,
+ and no backwards compatibility is planned due to the extensive nature of
+ these changes. Binaries using this library built prior to version 2.1 will
+ require updating and recompilation.
+
+New API replacing previous one
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If a new API proposed functionally replaces an existing one, when the new API
+becomes non-experimental then the old one is marked with ``__rte_deprecated``.
+Deprecated APIs are removed completely just after the next LTS.
+
+Reminder that old API should follow deprecation process to be removed.
+
+
+Experimental APIs
+-----------------
+
+APIs marked as ``experimental`` are not considered part of the ABI and may
+change without warning at any time. Since changes to APIs are most likely
+immediately after their introduction, as users begin to take advantage of
+those new APIs and start finding issues with them, new DPDK APIs will be
+automatically marked as ``experimental`` to allow for a period of stabilization
+before they become part of a tracked ABI.
+
+Note that marking an API as experimental is a multi step process.
+To mark an API as experimental, the symbols which are desired to be exported
+must be placed in an EXPERIMENTAL version block in the corresponding libraries'
+version map script.
+Secondly, the corresponding prototypes of those exported functions (in the
+development header files), must be marked with the ``__rte_experimental`` tag
+(see ``rte_compat.h``).
+The DPDK build makefiles perform a check to ensure that the map file and the
+C code reflect the same list of symbols.
+This check can be circumvented by defining ``ALLOW_EXPERIMENTAL_API``
+during compilation in the corresponding library Makefile.
+
+In addition to tagging the code with ``__rte_experimental``,
+the doxygen markup must also contain the EXPERIMENTAL string,
+and the MAINTAINERS file should note the EXPERIMENTAL libraries.
+
+For removing the experimental tag associated with an API, deprecation notice
+is not required. Though, an API should remain in experimental state for at least
+one release. Thereafter, normal process of posting patch for review to mailing
+list can be followed.
diff --git a/doc/guides/contributing/abi_versioning.rst b/doc/guides/contributing/abi_versioning.rst
new file mode 100644
index 0000000..53e6ac0
--- /dev/null
+++ b/doc/guides/contributing/abi_versioning.rst
@@ -0,0 +1,427 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+ Copyright 2018 The DPDK contributors
+
+.. library_versioning:
+
+Library versioning
+------------------
+
+Downstreams might want to provide different DPDK releases at the same time to
+support multiple consumers of DPDK linked against older and newer sonames.
+
+Also due to the interdependencies that DPDK libraries can have applications
+might end up with an executable space in which multiple versions of a library
+are mapped by ld.so.
+
+Think of LibA that got an ABI bump and LibB that did not get an ABI bump but is
+depending on LibA.
+
+.. note::
+
+ Application
+ \-> LibA.old
+ \-> LibB.new -> LibA.new
+
+That is a conflict which can be avoided by setting ``CONFIG_RTE_MAJOR_ABI``.
+If set, the value of ``CONFIG_RTE_MAJOR_ABI`` overwrites all - otherwise per
+library - versions defined in the libraries ``LIBABIVER``.
+An example might be ``CONFIG_RTE_MAJOR_ABI=16.11`` which will make all libraries
+``librte<?>.so.16.11`` instead of ``librte<?>.so.<LIBABIVER>``.
+
+
+ABI versioning
+--------------
+
+Versioning Macros
+~~~~~~~~~~~~~~~~~
+
+When a symbol is exported from a library to provide an API, it also provides a
+calling convention (ABI) that is embodied in its name, return type and
+arguments. Occasionally that function may need to change to accommodate new
+functionality or behavior. When that occurs, it is desirable to allow for
+backward compatibility for a time with older binaries that are dynamically
+linked to the DPDK.
+
+To support backward compatibility the ``rte_compat.h``
+header file provides macros to use when updating exported functions. These
+macros are used in conjunction with the ``rte_<library>_version.map`` file for
+a given library to allow multiple versions of a symbol to exist in a shared
+library so that older binaries need not be immediately recompiled.
+
+The macros exported are:
+
+* ``VERSION_SYMBOL(b, e, n)``: Creates a symbol version table entry binding
+ versioned symbol ``b@DPDK_n`` to the internal function ``b_e``.
+
+* ``BIND_DEFAULT_SYMBOL(b, e, n)``: Creates a symbol version entry instructing
+ the linker to bind references to symbol ``b`` to the internal symbol
+ ``b_e``.
+
+* ``MAP_STATIC_SYMBOL(f, p)``: Declare the prototype ``f``, and map it to the
+ fully qualified function ``p``, so that if a symbol becomes versioned, it
+ can still be mapped back to the public symbol name.
+
+Examples of ABI Macro use
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Updating a public API
+_____________________
+
+Assume we have a function as follows
+
+.. code-block:: c
+
+ /*
+ * Create an acl context object for apps to
+ * manipulate
+ */
+ struct rte_acl_ctx *
+ rte_acl_create(const struct rte_acl_param *param)
+ {
+ ...
+ }
+
+
+Assume that struct rte_acl_ctx is a private structure, and that a developer
+wishes to enhance the acl api so that a debugging flag can be enabled on a
+per-context basis. This requires an addition to the structure (which, being
+private, is safe), but it also requires modifying the code as follows
+
+.. code-block:: c
+
+ /*
+ * Create an acl context object for apps to
+ * manipulate
+ */
+ struct rte_acl_ctx *
+ rte_acl_create(const struct rte_acl_param *param, int debug)
+ {
+ ...
+ }
+
+
+Note also that, being a public function, the header file prototype must also be
+changed, as must all the call sites, to reflect the new ABI footprint. We will
+maintain previous ABI versions that are accessible only to previously compiled
+binaries
+
+The addition of a parameter to the function is ABI breaking as the function is
+public, and existing application may use it in its current form. However, the
+compatibility macros in DPDK allow a developer to use symbol versioning so that
+multiple functions can be mapped to the same public symbol based on when an
+application was linked to it. To see how this is done, we start with the
+requisite libraries version map file. Initially the version map file for the
+acl library looks like this
+
+.. code-block:: none
+
+ DPDK_2.0 {
+ global:
+
+ rte_acl_add_rules;
+ rte_acl_build;
+ rte_acl_classify;
+ rte_acl_classify_alg;
+ rte_acl_classify_scalar;
+ rte_acl_create;
+ rte_acl_dump;
+ rte_acl_find_existing;
+ rte_acl_free;
+ rte_acl_ipv4vlan_add_rules;
+ rte_acl_ipv4vlan_build;
+ rte_acl_list_dump;
+ rte_acl_reset;
+ rte_acl_reset_rules;
+ rte_acl_set_ctx_classify;
+
+ local: *;
+ };
+
+This file needs to be modified as follows
+
+.. code-block:: none
+
+ DPDK_2.0 {
+ global:
+
+ rte_acl_add_rules;
+ rte_acl_build;
+ rte_acl_classify;
+ rte_acl_classify_alg;
+ rte_acl_classify_scalar;
+ rte_acl_create;
+ rte_acl_dump;
+ rte_acl_find_existing;
+ rte_acl_free;
+ rte_acl_ipv4vlan_add_rules;
+ rte_acl_ipv4vlan_build;
+ rte_acl_list_dump;
+ rte_acl_reset;
+ rte_acl_reset_rules;
+ rte_acl_set_ctx_classify;
+
+ local: *;
+ };
+
+ DPDK_2.1 {
+ global:
+ rte_acl_create;
+
+ } DPDK_2.0;
+
+The addition of the new block tells the linker that a new version node is
+available (DPDK_2.1), which contains the symbol rte_acl_create, and inherits the
+symbols from the DPDK_2.0 node. This list is directly translated into a list of
+exported symbols when DPDK is compiled as a shared library
+
+Next, we need to specify in the code which function map to the rte_acl_create
+symbol at which versions. First, at the site of the initial symbol definition,
+we need to update the function so that it is uniquely named, and not in conflict
+with the public symbol name
+
+.. code-block:: c
+
+ struct rte_acl_ctx *
+ -rte_acl_create(const struct rte_acl_param *param)
+ +rte_acl_create_v20(const struct rte_acl_param *param)
+ {
+ size_t sz;
+ struct rte_acl_ctx *ctx;
+ ...
+
+Note that the base name of the symbol was kept intact, as this is conducive to
+the macros used for versioning symbols. That is our next step, mapping this new
+symbol name to the initial symbol name at version node 2.0. Immediately after
+the function, we add this line of code
+
+.. code-block:: c
+
+ VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
+
+Remembering to also add the rte_compat.h header to the requisite c file where
+these changes are being made. The above macro instructs the linker to create a
+new symbol ``rte_acl_create@DPDK_2.0``, which matches the symbol created in older
+builds, but now points to the above newly named function. We have now mapped
+the original rte_acl_create symbol to the original function (but with a new
+name)
+
+Next, we need to create the 2.1 version of the symbol. We create a new function
+name, with a different suffix, and implement it appropriately
+
+.. code-block:: c
+
+ struct rte_acl_ctx *
+ rte_acl_create_v21(const struct rte_acl_param *param, int debug);
+ {
+ struct rte_acl_ctx *ctx = rte_acl_create_v20(param);
+
+ ctx->debug = debug;
+
+ return ctx;
+ }
+
+This code serves as our new API call. Its the same as our old call, but adds
+the new parameter in place. Next we need to map this function to the symbol
+``rte_acl_create@DPDK_2.1``. To do this, we modify the public prototype of the call
+in the header file, adding the macro there to inform all including applications,
+that on re-link, the default rte_acl_create symbol should point to this
+function. Note that we could do this by simply naming the function above
+rte_acl_create, and the linker would chose the most recent version tag to apply
+in the version script, but we can also do this in the header file
+
+.. code-block:: c
+
+ struct rte_acl_ctx *
+ -rte_acl_create(const struct rte_acl_param *param);
+ +rte_acl_create(const struct rte_acl_param *param, int debug);
+ +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
+
+The BIND_DEFAULT_SYMBOL macro explicitly tells applications that include this
+header, to link to the rte_acl_create_v21 function and apply the DPDK_2.1
+version node to it. This method is more explicit and flexible than just
+re-implementing the exact symbol name, and allows for other features (such as
+linking to the old symbol version by default, when the new ABI is to be opt-in
+for a period.
+
+One last thing we need to do. Note that we've taken what was a public symbol,
+and duplicated it into two uniquely and differently named symbols. We've then
+mapped each of those back to the public symbol ``rte_acl_create`` with different
+version tags. This only applies to dynamic linking, as static linking has no
+notion of versioning. That leaves this code in a position of no longer having a
+symbol simply named ``rte_acl_create`` and a static build will fail on that
+missing symbol.
+
+To correct this, we can simply map a function of our choosing back to the public
+symbol in the static build with the ``MAP_STATIC_SYMBOL`` macro. Generally the
+assumption is that the most recent version of the symbol is the one you want to
+map. So, back in the C file where, immediately after ``rte_acl_create_v21`` is
+defined, we add this
+
+.. code-block:: c
+
+ struct rte_acl_ctx *
+ rte_acl_create_v21(const struct rte_acl_param *param, int debug)
+ {
+ ...
+ }
+ MAP_STATIC_SYMBOL(struct rte_acl_ctx *rte_acl_create(const struct rte_acl_param *param, int debug), rte_acl_create_v21);
+
+That tells the compiler that, when building a static library, any calls to the
+symbol ``rte_acl_create`` should be linked to ``rte_acl_create_v21``
+
+That's it, on the next shared library rebuild, there will be two versions of
+rte_acl_create, an old DPDK_2.0 version, used by previously built applications,
+and a new DPDK_2.1 version, used by future built applications.
+
+
+Deprecating part of a public API
+________________________________
+
+Lets assume that you've done the above update, and after a few releases have
+passed you decide you would like to retire the old version of the function.
+After having gone through the ABI deprecation announcement process, removal is
+easy. Start by removing the symbol from the requisite version map file:
+
+.. code-block:: none
+
+ DPDK_2.0 {
+ global:
+
+ rte_acl_add_rules;
+ rte_acl_build;
+ rte_acl_classify;
+ rte_acl_classify_alg;
+ rte_acl_classify_scalar;
+ rte_acl_dump;
+ - rte_acl_create
+ rte_acl_find_existing;
+ rte_acl_free;
+ rte_acl_ipv4vlan_add_rules;
+ rte_acl_ipv4vlan_build;
+ rte_acl_list_dump;
+ rte_acl_reset;
+ rte_acl_reset_rules;
+ rte_acl_set_ctx_classify;
+
+ local: *;
+ };
+
+ DPDK_2.1 {
+ global:
+ rte_acl_create;
+ } DPDK_2.0;
+
+
+Next remove the corresponding versioned export.
+
+.. code-block:: c
+
+ -VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
+
+
+Note that the internal function definition could also be removed, but its used
+in our example by the newer version _v21, so we leave it in place. This is a
+coding style choice.
+
+Lastly, we need to bump the LIBABIVER number for this library in the Makefile to
+indicate to applications doing dynamic linking that this is a later, and
+possibly incompatible library version:
+
+.. code-block:: c
+
+ -LIBABIVER := 1
+ +LIBABIVER := 2
+
+Deprecating an entire ABI version
+_________________________________
+
+While removing a symbol from and ABI may be useful, it is often more practical
+to remove an entire version node at once. If a version node completely
+specifies an API, then removing part of it, typically makes it incomplete. In
+those cases it is better to remove the entire node
+
+To do this, start by modifying the version map file, such that all symbols from
+the node to be removed are merged into the next node in the map
+
+In the case of our map above, it would transform to look as follows
+
+.. code-block:: none
+
+ DPDK_2.1 {
+ global:
+
+ rte_acl_add_rules;
+ rte_acl_build;
+ rte_acl_classify;
+ rte_acl_classify_alg;
+ rte_acl_classify_scalar;
+ rte_acl_dump;
+ rte_acl_create
+ rte_acl_find_existing;
+ rte_acl_free;
+ rte_acl_ipv4vlan_add_rules;
+ rte_acl_ipv4vlan_build;
+ rte_acl_list_dump;
+ rte_acl_reset;
+ rte_acl_reset_rules;
+ rte_acl_set_ctx_classify;
+
+ local: *;
+ };
+
+Then any uses of BIND_DEFAULT_SYMBOL that pointed to the old node should be
+updated to point to the new version node in any header files for all affected
+symbols.
+
+.. code-block:: c
+
+ -BIND_DEFAULT_SYMBOL(rte_acl_create, _v20, 2.0);
+ +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
+
+Lastly, any VERSION_SYMBOL macros that point to the old version node should be
+removed, taking care to keep, where need old code in place to support newer
+versions of the symbol.
+
+
+Running the ABI Validator
+-------------------------
+
+The ``devtools`` directory in the DPDK source tree contains a utility program,
+``validate-abi.sh``, for validating the DPDK ABI based on the Linux `ABI
+Compliance Checker
+<http://ispras.linuxbase.org/index.php/ABI_compliance_checker>`_.
+
+This has a dependency on the ``abi-compliance-checker`` and ``and abi-dumper``
+utilities which can be installed via a package manager. For example::
+
+ sudo yum install abi-compliance-checker
+ sudo yum install abi-dumper
+
+The syntax of the ``validate-abi.sh`` utility is::
+
+ ./devtools/validate-abi.sh <REV1> <REV2>
+
+Where ``REV1`` and ``REV2`` are valid gitrevisions(7)
+https://www.kernel.org/pub/software/scm/git/docs/gitrevisions.html
+on the local repo.
+
+For example::
+
+ # Check between the previous and latest commit:
+ ./devtools/validate-abi.sh HEAD~1 HEAD
+
+ # Check on a specific compilation target:
+ ./devtools/validate-abi.sh -t x86_64-native-linux-gcc HEAD~1 HEAD
+
+ # Check between two tags:
+ ./devtools/validate-abi.sh v2.0.0 v2.1.0
+
+ # Check between git master and local topic-branch "vhost-hacking":
+ ./devtools/validate-abi.sh master vhost-hacking
+
+After the validation script completes (it can take a while since it need to
+compile both tags) it will create compatibility reports in the
+``./abi-check/compat_report`` directory. Listed incompatibilities can be found
+as follows::
+
+ grep -lr Incompatible abi-check/compat_reports/
diff --git a/doc/guides/contributing/index.rst b/doc/guides/contributing/index.rst
index e2608d3..2fefd91 100644
--- a/doc/guides/contributing/index.rst
+++ b/doc/guides/contributing/index.rst
@@ -10,7 +10,8 @@ Contributor's Guidelines
coding_style
design
- versioning
+ abi_policy
+ abi_versioning
documentation
patches
vulnerability
diff --git a/doc/guides/contributing/versioning.rst b/doc/guides/contributing/versioning.rst
deleted file mode 100644
index 3ab2c43..0000000
--- a/doc/guides/contributing/versioning.rst
+++ /dev/null
@@ -1,591 +0,0 @@
-.. SPDX-License-Identifier: BSD-3-Clause
- Copyright 2018 The DPDK contributors
-
-DPDK ABI/API policy
-===================
-
-Description
------------
-
-This document details some methods for handling ABI management in the DPDK.
-
-General Guidelines
-------------------
-
-#. Whenever possible, ABI should be preserved
-#. ABI/API may be changed with a deprecation process
-#. The modification of symbols can generally be managed with versioning
-#. Libraries or APIs marked in ``experimental`` state may change without constraint
-#. New APIs will be marked as ``experimental`` for at least one release to allow
- any issues found by users of the new API to be fixed quickly
-#. The addition of symbols is generally not problematic
-#. The removal of symbols generally is an ABI break and requires bumping of the
- LIBABIVER macro
-#. Updates to the minimum hardware requirements, which drop support for hardware which
- was previously supported, should be treated as an ABI change.
-
-What is an ABI
-~~~~~~~~~~~~~~
-
-An ABI (Application Binary Interface) is the set of runtime interfaces exposed
-by a library. It is similar to an API (Application Programming Interface) but
-is the result of compilation. It is also effectively cloned when applications
-link to dynamic libraries. That is to say when an application is compiled to
-link against dynamic libraries, it is assumed that the ABI remains constant
-between the time the application is compiled/linked, and the time that it runs.
-Therefore, in the case of dynamic linking, it is critical that an ABI is
-preserved, or (when modified), done in such a way that the application is unable
-to behave improperly or in an unexpected fashion.
-
-
-ABI/API Deprecation
--------------------
-
-The DPDK ABI policy
-~~~~~~~~~~~~~~~~~~~
-
-ABI versions are set at the time of major release labeling, and the ABI may
-change multiple times, without warning, between the last release label and the
-HEAD label of the git tree.
-
-ABI versions, once released, are available until such time as their
-deprecation has been noted in the Release Notes for at least one major release
-cycle. For example consider the case where the ABI for DPDK 2.0 has been
-shipped and then a decision is made to modify it during the development of
-DPDK 2.1. The decision will be recorded in the Release Notes for the DPDK 2.1
-release and the modification will be made available in the DPDK 2.2 release.
-
-ABI versions may be deprecated in whole or in part as needed by a given
-update.
-
-Some ABI changes may be too significant to reasonably maintain multiple
-versions. In those cases ABI's may be updated without backward compatibility
-being provided. The requirements for doing so are:
-
-#. At least 3 acknowledgments of the need to do so must be made on the
- dpdk.org mailing list.
-
- - The acknowledgment of the maintainer of the component is mandatory, or if
- no maintainer is available for the component, the tree/sub-tree maintainer
- for that component must acknowledge the ABI change instead.
-
- - It is also recommended that acknowledgments from different "areas of
- interest" be sought for each deprecation, for example: from NIC vendors,
- CPU vendors, end-users, etc.
-
-#. The changes (including an alternative map file) can be included with
- deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option,
- to provide more details about oncoming changes.
- ``RTE_NEXT_ABI`` wrapper will be removed when it become the default ABI.
- More preferred way to provide this information is sending the feature
- as a separate patch and reference it in deprecation notice.
-
-#. A full deprecation cycle, as explained above, must be made to offer
- downstream consumers sufficient warning of the change.
-
-Note that the above process for ABI deprecation should not be undertaken
-lightly. ABI stability is extremely important for downstream consumers of the
-DPDK, especially when distributed in shared object form. Every effort should
-be made to preserve the ABI whenever possible. The ABI should only be changed
-for significant reasons, such as performance enhancements. ABI breakage due to
-changes such as reorganizing public structure fields for aesthetic or
-readability purposes should be avoided.
-
-.. note::
-
- Updates to the minimum hardware requirements, which drop support for hardware
- which was previously supported, should be treated as an ABI change, and
- follow the relevant deprecation policy procedures as above: 3 acks and
- announcement at least one release in advance.
-
-Examples of Deprecation Notices
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The following are some examples of ABI deprecation notices which would be
-added to the Release Notes:
-
-* The Macro ``#RTE_FOO`` is deprecated and will be removed with version 2.0,
- to be replaced with the inline function ``rte_foo()``.
-
-* The function ``rte_mbuf_grok()`` has been updated to include a new parameter
- in version 2.0. Backwards compatibility will be maintained for this function
- until the release of version 2.1
-
-* The members of ``struct rte_foo`` have been reorganized in release 2.0 for
- performance reasons. Existing binary applications will have backwards
- compatibility in release 2.0, while newly built binaries will need to
- reference the new structure variant ``struct rte_foo2``. Compatibility will
- be removed in release 2.2, and all applications will require updating and
- rebuilding to the new structure at that time, which will be renamed to the
- original ``struct rte_foo``.
-
-* Significant ABI changes are planned for the ``librte_dostuff`` library. The
- upcoming release 2.0 will not contain these changes, but release 2.1 will,
- and no backwards compatibility is planned due to the extensive nature of
- these changes. Binaries using this library built prior to version 2.1 will
- require updating and recompilation.
-
-New API replacing previous one
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-If a new API proposed functionally replaces an existing one, when the new API
-becomes non-experimental then the old one is marked with ``__rte_deprecated``.
-Deprecated APIs are removed completely just after the next LTS.
-
-Reminder that old API should follow deprecation process to be removed.
-
-
-Experimental APIs
------------------
-
-APIs marked as ``experimental`` are not considered part of the ABI and may
-change without warning at any time. Since changes to APIs are most likely
-immediately after their introduction, as users begin to take advantage of
-those new APIs and start finding issues with them, new DPDK APIs will be
-automatically marked as ``experimental`` to allow for a period of stabilization
-before they become part of a tracked ABI.
-
-Note that marking an API as experimental is a multi step process.
-To mark an API as experimental, the symbols which are desired to be exported
-must be placed in an EXPERIMENTAL version block in the corresponding libraries'
-version map script.
-Secondly, the corresponding prototypes of those exported functions (in the
-development header files), must be marked with the ``__rte_experimental`` tag
-(see ``rte_compat.h``).
-The DPDK build makefiles perform a check to ensure that the map file and the
-C code reflect the same list of symbols.
-This check can be circumvented by defining ``ALLOW_EXPERIMENTAL_API``
-during compilation in the corresponding library Makefile.
-
-In addition to tagging the code with ``__rte_experimental``,
-the doxygen markup must also contain the EXPERIMENTAL string,
-and the MAINTAINERS file should note the EXPERIMENTAL libraries.
-
-For removing the experimental tag associated with an API, deprecation notice
-is not required. Though, an API should remain in experimental state for at least
-one release. Thereafter, normal process of posting patch for review to mailing
-list can be followed.
-
-
-Library versioning
-------------------
-
-Downstreams might want to provide different DPDK releases at the same time to
-support multiple consumers of DPDK linked against older and newer sonames.
-
-Also due to the interdependencies that DPDK libraries can have applications
-might end up with an executable space in which multiple versions of a library
-are mapped by ld.so.
-
-Think of LibA that got an ABI bump and LibB that did not get an ABI bump but is
-depending on LibA.
-
-.. note::
-
- Application
- \-> LibA.old
- \-> LibB.new -> LibA.new
-
-That is a conflict which can be avoided by setting ``CONFIG_RTE_MAJOR_ABI``.
-If set, the value of ``CONFIG_RTE_MAJOR_ABI`` overwrites all - otherwise per
-library - versions defined in the libraries ``LIBABIVER``.
-An example might be ``CONFIG_RTE_MAJOR_ABI=16.11`` which will make all libraries
-``librte<?>.so.16.11`` instead of ``librte<?>.so.<LIBABIVER>``.
-
-
-ABI versioning
---------------
-
-Versioning Macros
-~~~~~~~~~~~~~~~~~
-
-When a symbol is exported from a library to provide an API, it also provides a
-calling convention (ABI) that is embodied in its name, return type and
-arguments. Occasionally that function may need to change to accommodate new
-functionality or behavior. When that occurs, it is desirable to allow for
-backward compatibility for a time with older binaries that are dynamically
-linked to the DPDK.
-
-To support backward compatibility the ``rte_compat.h``
-header file provides macros to use when updating exported functions. These
-macros are used in conjunction with the ``rte_<library>_version.map`` file for
-a given library to allow multiple versions of a symbol to exist in a shared
-library so that older binaries need not be immediately recompiled.
-
-The macros exported are:
-
-* ``VERSION_SYMBOL(b, e, n)``: Creates a symbol version table entry binding
- versioned symbol ``b@DPDK_n`` to the internal function ``b_e``.
-
-* ``BIND_DEFAULT_SYMBOL(b, e, n)``: Creates a symbol version entry instructing
- the linker to bind references to symbol ``b`` to the internal symbol
- ``b_e``.
-
-* ``MAP_STATIC_SYMBOL(f, p)``: Declare the prototype ``f``, and map it to the
- fully qualified function ``p``, so that if a symbol becomes versioned, it
- can still be mapped back to the public symbol name.
-
-Examples of ABI Macro use
-^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Updating a public API
-_____________________
-
-Assume we have a function as follows
-
-.. code-block:: c
-
- /*
- * Create an acl context object for apps to
- * manipulate
- */
- struct rte_acl_ctx *
- rte_acl_create(const struct rte_acl_param *param)
- {
- ...
- }
-
-
-Assume that struct rte_acl_ctx is a private structure, and that a developer
-wishes to enhance the acl api so that a debugging flag can be enabled on a
-per-context basis. This requires an addition to the structure (which, being
-private, is safe), but it also requires modifying the code as follows
-
-.. code-block:: c
-
- /*
- * Create an acl context object for apps to
- * manipulate
- */
- struct rte_acl_ctx *
- rte_acl_create(const struct rte_acl_param *param, int debug)
- {
- ...
- }
-
-
-Note also that, being a public function, the header file prototype must also be
-changed, as must all the call sites, to reflect the new ABI footprint. We will
-maintain previous ABI versions that are accessible only to previously compiled
-binaries
-
-The addition of a parameter to the function is ABI breaking as the function is
-public, and existing application may use it in its current form. However, the
-compatibility macros in DPDK allow a developer to use symbol versioning so that
-multiple functions can be mapped to the same public symbol based on when an
-application was linked to it. To see how this is done, we start with the
-requisite libraries version map file. Initially the version map file for the
-acl library looks like this
-
-.. code-block:: none
-
- DPDK_2.0 {
- global:
-
- rte_acl_add_rules;
- rte_acl_build;
- rte_acl_classify;
- rte_acl_classify_alg;
- rte_acl_classify_scalar;
- rte_acl_create;
- rte_acl_dump;
- rte_acl_find_existing;
- rte_acl_free;
- rte_acl_ipv4vlan_add_rules;
- rte_acl_ipv4vlan_build;
- rte_acl_list_dump;
- rte_acl_reset;
- rte_acl_reset_rules;
- rte_acl_set_ctx_classify;
-
- local: *;
- };
-
-This file needs to be modified as follows
-
-.. code-block:: none
-
- DPDK_2.0 {
- global:
-
- rte_acl_add_rules;
- rte_acl_build;
- rte_acl_classify;
- rte_acl_classify_alg;
- rte_acl_classify_scalar;
- rte_acl_create;
- rte_acl_dump;
- rte_acl_find_existing;
- rte_acl_free;
- rte_acl_ipv4vlan_add_rules;
- rte_acl_ipv4vlan_build;
- rte_acl_list_dump;
- rte_acl_reset;
- rte_acl_reset_rules;
- rte_acl_set_ctx_classify;
-
- local: *;
- };
-
- DPDK_2.1 {
- global:
- rte_acl_create;
-
- } DPDK_2.0;
-
-The addition of the new block tells the linker that a new version node is
-available (DPDK_2.1), which contains the symbol rte_acl_create, and inherits the
-symbols from the DPDK_2.0 node. This list is directly translated into a list of
-exported symbols when DPDK is compiled as a shared library
-
-Next, we need to specify in the code which function map to the rte_acl_create
-symbol at which versions. First, at the site of the initial symbol definition,
-we need to update the function so that it is uniquely named, and not in conflict
-with the public symbol name
-
-.. code-block:: c
-
- struct rte_acl_ctx *
- -rte_acl_create(const struct rte_acl_param *param)
- +rte_acl_create_v20(const struct rte_acl_param *param)
- {
- size_t sz;
- struct rte_acl_ctx *ctx;
- ...
-
-Note that the base name of the symbol was kept intact, as this is conducive to
-the macros used for versioning symbols. That is our next step, mapping this new
-symbol name to the initial symbol name at version node 2.0. Immediately after
-the function, we add this line of code
-
-.. code-block:: c
-
- VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
-
-Remembering to also add the rte_compat.h header to the requisite c file where
-these changes are being made. The above macro instructs the linker to create a
-new symbol ``rte_acl_create@DPDK_2.0``, which matches the symbol created in older
-builds, but now points to the above newly named function. We have now mapped
-the original rte_acl_create symbol to the original function (but with a new
-name)
-
-Next, we need to create the 2.1 version of the symbol. We create a new function
-name, with a different suffix, and implement it appropriately
-
-.. code-block:: c
-
- struct rte_acl_ctx *
- rte_acl_create_v21(const struct rte_acl_param *param, int debug);
- {
- struct rte_acl_ctx *ctx = rte_acl_create_v20(param);
-
- ctx->debug = debug;
-
- return ctx;
- }
-
-This code serves as our new API call. Its the same as our old call, but adds
-the new parameter in place. Next we need to map this function to the symbol
-``rte_acl_create@DPDK_2.1``. To do this, we modify the public prototype of the call
-in the header file, adding the macro there to inform all including applications,
-that on re-link, the default rte_acl_create symbol should point to this
-function. Note that we could do this by simply naming the function above
-rte_acl_create, and the linker would chose the most recent version tag to apply
-in the version script, but we can also do this in the header file
-
-.. code-block:: c
-
- struct rte_acl_ctx *
- -rte_acl_create(const struct rte_acl_param *param);
- +rte_acl_create(const struct rte_acl_param *param, int debug);
- +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
-
-The BIND_DEFAULT_SYMBOL macro explicitly tells applications that include this
-header, to link to the rte_acl_create_v21 function and apply the DPDK_2.1
-version node to it. This method is more explicit and flexible than just
-re-implementing the exact symbol name, and allows for other features (such as
-linking to the old symbol version by default, when the new ABI is to be opt-in
-for a period.
-
-One last thing we need to do. Note that we've taken what was a public symbol,
-and duplicated it into two uniquely and differently named symbols. We've then
-mapped each of those back to the public symbol ``rte_acl_create`` with different
-version tags. This only applies to dynamic linking, as static linking has no
-notion of versioning. That leaves this code in a position of no longer having a
-symbol simply named ``rte_acl_create`` and a static build will fail on that
-missing symbol.
-
-To correct this, we can simply map a function of our choosing back to the public
-symbol in the static build with the ``MAP_STATIC_SYMBOL`` macro. Generally the
-assumption is that the most recent version of the symbol is the one you want to
-map. So, back in the C file where, immediately after ``rte_acl_create_v21`` is
-defined, we add this
-
-.. code-block:: c
-
- struct rte_acl_ctx *
- rte_acl_create_v21(const struct rte_acl_param *param, int debug)
- {
- ...
- }
- MAP_STATIC_SYMBOL(struct rte_acl_ctx *rte_acl_create(const struct rte_acl_param *param, int debug), rte_acl_create_v21);
-
-That tells the compiler that, when building a static library, any calls to the
-symbol ``rte_acl_create`` should be linked to ``rte_acl_create_v21``
-
-That's it, on the next shared library rebuild, there will be two versions of
-rte_acl_create, an old DPDK_2.0 version, used by previously built applications,
-and a new DPDK_2.1 version, used by future built applications.
-
-
-Deprecating part of a public API
-________________________________
-
-Lets assume that you've done the above update, and after a few releases have
-passed you decide you would like to retire the old version of the function.
-After having gone through the ABI deprecation announcement process, removal is
-easy. Start by removing the symbol from the requisite version map file:
-
-.. code-block:: none
-
- DPDK_2.0 {
- global:
-
- rte_acl_add_rules;
- rte_acl_build;
- rte_acl_classify;
- rte_acl_classify_alg;
- rte_acl_classify_scalar;
- rte_acl_dump;
- - rte_acl_create
- rte_acl_find_existing;
- rte_acl_free;
- rte_acl_ipv4vlan_add_rules;
- rte_acl_ipv4vlan_build;
- rte_acl_list_dump;
- rte_acl_reset;
- rte_acl_reset_rules;
- rte_acl_set_ctx_classify;
-
- local: *;
- };
-
- DPDK_2.1 {
- global:
- rte_acl_create;
- } DPDK_2.0;
-
-
-Next remove the corresponding versioned export.
-
-.. code-block:: c
-
- -VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
-
-
-Note that the internal function definition could also be removed, but its used
-in our example by the newer version _v21, so we leave it in place. This is a
-coding style choice.
-
-Lastly, we need to bump the LIBABIVER number for this library in the Makefile to
-indicate to applications doing dynamic linking that this is a later, and
-possibly incompatible library version:
-
-.. code-block:: c
-
- -LIBABIVER := 1
- +LIBABIVER := 2
-
-Deprecating an entire ABI version
-_________________________________
-
-While removing a symbol from and ABI may be useful, it is often more practical
-to remove an entire version node at once. If a version node completely
-specifies an API, then removing part of it, typically makes it incomplete. In
-those cases it is better to remove the entire node
-
-To do this, start by modifying the version map file, such that all symbols from
-the node to be removed are merged into the next node in the map
-
-In the case of our map above, it would transform to look as follows
-
-.. code-block:: none
-
- DPDK_2.1 {
- global:
-
- rte_acl_add_rules;
- rte_acl_build;
- rte_acl_classify;
- rte_acl_classify_alg;
- rte_acl_classify_scalar;
- rte_acl_dump;
- rte_acl_create
- rte_acl_find_existing;
- rte_acl_free;
- rte_acl_ipv4vlan_add_rules;
- rte_acl_ipv4vlan_build;
- rte_acl_list_dump;
- rte_acl_reset;
- rte_acl_reset_rules;
- rte_acl_set_ctx_classify;
-
- local: *;
- };
-
-Then any uses of BIND_DEFAULT_SYMBOL that pointed to the old node should be
-updated to point to the new version node in any header files for all affected
-symbols.
-
-.. code-block:: c
-
- -BIND_DEFAULT_SYMBOL(rte_acl_create, _v20, 2.0);
- +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
-
-Lastly, any VERSION_SYMBOL macros that point to the old version node should be
-removed, taking care to keep, where need old code in place to support newer
-versions of the symbol.
-
-
-Running the ABI Validator
--------------------------
-
-The ``devtools`` directory in the DPDK source tree contains a utility program,
-``validate-abi.sh``, for validating the DPDK ABI based on the Linux `ABI
-Compliance Checker
-<http://ispras.linuxbase.org/index.php/ABI_compliance_checker>`_.
-
-This has a dependency on the ``abi-compliance-checker`` and ``and abi-dumper``
-utilities which can be installed via a package manager. For example::
-
- sudo yum install abi-compliance-checker
- sudo yum install abi-dumper
-
-The syntax of the ``validate-abi.sh`` utility is::
-
- ./devtools/validate-abi.sh <REV1> <REV2>
-
-Where ``REV1`` and ``REV2`` are valid gitrevisions(7)
-https://www.kernel.org/pub/software/scm/git/docs/gitrevisions.html
-on the local repo.
-
-For example::
-
- # Check between the previous and latest commit:
- ./devtools/validate-abi.sh HEAD~1 HEAD
-
- # Check on a specific compilation target:
- ./devtools/validate-abi.sh -t x86_64-native-linux-gcc HEAD~1 HEAD
-
- # Check between two tags:
- ./devtools/validate-abi.sh v2.0.0 v2.1.0
-
- # Check between git master and local topic-branch "vhost-hacking":
- ./devtools/validate-abi.sh master vhost-hacking
-
-After the validation script completes (it can take a while since it need to
-compile both tags) it will create compatibility reports in the
-``./abi-check/compat_report`` directory. Listed incompatibilities can be found
-as follows::
-
- grep -lr Incompatible abi-check/compat_reports/
--
2.7.4
^ permalink raw reply [relevance 13%]
* [dpdk-dev] [PATCH v4 2/4] doc: changes to abi policy introducing major abi versions
2019-09-25 10:23 10% [dpdk-dev] [PATCH v4 0/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
2019-09-25 10:23 13% ` [dpdk-dev] [PATCH v4 1/4] doc: separate versioning.rst into version and policy Ray Kinsella
@ 2019-09-25 10:23 31% ` Ray Kinsella
2019-09-25 10:23 30% ` [dpdk-dev] [PATCH v4 3/4] doc: updates to versioning guide for " Ray Kinsella
2019-09-25 10:23 13% ` [dpdk-dev] [PATCH v4 4/4] doc: add maintainer for abi policy Ray Kinsella
3 siblings, 0 replies; 200+ results
From: Ray Kinsella @ 2019-09-25 10:23 UTC (permalink / raw)
To: dev
Cc: mdr, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, nhorman,
maxime.coquelin, john.mcnamara, marko.kovacevic, hemant.agrawal,
ktraynor
This policy change introduces major ABI versions, these are
declared every year, typically aligned with the LTS release
and are supported by subsequent releases in the following year.
This change is intended to improve ABI stabilty for those projects
consuming DPDK.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
doc/guides/contributing/abi_policy.rst | 307 ++++++++++++++++++++++++---------
doc/guides/contributing/stable.rst | 12 +-
2 files changed, 227 insertions(+), 92 deletions(-)
diff --git a/doc/guides/contributing/abi_policy.rst b/doc/guides/contributing/abi_policy.rst
index 55bacb4..7446ec3 100644
--- a/doc/guides/contributing/abi_policy.rst
+++ b/doc/guides/contributing/abi_policy.rst
@@ -1,33 +1,46 @@
.. SPDX-License-Identifier: BSD-3-Clause
- Copyright 2018 The DPDK contributors
+ Copyright 2019 The DPDK contributors
-.. abi_api_policy:
+.. _abi_policy:
-DPDK ABI/API policy
-===================
+ABI Policy
+==========
Description
-----------
-This document details some methods for handling ABI management in the DPDK.
+This document details the management policy that ensures the long-term stability
+of the DPDK ABI and API.
General Guidelines
------------------
-#. Whenever possible, ABI should be preserved
-#. ABI/API may be changed with a deprecation process
-#. The modification of symbols can generally be managed with versioning
-#. Libraries or APIs marked in ``experimental`` state may change without constraint
-#. New APIs will be marked as ``experimental`` for at least one release to allow
- any issues found by users of the new API to be fixed quickly
-#. The addition of symbols is generally not problematic
-#. The removal of symbols generally is an ABI break and requires bumping of the
- LIBABIVER macro
-#. Updates to the minimum hardware requirements, which drop support for hardware which
- was previously supported, should be treated as an ABI change.
-
-What is an ABI
-~~~~~~~~~~~~~~
+#. Major ABI versions are declared every **year** and are then supported for one
+ year, typically aligned with the :ref:`LTS release <stable_lts_releases>`.
+#. The ABI version is managed at a project level in DPDK, with the ABI version
+ reflected in all :ref:`library's soname <what_is_soname>`.
+#. The ABI should be preserved and not changed lightly. ABI changes must follow
+ the outlined :ref:`deprecation process <abi_changes>`.
+#. The addition of symbols is generally not problematic. The modification of
+ symbols is managed with :ref:`ABI Versioning <abi_versioning>`.
+#. The removal of symbols is considered an :ref:`ABI breakage <abi_breakages>`,
+ once approved these will form part of the next ABI version.
+#. Libraries or APIs marked as :ref:`Experimental <experimental_apis>` are not
+ considered part of an ABI version and may change without constraint.
+#. Updates to the :ref:`minimum hardware requirements <hw_rqmts>`, which drop
+ support for hardware which was previously supported, should be treated as an
+ ABI change.
+
+.. note::
+
+ In 2019, the DPDK community stated it's intention to move to ABI stable
+ releases, over a number of release cycles. Beginning with maintaining ABI
+ stability through one year of DPDK releases starting from DPDK 19.11. This
+ policy will be reviewed in 2020, with intention of lengthening the stability
+ period.
+
+What is an ABI?
+~~~~~~~~~~~~~~~
An ABI (Application Binary Interface) is the set of runtime interfaces exposed
by a library. It is similar to an API (Application Programming Interface) but
@@ -39,30 +52,66 @@ Therefore, in the case of dynamic linking, it is critical that an ABI is
preserved, or (when modified), done in such a way that the application is unable
to behave improperly or in an unexpected fashion.
+What is an ABI version?
+~~~~~~~~~~~~~~~~~~~~~~~
-ABI/API Deprecation
--------------------
+An ABI version is an instance of a library's ABI at a specific release. Certain
+releases are considered by the community to be milestone releases, the yearly
+LTS for example. Supporting those milestone release's ABI for some number of
+subsequent releases is desirable to facilitate application upgrade. Those ABI
+version's aligned with milestones release are therefore called 'ABI major
+versions' and are supported for some number of releases.
+
+More details on major ABI version can be found in the :ref:`ABI versioning
+<major_abi_versions>` guide.
The DPDK ABI policy
-~~~~~~~~~~~~~~~~~~~
+-------------------
+
+A major ABI version is declared every year, aligned with that year's LTS
+release, e.g. v19.11. This ABI version is then supported for one year by all
+subsequent releases within that time period, until the next LTS release, e.g.
+v20.11.
+
+At the declaration of a major ABI version, major version numbers encoded in
+libraries soname's are bumped to indicate the new version, with the minor
+version reset to ``0``. An example would be ``librte_eal.so.20.3`` would become
+``librte_eal.so.21.0``.
+
+The ABI may then change multiple times, without warning, between the last major
+ABI version increment and the HEAD label of the git tree, with the condition
+that ABI compatibility with the major ABI version is preserved and therefore
+soname's do not change.
+
+Minor versions are incremented to indicate the release of a new ABI compatible
+DPDK release, typically the DPDK quarterly releases. An example of this, might
+be that ``librte_eal.so.20.1`` would indicate the first ABI compatible DPDK
+release, following the declaration of the new major ABI version ``20``.
-ABI versions are set at the time of major release labeling, and the ABI may
-change multiple times, without warning, between the last release label and the
-HEAD label of the git tree.
+ABI versions, are supported by each release until such time as the next major
+ABI version is declared. At that time, the deprecation of the previous major ABI
+version will be noted in the Release Notes with guidance on individual symbol
+depreciation and upgrade notes provided.
-ABI versions, once released, are available until such time as their
-deprecation has been noted in the Release Notes for at least one major release
-cycle. For example consider the case where the ABI for DPDK 2.0 has been
-shipped and then a decision is made to modify it during the development of
-DPDK 2.1. The decision will be recorded in the Release Notes for the DPDK 2.1
-release and the modification will be made available in the DPDK 2.2 release.
+.. _abi_changes:
-ABI versions may be deprecated in whole or in part as needed by a given
-update.
+ABI Changes
+~~~~~~~~~~~
-Some ABI changes may be too significant to reasonably maintain multiple
-versions. In those cases ABI's may be updated without backward compatibility
-being provided. The requirements for doing so are:
+The ABI may still change after the declaration of a major ABI version, that is
+new APIs may be still added or existing APIs may be modified.
+
+.. Warning::
+
+ Note that, this policy details the method by which the ABI may be changed,
+ with due regard to preserving compatibility and observing depreciation
+ notices. This process however should not be undertaken lightly, as a general
+ rule ABI stability is extremely important for downstream consumers of DPDK.
+ The ABI should only be changed for significant reasons, such as performance
+ enhancements. ABI breakages due to changes such as reorganizing public
+ structure fields for aesthetic or readability purposes should be avoided.
+
+The requirements for changing the ABI are:
#. At least 3 acknowledgments of the need to do so must be made on the
dpdk.org mailing list.
@@ -71,34 +120,119 @@ being provided. The requirements for doing so are:
no maintainer is available for the component, the tree/sub-tree maintainer
for that component must acknowledge the ABI change instead.
+ - The acknowledgment of a member of the technical board, as a delegate of the
+ `technical board <https://core.dpdk.org/techboard/>`_ acknowledging the
+ need for the ABI change, is also mandatory.
+
- It is also recommended that acknowledgments from different "areas of
interest" be sought for each deprecation, for example: from NIC vendors,
CPU vendors, end-users, etc.
-#. The changes (including an alternative map file) can be included with
- deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option,
- to provide more details about oncoming changes.
- ``RTE_NEXT_ABI`` wrapper will be removed when it become the default ABI.
- More preferred way to provide this information is sending the feature
- as a separate patch and reference it in deprecation notice.
+#. Backward compatibility with the major ABI version must be maintained through
+ :ref:`abi_versioning`, with :ref:`forward-only <forward-only>` compatibility
+ offered for any ABI changes that are indicated to be part of the next ABI
+ version.
-#. A full deprecation cycle, as explained above, must be made to offer
- downstream consumers sufficient warning of the change.
+ - In situations where backward compatibility is not possible, read the
+ section on :ref:`abi_breakages`.
-Note that the above process for ABI deprecation should not be undertaken
-lightly. ABI stability is extremely important for downstream consumers of the
-DPDK, especially when distributed in shared object form. Every effort should
-be made to preserve the ABI whenever possible. The ABI should only be changed
-for significant reasons, such as performance enhancements. ABI breakage due to
-changes such as reorganizing public structure fields for aesthetic or
-readability purposes should be avoided.
+ - No backward or forward compatibility is offered for API changes marked as
+ ``experimental``, as described in the section on :ref:`Experimental APIs
+ and Libraries <experimental_apis>`.
-.. note::
+#. If a newly proposed API functionally replaces an existing one, when the new
+ API becomes non-experimental, then the old one is marked with
+ ``__rte_deprecated``.
+
+ - The depreciated API should follow the notification process to be removed,
+ see :ref:`deprecation_notices`.
+
+ - At the declaration of the next major ABI version, those ABI changes then
+ become a formal part of the new ABI and the requirement to preserve ABI
+ compatibility with the last major ABI version is then dropped.
+
+ - The responsibility for removing redundant ABI compatibility code rests
+ with the original contributor of the ABI changes, failing that, then with
+ the contributor's company and then finally with the maintainer.
+
+.. _forward-only:
+
+.. Note::
+
+ Note that forward-only compatibility is offered for those changes made
+ between major ABI versions. As a library's soname can only describe
+ compatibility with the last major ABI version, until the next major ABI
+ version is declared, these changes therefore cannot be resolved as a runtime
+ dependency through the soname. Therefore any application wishing to make use
+ of these ABI changes can only ensure that it's runtime dependencies are met
+ through Operating System package versioning.
+
+.. _hw_rqmts:
+
+.. Note::
Updates to the minimum hardware requirements, which drop support for hardware
which was previously supported, should be treated as an ABI change, and
- follow the relevant deprecation policy procedures as above: 3 acks and
- announcement at least one release in advance.
+ follow the relevant deprecation policy procedures as above: 3 acks, technical
+ board approval and announcement at least one release in advance.
+
+.. _abi_breakages:
+
+ABI Breakages
+~~~~~~~~~~~~~
+
+For those ABI changes that are too significant to reasonably maintain multiple
+symbol versions, there is an amended process. In these cases, ABIs may be
+updated without the requirement of backward compatibility being provided. These
+changes must follow the `same process :ref:`described above <abi_changes>` as non-breaking
+changes, however with the following additional requirements:
+
+#. ABI breaking changes (including an alternative map file) can be included with
+ deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option, to provide
+ more details about oncoming changes. ``RTE_NEXT_ABI`` wrapper will be removed
+ at the declaration of the next major ABI version.
+
+#. Once approved, and after the depreciation notice has been observed these
+ changes will form part of the next declared major ABI version.
+
+Examples of ABI Changes
+~~~~~~~~~~~~~~~~~~~~~~~
+
+The following are examples of allowable ABI changes occurring between
+declarations of major ABI versions.
+
+* DPDK 19.11 release, defines the function ``rte_foo()``, and ``rte_foo()``
+ as part of the major ABI version ``20``.
+
+* DPDK 20.02 release defines a new function ``rte_foo(uint8_t bar)``, and
+ this is not a problem as long as the symbol ``rte_foo@DPDK20`` is
+ preserved through :ref:`abi_versioning`.
+
+ - The new function may be marked with the ``__rte_experimental`` tag for a
+ number of releases, as described in the section :ref:`experimental_apis`.
+
+ - Once ``rte_foo(uint8_t bar)`` becomes non-experimental ``rte_foo()`` is then
+ declared as ``__rte_depreciated``, with an associated deprecation notice
+ provided.
+
+* DPDK 19.11 is not re-released to include ``rte_foo(uint8_t bar)``, the new
+ version of ``rte_foo`` only exists from DPDK 20.02 onwards as described in the
+ :ref:`note on forward-only compatibility<forward-only>`.
+
+* DPDK 20.02 release defines the experimental function ``__rte_experimental
+ rte_baz()``. This function may or may not exist in the DPDK 20.05 release.
+
+* An application ``dPacket`` wishes to use ``rte_foo(uint8_t bar)``, before the
+ declaration of the DPDK ``21`` major API version. The application can only
+ ensure it's runtime dependencies are met by specifying ``DPDK (>= 20.2)`` as
+ an explicit package dependency, as the soname only may only indicate the
+ supported major ABI version.
+
+* At the release of DPDK 20.11, the function ``rte_foo(uint8_t bar)`` becomes
+ formally part of then new major ABI version DPDK 21.0 and ``rte_foo()`` may be
+ removed.
+
+.. _deprecation_notices:
Examples of Deprecation Notices
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -106,46 +240,42 @@ Examples of Deprecation Notices
The following are some examples of ABI deprecation notices which would be
added to the Release Notes:
-* The Macro ``#RTE_FOO`` is deprecated and will be removed with version 2.0,
- to be replaced with the inline function ``rte_foo()``.
+* The Macro ``#RTE_FOO`` is deprecated and will be removed with ABI version
+ 21, to be replaced with the inline function ``rte_foo()``.
* The function ``rte_mbuf_grok()`` has been updated to include a new parameter
- in version 2.0. Backwards compatibility will be maintained for this function
- until the release of version 2.1
+ in version 20.2. Backwards compatibility will be maintained for this function
+ until the release of the new DPDK major ABI version 21, in DPDK version
+ 20.11.
-* The members of ``struct rte_foo`` have been reorganized in release 2.0 for
+* The members of ``struct rte_foo`` have been reorganized in DPDK 20.02 for
performance reasons. Existing binary applications will have backwards
- compatibility in release 2.0, while newly built binaries will need to
- reference the new structure variant ``struct rte_foo2``. Compatibility will
- be removed in release 2.2, and all applications will require updating and
+ compatibility in release 20.02, while newly built binaries will need to
+ reference the new structure variant ``struct rte_foo2``. Compatibility will be
+ removed in release 20.11, and all applications will require updating and
rebuilding to the new structure at that time, which will be renamed to the
original ``struct rte_foo``.
* Significant ABI changes are planned for the ``librte_dostuff`` library. The
- upcoming release 2.0 will not contain these changes, but release 2.1 will,
+ upcoming release 20.02 will not contain these changes, but release 20.11 will,
and no backwards compatibility is planned due to the extensive nature of
- these changes. Binaries using this library built prior to version 2.1 will
+ these changes. Binaries using this library built prior to ABI version 21 will
require updating and recompilation.
-New API replacing previous one
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-If a new API proposed functionally replaces an existing one, when the new API
-becomes non-experimental then the old one is marked with ``__rte_deprecated``.
-Deprecated APIs are removed completely just after the next LTS.
-
-Reminder that old API should follow deprecation process to be removed.
+.. _experimental_apis:
+Experimental
+------------
-Experimental APIs
------------------
+APIs
+~~~~
-APIs marked as ``experimental`` are not considered part of the ABI and may
-change without warning at any time. Since changes to APIs are most likely
-immediately after their introduction, as users begin to take advantage of
-those new APIs and start finding issues with them, new DPDK APIs will be
-automatically marked as ``experimental`` to allow for a period of stabilization
-before they become part of a tracked ABI.
+APIs marked as ``experimental`` are not considered part of an ABI version and
+may change without warning at any time. Since changes to APIs are most likely
+immediately after their introduction, as users begin to take advantage of those
+new APIs and start finding issues with them, new DPDK APIs will be automatically
+marked as ``experimental`` to allow for a period of stabilization before they
+become part of a tracked ABI version.
Note that marking an API as experimental is a multi step process.
To mark an API as experimental, the symbols which are desired to be exported
@@ -163,7 +293,16 @@ In addition to tagging the code with ``__rte_experimental``,
the doxygen markup must also contain the EXPERIMENTAL string,
and the MAINTAINERS file should note the EXPERIMENTAL libraries.
-For removing the experimental tag associated with an API, deprecation notice
-is not required. Though, an API should remain in experimental state for at least
-one release. Thereafter, normal process of posting patch for review to mailing
-list can be followed.
+For removing the experimental tag associated with an API, deprecation notice is
+not required. Though, an API should remain in experimental state for at least
+one release. Thereafter, the normal process of posting patch for review to
+mailing list can be followed.
+
+Libraries
+~~~~~~~~~
+
+Libraries marked as ``experimental`` are entirely not considered part of an ABI
+version, and may change without warning at any time. Experimental libraries
+always have a major version of ``0`` to indicate they exist outside of
+:ref:`abi_versioning` , with the minor version incremented with each ABI change
+to library.
diff --git a/doc/guides/contributing/stable.rst b/doc/guides/contributing/stable.rst
index 6a5eee9..2b563d4 100644
--- a/doc/guides/contributing/stable.rst
+++ b/doc/guides/contributing/stable.rst
@@ -1,7 +1,7 @@
.. SPDX-License-Identifier: BSD-3-Clause
Copyright 2018 The DPDK contributors
-.. stable_lts_releases:
+.. _stable_lts_releases:
DPDK Stable Releases and Long Term Support
==========================================
@@ -53,6 +53,9 @@ year's November (X.11) release will be maintained as an LTS for 2 years.
After the X.11 release, an LTS branch will be created for it at
http://git.dpdk.org/dpdk-stable where bugfixes will be backported to.
+A LTS release may align with the declaration of a new major ABI version,
+please read the :ref:`abi_policy` for more information.
+
It is anticipated that there will be at least 4 releases per year of the LTS
or approximately 1 every 3 months. However, the cadence can be shorter or
longer depending on the number and criticality of the backported
@@ -119,10 +122,3 @@ A Stable Release will be released by:
list.
Stable releases are available on the `dpdk.org download page <http://core.dpdk.org/download/>`_.
-
-
-ABI
----
-
-The Stable Release should not be seen as a way of breaking or circumventing
-the DPDK ABI policy.
--
2.7.4
^ permalink raw reply [relevance 31%]
* [dpdk-dev] [PATCH v4 0/4] doc: changes to abi policy introducing major abi versions
@ 2019-09-25 10:23 10% Ray Kinsella
2019-09-25 10:23 13% ` [dpdk-dev] [PATCH v4 1/4] doc: separate versioning.rst into version and policy Ray Kinsella
` (3 more replies)
0 siblings, 4 replies; 200+ results
From: Ray Kinsella @ 2019-09-25 10:23 UTC (permalink / raw)
To: dev
Cc: mdr, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, nhorman,
maxime.coquelin, john.mcnamara, marko.kovacevic, hemant.agrawal,
ktraynor
TL;DR abbreviation:
A major ABI version that all DPDK releases during a one year period
support. ABI versioning is managed at a project-level, in place of library-level
management. ABI changes to add new features are permitted, as long as ABI
compatibility with the major ABI version is maintained.
Detail:
This patch introduces major ABI versions, supported for one year and released
aligned with the LTS release. This ABI version is then supported by all
subsequent releases within that one year period. The intention is that the one
year support period, will then be reviewed after the initial year with the
intention of lengthing the support period for the next ABI version.
ABI changes that preserve ABI compatibility with the major ABI version are
permitted in subsequent releases. ABI changes, follow similar approval rules as
before with the additional gate of now requiring technical board approval. The
merging and release of ABI breaking changes would now be pushed to the
declaration of the next major ABI version.
This change encourages developers to maintain ABI compatibility with the major
ABI version, by promoting a permissive culture around those changes that
preserve ABI compatibility. This approach begins to align DPDK with those
projects that declare major ABI versions (e.g. version 2.x, 3.x) and support
those versions for some period, typically two years or more.
To provide an example of how this might work in practice:
* DPDK v20 is declared as the supported ABI version for one year, aligned with
the DPDK v19.11 (LTS) release. All library sonames are updated to reflect the
new ABI version, e.g. librte_eal.so.20, librte_acl.so.20...
* DPDK v20.02 .. v20.08 releases are ABI compatible with the DPDK v20 ABI. ABI
changes are permitted from DPDK v20.02 onwards, with the condition that ABI
compatibility with DPDK v20 is preserved.
* DPDK v21 is declared as the new supported ABI version for two years, aligned
with the DPDK v20.11 (LTS) release. The DPDK v20 ABI is now depreciated,
library sonames are updated to v21 and ABI compatibility breaking changes may
be introduced.
v4
* Removed changes to stable.rst, fixed typos and clarified the ABI policy
"warning".
v3
* Added myself as the maintainer of the ABI policy.
* Updated the policy and versioning guides to use the year of the LTS+1 (e.g.
v20), as the abi major version number.
v2
* Restructured the patch into 3 patches:
1. Splits the original versioning document into an ABI policy document
and ABI versioning document.
2. Add changes to the policy document introducing major ABI versions.
3. Fixes up the versioning document in light of major ABI versioning.
* Reduces the initial ABI stability from two years to one year, with a review
after the first year.
* Adds detail around ABI version handling for experimental libraries.
* Adds detail around chain of responsility for removing deprecated symbols.
Ray Kinsella (4):
doc: separate versioning.rst into version and policy
doc: changes to abi policy introducing major abi versions
doc: updates to versioning guide for abi versions
doc: add maintainer for abi policy
MAINTAINERS | 4 +
doc/guides/contributing/abi_policy.rst | 308 +++++++++++++++
doc/guides/contributing/abi_versioning.rst | 515 +++++++++++++++++++++++++
doc/guides/contributing/index.rst | 3 +-
doc/guides/contributing/patches.rst | 6 +-
doc/guides/contributing/stable.rst | 12 +-
doc/guides/contributing/versioning.rst | 591 -----------------------------
doc/guides/rel_notes/deprecation.rst | 2 +-
8 files changed, 837 insertions(+), 604 deletions(-)
create mode 100644 doc/guides/contributing/abi_policy.rst
create mode 100644 doc/guides/contributing/abi_versioning.rst
delete mode 100644 doc/guides/contributing/versioning.rst
--
2.7.4
^ permalink raw reply [relevance 10%]
* [dpdk-dev] [PATCH v6] eal: make lcore_config private
@ 2019-09-24 17:39 3% Stephen Hemminger
0 siblings, 0 replies; 200+ results
From: Stephen Hemminger @ 2019-09-24 17:39 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger
The internal structure of lcore_config is no longer be part of
visible API/ABI. Make it private to EAL.
Rearrange and resize the fields in the structure so it takes
less memory (and cache footprint).
Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
v6 - rebase for 19.11
follows my earlier efforts at hiding lcore_config
which were then taken up by David Marchand.
lib/librte_eal/common/eal_common_launch.c | 2 ++
lib/librte_eal/common/eal_private.h | 22 +++++++++++++++++++++
lib/librte_eal/common/include/rte_lcore.h | 24 -----------------------
lib/librte_eal/common/rte_service.c | 2 ++
lib/librte_eal/rte_eal_version.map | 1 -
5 files changed, 26 insertions(+), 25 deletions(-)
diff --git a/lib/librte_eal/common/eal_common_launch.c b/lib/librte_eal/common/eal_common_launch.c
index fe0ba3f0d617..cf52d717f68e 100644
--- a/lib/librte_eal/common/eal_common_launch.c
+++ b/lib/librte_eal/common/eal_common_launch.c
@@ -15,6 +15,8 @@
#include <rte_per_lcore.h>
#include <rte_lcore.h>
+#include "eal_private.h"
+
/*
* Wait until a lcore finished its job.
*/
diff --git a/lib/librte_eal/common/eal_private.h b/lib/librte_eal/common/eal_private.h
index 798ede553b21..25e80547904f 100644
--- a/lib/librte_eal/common/eal_private.h
+++ b/lib/librte_eal/common/eal_private.h
@@ -10,6 +10,28 @@
#include <stdio.h>
#include <rte_dev.h>
+#include <rte_lcore.h>
+
+/**
+ * Structure storing internal configuration (per-lcore)
+ */
+struct lcore_config {
+ uint32_t core_id; /**< core number on socket for this lcore */
+ uint32_t core_index; /**< relative index, starting from 0 */
+ uint16_t socket_id; /**< physical socket id for this lcore */
+ uint8_t core_role; /**< role of core eg: OFF, RTE, SERVICE */
+ uint8_t detected; /**< true if lcore was detected */
+ volatile enum rte_lcore_state_t state; /**< lcore state */
+ rte_cpuset_t cpuset; /**< cpu set which the lcore affinity to */
+ pthread_t thread_id; /**< pthread identifier */
+ int pipe_master2slave[2]; /**< communication pipe with master */
+ int pipe_slave2master[2]; /**< communication pipe with master */
+ lcore_function_t * volatile f; /**< function to call */
+ void * volatile arg; /**< argument of function */
+ volatile int ret; /**< return value of function */
+};
+
+extern struct lcore_config lcore_config[RTE_MAX_LCORE];
/**
* Initialize the memzone subsystem (private to eal).
diff --git a/lib/librte_eal/common/include/rte_lcore.h b/lib/librte_eal/common/include/rte_lcore.h
index c86f72eb12a8..0c683919564e 100644
--- a/lib/librte_eal/common/include/rte_lcore.h
+++ b/lib/librte_eal/common/include/rte_lcore.h
@@ -66,30 +66,6 @@ typedef cpuset_t rte_cpuset_t;
} while (0)
#endif
-/**
- * Structure storing internal configuration (per-lcore)
- */
-struct lcore_config {
- unsigned detected; /**< true if lcore was detected */
- pthread_t thread_id; /**< pthread identifier */
- int pipe_master2slave[2]; /**< communication pipe with master */
- int pipe_slave2master[2]; /**< communication pipe with master */
- lcore_function_t * volatile f; /**< function to call */
- void * volatile arg; /**< argument of function */
- volatile int ret; /**< return value of function */
- volatile enum rte_lcore_state_t state; /**< lcore state */
- unsigned socket_id; /**< physical socket id for this lcore */
- unsigned core_id; /**< core number on socket for this lcore */
- int core_index; /**< relative index, starting from 0 */
- rte_cpuset_t cpuset; /**< cpu set which the lcore affinity to */
- uint8_t core_role; /**< role of core eg: OFF, RTE, SERVICE */
-};
-
-/**
- * Internal configuration (per-lcore)
- */
-extern struct lcore_config lcore_config[RTE_MAX_LCORE];
-
RTE_DECLARE_PER_LCORE(unsigned, _lcore_id); /**< Per thread "lcore id". */
RTE_DECLARE_PER_LCORE(rte_cpuset_t, _cpuset); /**< Per thread "cpuset". */
diff --git a/lib/librte_eal/common/rte_service.c b/lib/librte_eal/common/rte_service.c
index c3653ebae46c..6e21f549051b 100644
--- a/lib/librte_eal/common/rte_service.c
+++ b/lib/librte_eal/common/rte_service.c
@@ -21,6 +21,8 @@
#include <rte_memory.h>
#include <rte_malloc.h>
+#include "eal_private.h"
+
#define RTE_SERVICE_NUM_MAX 64
#define SERVICE_F_REGISTERED (1 << 0)
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index 7cbf82d37b0a..aeedf397764f 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -4,7 +4,6 @@ DPDK_2.0 {
__rte_panic;
eal_parse_sysfs_value;
eal_timer_source;
- lcore_config;
per_lcore__lcore_id;
per_lcore__rte_errno;
rte_calloc;
--
2.20.1
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] RFC: hiding struct rte_eth_dev
2019-09-24 16:42 0% ` Jerin Jacob
@ 2019-09-24 16:50 3% ` Ananyev, Konstantin
2019-09-26 11:13 4% ` Andrew Rybchenko
0 siblings, 1 reply; 200+ results
From: Ananyev, Konstantin @ 2019-09-24 16:50 UTC (permalink / raw)
To: Jerin Jacob, Ray Kinsella
Cc: dpdk-dev, Richardson, Bruce, Jerin Jacob Kollanukkaran,
Hemant Agrawal, Thomas Monjalon, Stephen Hemminger, Yigit,
Ferruh, maxime.coquelin, David Marchand, Zapolski, MarcinX A
Hi everyone,
> >
> > Hi folks,
> >
> > The ABI Stability proposals should be pretty well known at this point.
> > The latest rev is here ...
> >
> > http://inbox.dpdk.org/dev/1565864619-17206-1-git-send-email-mdr@ashroe.eu/
> >
> > As has been discussed public data structure's are risky for ABI
> > stability, as any changes to a data structure can change the ABI. As a
> > general rule you want to expose as few as possible (ideally none), and
> > keep them as small as possible.
> >
> > One of the key data structures in DPDK is `struct rte_eth_dev`. In this
> > case, rte_eth_dev is exposed public-ally, as a side-effect of the
> > inlining of the [rx,tx]_burst functions.
> >
> > Marcin Zapolski has been looking at what to do about it, with no current
> > consensus on a path forward. The options on our table is:-
> >
> > 1. Do nothing, live with the risk to DPDK v20 ABI stability.
> >
> > 2. Pad rte_eth_dev, add some extra bytes to the structure "in case" we
> > need to add a field during the v20 ABI (through to 20.11).
> >
> > 3. Break rte_eth_dev into public and private structs.
> > - See
> > http://inbox.dpdk.org/dev/20190906131813.1343-1-marcinx.a.zapolski@intel.com/
> > - This ends up quiet an invasive patch, late in the cycle, however it
> > does have no performance penalty.
> >
> > 4. Uninline [rx,tx]_burst functions
> > - See
> > http://inbox.dpdk.org/dev/20190730124950.1293-1-marcinx.a.zapolski@intel.com/
> > - This has a performance penalty of ~2% with testpmd, impact on a "real
> > workload" is likely to be in the noise.
> >
> > We need to agree an approach for v19.11, and that may be we agree to do
> > nothing. My personal vote is 4. as the simplest with minimal impact.
>
> My preference NOT to do #4. Reasons are:
> - I have seen performance drop from 1.5% to 3.5% based on the arm64
> cores in use(Embedded vs Server cores)
> - We need the correct approach to cater to cryptodev and eventdev as
> well. If #4 is checked in, We will
> take shotcut for cryptodev and eventdev
>
> My preference #1, do nothing, is probably ok and could live with #2,
> adding padding,
> and fix properly with #3 as when needed and use #3 scheme for crypto
> dev and eventdev as well.
>
>
My preference would be #4 also.
If that's not an option, then I suppose #1 for 19.11 and #3 for next release
when ABI breakage would be allowed.
BTW, good point that we need similar thing for other dev types too.
Konstantin
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] RFC: hiding struct rte_eth_dev
2019-09-23 16:19 5% [dpdk-dev] RFC: hiding struct rte_eth_dev Ray Kinsella
2019-09-23 16:35 0% ` Bruce Richardson
2019-09-24 9:07 0% ` Morten Brørup
@ 2019-09-24 16:42 0% ` Jerin Jacob
2019-09-24 16:50 3% ` Ananyev, Konstantin
2 siblings, 1 reply; 200+ results
From: Jerin Jacob @ 2019-09-24 16:42 UTC (permalink / raw)
To: Ray Kinsella
Cc: dpdk-dev, Richardson, Bruce, Jerin Jacob Kollanukkaran,
Hemant Agrawal, Thomas Monjalon, Stephen Hemminger, Yigit,
Ferruh, Ananyev, Konstantin, maxime.coquelin, David Marchand,
Marcin Zapolski
On Mon, Sep 23, 2019 at 9:49 PM Ray Kinsella <mdr@ashroe.eu> wrote:
>
> Hi folks,
>
> The ABI Stability proposals should be pretty well known at this point.
> The latest rev is here ...
>
> http://inbox.dpdk.org/dev/1565864619-17206-1-git-send-email-mdr@ashroe.eu/
>
> As has been discussed public data structure's are risky for ABI
> stability, as any changes to a data structure can change the ABI. As a
> general rule you want to expose as few as possible (ideally none), and
> keep them as small as possible.
>
> One of the key data structures in DPDK is `struct rte_eth_dev`. In this
> case, rte_eth_dev is exposed public-ally, as a side-effect of the
> inlining of the [rx,tx]_burst functions.
>
> Marcin Zapolski has been looking at what to do about it, with no current
> consensus on a path forward. The options on our table is:-
>
> 1. Do nothing, live with the risk to DPDK v20 ABI stability.
>
> 2. Pad rte_eth_dev, add some extra bytes to the structure "in case" we
> need to add a field during the v20 ABI (through to 20.11).
>
> 3. Break rte_eth_dev into public and private structs.
> - See
> http://inbox.dpdk.org/dev/20190906131813.1343-1-marcinx.a.zapolski@intel.com/
> - This ends up quiet an invasive patch, late in the cycle, however it
> does have no performance penalty.
>
> 4. Uninline [rx,tx]_burst functions
> - See
> http://inbox.dpdk.org/dev/20190730124950.1293-1-marcinx.a.zapolski@intel.com/
> - This has a performance penalty of ~2% with testpmd, impact on a "real
> workload" is likely to be in the noise.
>
> We need to agree an approach for v19.11, and that may be we agree to do
> nothing. My personal vote is 4. as the simplest with minimal impact.
My preference NOT to do #4. Reasons are:
- I have seen performance drop from 1.5% to 3.5% based on the arm64
cores in use(Embedded vs Server cores)
- We need the correct approach to cater to cryptodev and eventdev as
well. If #4 is checked in, We will
take shotcut for cryptodev and eventdev
My preference #1, do nothing, is probably ok and could live with #2,
adding padding,
and fix properly with #3 as when needed and use #3 scheme for crypto
dev and eventdev as well.
>
> Thanks,
>
> Ray K
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [RFC] ethdev: add new fields for max LRO session size
@ 2019-09-24 12:03 3% ` Matan Azrad
0 siblings, 0 replies; 200+ results
From: Matan Azrad @ 2019-09-24 12:03 UTC (permalink / raw)
To: Ferruh Yigit, dev
Cc: Thomas Monjalon, Andrew Rybchenko, Konstantin Ananyev, Olivier Matz
Hi
From: Ferruh Yigit
> On 9/15/2019 8:48 AM, Matan Azrad wrote:
> > Hi Ferruh
> >
> > From: Ferruh Yigit <ferruh.yigit@intel.com>
> >> On 8/29/2019 8:47 AM, Matan Azrad wrote:
> >>> It may be needed by the user to limit the LRO session packet size.
> >>> In order to allow the above limitation, add new Rx configuration for
> >>> the maximum LRO session size.
> >>>
> >>> In addition, Add a new capability to expose the maximum LRO session
> >>> size supported by the port.
> >>>
> >>> Signed-off-by: Matan Azrad <matan@mellanox.com>
> >>
> >> Hi Matan,
> >>
> >> Is there any existing user of this new field?
> >
> > All the LRO users need it due to the next reasons:
> >
> > 1. If scatter is enabled - The dpdk user can limit the LRO session size created
> by the HW by this field, if no field like that - there is no way to limit it.
> > 2. No scatter - the dpdk user may want to limit the LRO packet size in order
> to save enough tail-room in the mbuf for its own usage.
> > 3. The limitation of max_rx_pkt_len is not enough - doesn't make sense to
> limit LRO traffic as single packet.
> >
>
> So should there be more complement patches to this RFC? To update the
> users of the field with the new field.
We already exposed it as ABI breakage in the last deprecation notice.
We probably cannot complete it for 19.11 version, hopefully for 20.02 it will be completed.
Matan
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] [PATCH v3 2/4] doc: changes to abi policy introducing major abi versions
2019-08-30 16:20 10% ` Kevin Traynor
@ 2019-09-24 11:32 10% ` Ray Kinsella
2019-09-25 12:29 5% ` Kevin Traynor
0 siblings, 1 reply; 200+ results
From: Ray Kinsella @ 2019-09-24 11:32 UTC (permalink / raw)
To: Kevin Traynor, dev
Cc: thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, nhorman,
maxime.coquelin, john.mcnamara, marko.kovacevic, hemant.agrawal,
Luca Boccassi, David Marchand
Thanks Kevin for working through all this.
Other comments are inline.
On 30/08/2019 17:20, Kevin Traynor wrote:
> Hi Ray,
>
> On 15/08/2019 11:23, Ray Kinsella wrote:
>> This policy change introduces major ABI versions, these are
>> declared every year, typically aligned with the LTS release
>> and are supported by subsequent releases in the following year.
>> This change is intended to improve ABI stabilty for those projects
>> consuming DPDK.
>>
>> Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
>> ---
>> doc/guides/contributing/abi_policy.rst | 308 ++++++++++++++++++++++++---------
>> doc/guides/contributing/stable.rst | 38 ++--
>> 2 files changed, 245 insertions(+), 101 deletions(-)
>>
>> diff --git a/doc/guides/contributing/abi_policy.rst b/doc/guides/contributing/abi_policy.rst
>> index 55bacb4..6190bdc 100644
>> --- a/doc/guides/contributing/abi_policy.rst
>> +++ b/doc/guides/contributing/abi_policy.rst
>> @@ -1,33 +1,46 @@
>> .. SPDX-License-Identifier: BSD-3-Clause
>> - Copyright 2018 The DPDK contributors
>> + Copyright 2019 The DPDK contributors
>>
>> -.. abi_api_policy:
>> +.. _abi_policy:
>>
>> -DPDK ABI/API policy
>> -===================
>> +ABI Policy
>> +==========
>>
>> Description
>> -----------
>>
>> -This document details some methods for handling ABI management in the DPDK.
>> +This document details the management policy that ensures the long-term stability
>> +of the DPDK ABI and API.
>>
>> General Guidelines
>> ------------------
>>
>> -#. Whenever possible, ABI should be preserved
>> -#. ABI/API may be changed with a deprecation process
>> -#. The modification of symbols can generally be managed with versioning
>> -#. Libraries or APIs marked in ``experimental`` state may change without constraint
>> -#. New APIs will be marked as ``experimental`` for at least one release to allow
>> - any issues found by users of the new API to be fixed quickly
>> -#. The addition of symbols is generally not problematic
>> -#. The removal of symbols generally is an ABI break and requires bumping of the
>> - LIBABIVER macro
>> -#. Updates to the minimum hardware requirements, which drop support for hardware which
>> - was previously supported, should be treated as an ABI change.
>> -
>> -What is an ABI
>> -~~~~~~~~~~~~~~
>> +#. Major ABI versions are declared every **year** and are then supported for one
>> + year, typically aligned with the :ref:`LTS release <stable_lts_releases>`.
>> +#. The ABI version is managed at a project level in DPDK, with the ABI version
>> + reflected in all :ref:`library's soname <what_is_soname>`.
>> +#. The ABI should be preserved and not changed lightly. ABI changes must follow
>> + the outlined :ref:`deprecation process <abi_changes>`.
>> +#. The addition of symbols is generally not problematic. The modification of
>> + symbols is managed with :ref:`ABI Versioning <abi_versioning>`.
>> +#. The removal of symbols is considered an :ref:`ABI breakage <abi_breakages>`,
>> + once approved these will form part of the next ABI version.
>> +#. Libraries or APIs marked as :ref:`Experimental <experimental_apis>` are not
>> + considered part of an ABI version and may change without constraint.
>> +#. Updates to the :ref:`minimum hardware requirements <hw_rqmts>`, which drop
>> + support for hardware which was previously supported, should be treated as an
>> + ABI change.
>> +
>> +.. note::
>> +
>> + In 2019, the DPDK community stated it's intention to move to ABI stable
>> + releases, over a number of release cycles. Beginning with maintaining ABI
>> + stability through one year of DPDK releases starting from DPDK 19.11. This
>> + policy will be reviewed in 2020, with intention of lengthening the stability
>> + period.
>> +
>> +What is an ABI?
>> +~~~~~~~~~~~~~~~
>>
>> An ABI (Application Binary Interface) is the set of runtime interfaces exposed
>> by a library. It is similar to an API (Application Programming Interface) but
>> @@ -39,30 +52,67 @@ Therefore, in the case of dynamic linking, it is critical that an ABI is
>> preserved, or (when modified), done in such a way that the application is unable
>> to behave improperly or in an unexpected fashion.
>>
>> +What is an ABI version?
>> +~~~~~~~~~~~~~~~~~~~~~~~
>>
>> -ABI/API Deprecation
>> --------------------
>> +An ABI version is an instance of a library's ABI at a specific release. Certain
>> +releases are considered by the community to be milestone releases, the yearly
>> +LTS for example. Supporting those milestone release's ABI for some number of
>> +subsequent releases is desirable to facilitate application upgrade. Those ABI
>> +version's aligned with milestones release are therefore called 'ABI major
>> +versions' and are supported for some number of releases.
>> +
>> +More details on major ABI version can be found in the :ref:`ABI versioning
>> +<major_abi_versions>` guide.
>>
>> The DPDK ABI policy
>> -~~~~~~~~~~~~~~~~~~~
>> +-------------------
>> +
>> +A major ABI version is declared every year, aligned with that year's LTS
>> +release, e.g. v19.11. This ABI version is then supported for one year by all
>> +subsequent releases within that time period, until the next LTS release, e.g.
>> +v20.11.
>> +
>> +At the declaration of a major ABI version, major version numbers encoded in
>> +libraries soname's are bumped to indicate the new version, with the minor
>> +version reset to ``0``. An example would be ``librte_eal.so.20.3`` would become
>> +``librte_eal.so.21.0``.
>> +
>> +The ABI may then change multiple times, without warning, between the last major
>> +ABI version increment and the HEAD label of the git tree, with the condition
>> +that ABI compatibility with the major ABI version is preserved and therefore
>> +soname's do not change.
>> +
>> +Minor versions are incremented to indicate the release of a new ABI compatible
>> +DPDK release, typically the DPDK quarterly releases. An example of this, might
>> +be that ``librte_eal.so.20.1`` would indicate the first ABI compatible DPDK
>> +release, following the declaration of the new major ABI version ``20``.
>> +
>> +ABI versions, are supported by each release until such time as the next major
>> +ABI version is declared. At that time, the deprecation of the previous major ABI
>> +version will be noted in the Release Notes with guidance on individual symbol
>> +depreciation and upgrade notes provided.
>>
>> -ABI versions are set at the time of major release labeling, and the ABI may
>> -change multiple times, without warning, between the last release label and the
>> -HEAD label of the git tree.
>> +.. _abi_changes:
>>
>> -ABI versions, once released, are available until such time as their
>> -deprecation has been noted in the Release Notes for at least one major release
>> -cycle. For example consider the case where the ABI for DPDK 2.0 has been
>> -shipped and then a decision is made to modify it during the development of
>> -DPDK 2.1. The decision will be recorded in the Release Notes for the DPDK 2.1
>> -release and the modification will be made available in the DPDK 2.2 release.
>> +ABI Changes
>> +~~~~~~~~~~~
>>
>> -ABI versions may be deprecated in whole or in part as needed by a given
>> -update.
>> +The ABI may still change after the declaration of a major ABI version, that is
>> +new APIs may be still added or existing APIs may be modified.
>>
>> -Some ABI changes may be too significant to reasonably maintain multiple
>> -versions. In those cases ABI's may be updated without backward compatibility
>> -being provided. The requirements for doing so are:
>> +.. Warning::
>> +
>> + Note that, the process for ABI deprecation should not be undertaken lightly.
>> + ABI stability is extremely important for downstream consumers of the DPDK,
>
>> + especially when distributed in shared object form. Every effort should be
>> + made to preserve the ABI whenever possible. The ABI should only be changed
>> + for significant reasons, such as performance enhancements. ABI breakage due
>> + to changes such as reorganizing public structure fields for aesthetic or
>> + readability purposes should be avoided.
>> +
>
> This text is not changed and it reads like *any* performance enhancement
> is a good enough reason for an ABI break. Can't obviously quantify it,
> but maybe "major performance enhancement" is closer to the intended
> tone? Sorry for nit-picking over one word!
I agree, I was in two minds about whether to clarify this section or if
it was fine as-is. I left it there as a general warning to stop and
think before you ask to change the ABI. A performance gain alone doesn't
absolve the contributor from an obligation to preserve ABI compatibility.
Perhaps reword as follows?
.. Warning::
Note that, this policy details the method by which the ABI may be
changed, with due regard to preserving compatibility and observing
depreciation notices. This process however should not be undertaken
lightly, as a general rule ABI stability is extremely important for
downstream consumers of DPDK. The ABI should only be changed for
significant reasons, such as performance enhancements. ABI breakages due
to changes such as reorganizing public structure fields for aesthetic or
readability purposes should be avoided.
>
>> +
>> +The requirements for changing the ABI are:
>>
>> #. At least 3 acknowledgments of the need to do so must be made on the
>> dpdk.org mailing list.
>> @@ -71,34 +121,119 @@ being provided. The requirements for doing so are:
>> no maintainer is available for the component, the tree/sub-tree maintainer
>> for that component must acknowledge the ABI change instead.
>>
>> + - The acknowledgment of a member of the technical board, as a delegate of the
>> + `technical board <https://core.dpdk.org/techboard/>`_ acknowledging the
>> + need for the ABI change, is also mandatory.
>> +
>> - It is also recommended that acknowledgments from different "areas of
>> interest" be sought for each deprecation, for example: from NIC vendors,
>> CPU vendors, end-users, etc.
>>
>> -#. The changes (including an alternative map file) can be included with
>> - deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option,
>> - to provide more details about oncoming changes.
>> - ``RTE_NEXT_ABI`` wrapper will be removed when it become the default ABI.
>> - More preferred way to provide this information is sending the feature
>> - as a separate patch and reference it in deprecation notice.
>> +#. Backward compatibly with the major ABI version must be maintained through
>
> s/compatibly/compatibility/
ACK
>
>> + :ref:`abi_versioning`, with :ref:`forward-only <forward-only>` compatibility
>> + offered for any ABI changes that are indicated to be part of the next ABI
>> + version.
>>
>> -#. A full deprecation cycle, as explained above, must be made to offer
>> - downstream consumers sufficient warning of the change.
>> + - In situations were backward compatibility is not possible, read the
>
> s/were/where/
ACK
>
>> + section on :ref:`abi_breakages`.
>>
>> -Note that the above process for ABI deprecation should not be undertaken
>> -lightly. ABI stability is extremely important for downstream consumers of the
>> -DPDK, especially when distributed in shared object form. Every effort should
>> -be made to preserve the ABI whenever possible. The ABI should only be changed
>> -for significant reasons, such as performance enhancements. ABI breakage due to
>> -changes such as reorganizing public structure fields for aesthetic or
>> -readability purposes should be avoided.
>> + - No backward or forward compatibility is offered for API changes marked as
>> + ``experimental``, as described in the section on :ref:`Experimental APIs
>> + and Libraries <experimental_apis>`.
>>
>> -.. note::
>> +#. If a newly proposed API functionally replaces an existing one, when the new
>> + API becomes non-experimental, then the old one is marked with
>> + ``__rte_deprecated``.
>> +
>> + - The depreciated API should follow the notification process to be removed,
>> + see :ref:`deprecation_notices`.
>> +
>> + - At the declaration of the next major ABI version, those ABI changes then
>> + become a formal part of the new ABI and the requirement to preserve ABI
>> + compatibility with the last major ABI version is then dropped.
>> +
>> + - The responsibility for removing redundant ABI compatibility code rests
>> + with the original contributor of the ABI changes, failing that, then with
>> + the contributor's company and then finally with the maintainer.
>> +
>> +.. _forward-only:
>> +
>> +.. Note::
>> +
>> + Note that forward-only compatibility is offered for those changes made
>> + between major ABI versions. As a library's soname can only describe
>> + compatibility with the last major ABI version, until the next major ABI
>> + version is declared, these changes therefore cannot be resolved as a runtime
>> + dependency through the soname. Therefore any application wishing to make use
>> + of these ABI changes can only ensure that it's runtime dependencies are met
>> + through Operating System package versioning.
>> +
>> +.. _hw_rqmts:
>> +
>> +.. Note::
>>
>> Updates to the minimum hardware requirements, which drop support for hardware
>> which was previously supported, should be treated as an ABI change, and
>> - follow the relevant deprecation policy procedures as above: 3 acks and
>> - announcement at least one release in advance.
>> + follow the relevant deprecation policy procedures as above: 3 acks, technical
>> + board approval and announcement at least one release in advance.
>> +
>> +.. _abi_breakages:
>> +
>> +ABI Breakages
>> +~~~~~~~~~~~~~
>> +
>> +For those ABI changes that are too significant to reasonably maintain multiple
>> +symbol versions, there is an amended process. In these cases, ABIs may be
>> +updated without the requirement of backward compatibility being provided. These
>> +changes must follow the `same process :ref:`described above <abi_changes>` as non-breaking
>> +changes, however with the following additional requirements:
>> +
>> +#. ABI breaking changes (including an alternative map file) can be included with
>> + deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option, to provide
>> + more details about oncoming changes. ``RTE_NEXT_ABI`` wrapper will be removed
>> + at the declaration of the next major ABI version.
>> +
>> +#. Once approved, and after the depreciation notice has been observed these
>> + changes will form part of the next declared major ABI version.
>> +
>> +Examples of ABI Changes
>> +~~~~~~~~~~~~~~~~~~~~~~~
>> +
>> +The following are examples of allowable ABI changes occurring between
>> +declarations of major ABI versions.
>> +
>> +* DPDK 19.11 release, defines the function ``rte_foo()``, and ``rte_foo()``
>> + as part of the major ABI version ``20``.
>> +
>> +* DPDK 20.02 release defines a new function ``rte_foo(uint8_t bar)``, and
>> + this is not a problem as long as the symbol ``rte_foo@DPDK20`` is
>> + preserved through :ref:`abi_versioning`.
>> +
>> + - The new function may be marked with the ``__rte_experimental`` tag for a
>> + number of releases, as described in the section :ref:`experimental_apis`.
>> +
>> + - Once ``rte_foo(uint8_t bar)`` becomes non-experimental ``rte_foo()`` is then
>> + declared as ``__rte_depreciated``, with an associated deprecation notice
>> + provided.
>> +
>> +* DPDK 19.11 is not re-released to include ``rte_foo(uint8_t bar)``, the new
>> + version of ``rte_foo`` only exists from DPDK 20.02 onwards as described in the
>> + :ref:`note on forward-only compatibility<forward-only>`.
>> +
>> +* DPDK 20.02 release defines the experimental function ``__rte_experimental
>> + rte_baz()``. This function may or may not exist in the DPDK 20.05 release.
>> +
>> +* An application ``dPacket`` wishes to use ``rte_foo(uint8_t bar)``, before the
>> + declaration of the DPDK ``21`` major API version. The application can only
>> + ensure it's runtime dependencies are met by specifying ``DPDK (>= 20.2)`` as
>> + an explicit package dependency, as the soname only may only indicate the
>> + supported major ABI version.
>> +
>> +* At the release of DPDK 20.11, the function ``rte_foo(uint8_t bar)`` becomes
>> + formally part of then new major ABI version DPDK 21.0 and ``rte_foo()`` may be
>> + removed.
>> +
>> +.. _deprecation_notices:
>>
>> Examples of Deprecation Notices
>> ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
>> @@ -106,46 +241,42 @@ Examples of Deprecation Notices
>> The following are some examples of ABI deprecation notices which would be
>> added to the Release Notes:
>>
>> -* The Macro ``#RTE_FOO`` is deprecated and will be removed with version 2.0,
>> - to be replaced with the inline function ``rte_foo()``.
>> +* The Macro ``#RTE_FOO`` is deprecated and will be removed with ABI version
>> + 21, to be replaced with the inline function ``rte_foo()``.
>>
>> * The function ``rte_mbuf_grok()`` has been updated to include a new parameter
>> - in version 2.0. Backwards compatibility will be maintained for this function
>> - until the release of version 2.1
>> + in version 20.2. Backwards compatibility will be maintained for this function
>> + until the release of the new DPDK major ABI version 21, in DPDK version
>> + 20.11.
>>
>> -* The members of ``struct rte_foo`` have been reorganized in release 2.0 for
>> +* The members of ``struct rte_foo`` have been reorganized in DPDK 20.02 for
>> performance reasons. Existing binary applications will have backwards
>> - compatibility in release 2.0, while newly built binaries will need to
>> - reference the new structure variant ``struct rte_foo2``. Compatibility will
>> - be removed in release 2.2, and all applications will require updating and
>> + compatibility in release 20.02, while newly built binaries will need to
>> + reference the new structure variant ``struct rte_foo2``. Compatibility will be
>> + removed in release 20.11, and all applications will require updating and
>> rebuilding to the new structure at that time, which will be renamed to the
>> original ``struct rte_foo``.
>>
>> * Significant ABI changes are planned for the ``librte_dostuff`` library. The
>> - upcoming release 2.0 will not contain these changes, but release 2.1 will,
>> + upcoming release 20.02 will not contain these changes, but release 20.11 will,
>> and no backwards compatibility is planned due to the extensive nature of
>> - these changes. Binaries using this library built prior to version 2.1 will
>> + these changes. Binaries using this library built prior to ABI version 21 will
>> require updating and recompilation.
>>
>> -New API replacing previous one
>> -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
>> -
>> -If a new API proposed functionally replaces an existing one, when the new API
>> -becomes non-experimental then the old one is marked with ``__rte_deprecated``.
>> -Deprecated APIs are removed completely just after the next LTS.
>> -
>> -Reminder that old API should follow deprecation process to be removed.
>> +.. _experimental_apis:
>>
>> +Experimental
>> +------------
>>
>> -Experimental APIs
>> ------------------
>> +APIs
>> +~~~~
>>
>> -APIs marked as ``experimental`` are not considered part of the ABI and may
>> -change without warning at any time. Since changes to APIs are most likely
>> -immediately after their introduction, as users begin to take advantage of
>> -those new APIs and start finding issues with them, new DPDK APIs will be
>> -automatically marked as ``experimental`` to allow for a period of stabilization
>> -before they become part of a tracked ABI.
>> +APIs marked as ``experimental`` are not considered part of an ABI version and
>> +may change without warning at any time. Since changes to APIs are most likely
>> +immediately after their introduction, as users begin to take advantage of those
>> +new APIs and start finding issues with them, new DPDK APIs will be automatically
>> +marked as ``experimental`` to allow for a period of stabilization before they
>> +become part of a tracked ABI version.
>>
>> Note that marking an API as experimental is a multi step process.
>> To mark an API as experimental, the symbols which are desired to be exported
>> @@ -163,7 +294,16 @@ In addition to tagging the code with ``__rte_experimental``,
>> the doxygen markup must also contain the EXPERIMENTAL string,
>> and the MAINTAINERS file should note the EXPERIMENTAL libraries.
>>
>> -For removing the experimental tag associated with an API, deprecation notice
>> -is not required. Though, an API should remain in experimental state for at least
>> -one release. Thereafter, normal process of posting patch for review to mailing
>> -list can be followed.
>> +For removing the experimental tag associated with an API, deprecation notice is
>> +not required. Though, an API should remain in experimental state for at least
>> +one release. Thereafter, the normal process of posting patch for review to
>> +mailing list can be followed.
>> +
>> +Libraries
>> +~~~~~~~~~
>> +
>> +Libraries marked as ``experimental`` are entirely not considered part of an ABI
>> +version, and may change without warning at any time. Experimental libraries
>> +always have a major version of ``0`` to indicate they exist outside of
>> +:ref:`abi_versioning` , with the minor version incremented with each ABI change
>> +to library.
>> diff --git a/doc/guides/contributing/stable.rst b/doc/guides/contributing/stable.rst
>> index 6a5eee9..d95c200 100644
>> --- a/doc/guides/contributing/stable.rst
>> +++ b/doc/guides/contributing/stable.rst
>> @@ -1,7 +1,7 @@
>> .. SPDX-License-Identifier: BSD-3-Clause
>> Copyright 2018 The DPDK contributors
>>
>> -.. stable_lts_releases:
>> +.. _stable_lts_releases:
>>
>> DPDK Stable Releases and Long Term Support
>> ==========================================
>> @@ -53,6 +53,9 @@ year's November (X.11) release will be maintained as an LTS for 2 years.
>> After the X.11 release, an LTS branch will be created for it at
>> http://git.dpdk.org/dpdk-stable where bugfixes will be backported to.
>>
>> +A LTS release may align with the declaration of a new major ABI version,
>> +please read the :ref:`abi_policy` for more information.
>> +
>
> Above is worth to mention, but as discussed on call earlier today, the
> changes below should be dropped from this patchset. At present each LTS
> minor release (e.g. 18.11.2) maintains the API/ABI of the original LTS
> release (e.g. 18.11) and that is not changing.
ACK, I will remove
>
> What type of non-ABI breaking things are backported to LTS branches can
> be discussed during the LTS presentation in DPDK userspace.
Do you anticipate any updates here?
Thanks,
Ray K
>
> thanks,
> Kevin.
>
>> It is anticipated that there will be at least 4 releases per year of the LTS
>> or approximately 1 every 3 months. However, the cadence can be shorter or
>> longer depending on the number and criticality of the backported
>> @@ -68,10 +71,13 @@ point the LTS branch will no longer be maintained with no further releases.
>> What changes should be backported
>> ---------------------------------
>>
>> -Backporting should be limited to bug fixes. All patches accepted on the master
>> -branch with a Fixes: tag should be backported to the relevant stable/LTS
>> -branches, unless the submitter indicates otherwise. If there are exceptions,
>> -they will be discussed on the mailing lists.
>> +Backporting is a naturally conservative activity, and therefore should only
>> +include bug fixes and support for new hardware, were adding support does not
>> +necessitate DPDK ABI/API changes.
>> +
>> +All patches accepted on the master branch with a Fixes: tag should be backported
>> +to the relevant stable/LTS branches, unless the submitter indicates otherwise.
>> +If there are exceptions, they will be discussed on the mailing lists.
>>
>> Fixes suitable for backport should have a ``Cc: stable@dpdk.org`` tag in the
>> commit message body as follows::
>> @@ -86,13 +92,18 @@ commit message body as follows::
>> Signed-off-by: Alex Smith <alex.smith@example.com>
>>
>>
>> -Fixes not suitable for backport should not include the ``Cc: stable@dpdk.org`` tag.
>> +Fixes not suitable for backport should not include the ``Cc: stable@dpdk.org``
>> +tag.
>>
>> -Features should not be backported to stable releases. It may be acceptable, in
>> -limited cases, to back port features for the LTS release where:
>> +New features, with the exception of new hardware support, should not be
>> +backported to stable releases. In the case of new hardware support or any other
>> +exceptional circumstances limited backporting maybe permitted to the LTS release
>> +where:
>>
>> -* There is a justifiable use case (for example a new PMD).
>> -* The change is non-invasive.
>> +* There is a justifiable use case, for example the change is required to support
>> + a new platform or device (for example a new PMD).
>> +* The change is ABI/API preserving, it does not present an obvious "new feature"
>> + to end consumer.
>> * The work of preparing the backport is done by the proposer.
>> * There is support within the community.
>>
>> @@ -119,10 +130,3 @@ A Stable Release will be released by:
>> list.
>>
>> Stable releases are available on the `dpdk.org download page <http://core.dpdk.org/download/>`_.
>> -
>> -
>> -ABI
>> ----
>> -
>> -The Stable Release should not be seen as a way of breaking or circumventing
>> -the DPDK ABI policy.
>>
^ permalink raw reply [relevance 10%]
* Re: [dpdk-dev] RFC: hiding struct rte_eth_dev
2019-09-23 16:19 5% [dpdk-dev] RFC: hiding struct rte_eth_dev Ray Kinsella
2019-09-23 16:35 0% ` Bruce Richardson
@ 2019-09-24 9:07 0% ` Morten Brørup
2019-09-24 16:42 0% ` Jerin Jacob
2 siblings, 0 replies; 200+ results
From: Morten Brørup @ 2019-09-24 9:07 UTC (permalink / raw)
To: Ray Kinsella, dpdk-dev
Cc: Richardson, Bruce, Jerin Jacob Kollanukkaran, Hemant Agrawal,
Thomas Monjalon, Stephen Hemminger, Yigit, Ferruh, Ananyev,
Konstantin, maxime.coquelin, David Marchand, Marcin Zapolski
> -----Original Message-----
> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Ray Kinsella
> Sent: Monday, September 23, 2019 6:19 PM
> To: dpdk-dev
> Cc: Richardson, Bruce; Jerin Jacob Kollanukkaran; Hemant Agrawal;
> Thomas Monjalon; Stephen Hemminger; Yigit, Ferruh; Ananyev, Konstantin;
> maxime.coquelin@redhat.com; David Marchand; Marcin Zapolski
> Subject: [dpdk-dev] RFC: hiding struct rte_eth_dev
>
> Hi folks,
>
> The ABI Stability proposals should be pretty well known at this point.
> The latest rev is here ...
>
> http://inbox.dpdk.org/dev/1565864619-17206-1-git-send-email-
> mdr@ashroe.eu/
>
> As has been discussed public data structure's are risky for ABI
> stability, as any changes to a data structure can change the ABI. As a
> general rule you want to expose as few as possible (ideally none), and
> keep them as small as possible.
>
> One of the key data structures in DPDK is `struct rte_eth_dev`. In this
> case, rte_eth_dev is exposed public-ally, as a side-effect of the
> inlining of the [rx,tx]_burst functions.
>
> Marcin Zapolski has been looking at what to do about it, with no
> current
> consensus on a path forward. The options on our table is:-
>
> 1. Do nothing, live with the risk to DPDK v20 ABI stability.
>
> 2. Pad rte_eth_dev, add some extra bytes to the structure "in case" we
> need to add a field during the v20 ABI (through to 20.11).
>
> 3. Break rte_eth_dev into public and private structs.
> - See
> http://inbox.dpdk.org/dev/20190906131813.1343-1-
> marcinx.a.zapolski@intel.com/
> - This ends up quiet an invasive patch, late in the cycle, however it
> does have no performance penalty.
>
> 4. Uninline [rx,tx]_burst functions
> - See
> http://inbox.dpdk.org/dev/20190730124950.1293-1-
> marcinx.a.zapolski@intel.com/
> - This has a performance penalty of ~2% with testpmd, impact on a
> "real
> workload" is likely to be in the noise.
>
> We need to agree an approach for v19.11, and that may be we agree to do
> nothing. My personal vote is 4. as the simplest with minimal impact.
>
> Thanks,
>
> Ray K
First of all, consider why an application would need to access the rte_eth_dev structure directly. I don't see why. So I assume that the only issue with hiding it behind an opaque handle is its use in the [rx,tx]_burst functions.
Next, let's consider the impact of uninlining the [rx,tx]_burst functions. An actual application would have to be extremely simple for the impact to be anywhere near the 2 % seen with testpmd.
Thus I vote for #4.
#3 is even better than #4, assuming performance *of core functionality* trumps general readability and maintainability. And it probably doesn't affect readability and maintainability for DPDK users/consumers. So I also vote for #3.
I think the decision about uninlining core functions should be a decision about the general principle, which should be discussed thoroughly, perhaps using this as a reference example.
Furthermore, I would like to repeat my old request for a more realistic reference performance benchmark application than testpmd if performance benchmark results are used as arguments for or against suggestions like this.
With testpmd, everything flies directly through the cache. I would be so bold as to claim that if you shuffle the mbuf fields between the first and second cache line, you probably wouldn't see any difference with testpmd; the packet goes directly to the tx step after the rx step, and the cache miss that would normally be seen in the tx step is instead taken in the rx step, so there is no performance difference.
A reference performance benchmark application should have at least two or three separate steps, each processing a sufficiently large amount of packets, so the cache is only efficient within each step. It could be an ingress step for receiving and enqueueing the packets, a delay step holding enough packets for their information to disappear from the cache, and an egress step for dequeueing and transmitting the packets.
Med venlig hilsen / kind regards
- Morten Brørup
^ permalink raw reply [relevance 0%]
* [dpdk-dev] [RFC] Proposals and notes from ABI stability panel @ DPDK Userspace
@ 2019-09-23 17:51 9% Ray Kinsella
2019-09-25 13:31 8% ` Kevin Traynor
0 siblings, 1 reply; 200+ results
From: Ray Kinsella @ 2019-09-23 17:51 UTC (permalink / raw)
To: dpdk-dev, O'Driscoll, Tim, Brian; +Cc: techboard
Folks,
As you may be aware, there was a panel on ABI Stability @ DPDK
Userspace. There where a number of proposed amendments to the ABI
stability proposal made, as well as a number of points and comments, you
will find all these below. The proposals needs further discussion so
please chime in below.
Thanks to Tim for capturing while I was busy on the stage.
Thanks,
Ray K
Table of Contents
_________________
1. Proposals from the panel discussion.
.. 1. Developer releases (versus User Releases)
.. 2. Core and non-core packaging
.. 3. Approach for public data structures
.. 4. Delaying v19.11
2. Other notes from the panel discussion.
.. 1. Performance as the paramount goal.
.. 2. Length of ABI Stability.
.. 3. Testing ABI Stability
.. 4. Call to action
1 Proposals from the panel discussion.
======================================
1.1 Developer releases (versus User Releases)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- Summary: Differentiate between "developer" and "user" releases.
- 1 year is too long to wait to upstream a new feature which breaks
ABI.
- Developer releases would be for use by the development community
only.
- A proposed compromise was that the .08 release would be the only
"developer release".
1.2 Core and non-core packaging
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- Summary: OS packaging doesn't include all libraries.
- This would create a delta between the community ABI, and the OS
packaging.
- OS packagers rational is that some libraries are used very rarely.
1.3 Approach for public data structures
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- Summary: Public/exposed data structures are tricky for ABI
stability.
- See discussion @
<http://inbox.dpdk.org/dev/980083c6-130a-9658-f82b-0c9ddc7cc0cc@ashroe.eu/>
1.4 Delaying v19.11
~~~~~~~~~~~~~~~~~~~
- Summary: push v19.11 to v19.12 to make more time to prepare and wave
the depreciation process.
- OVS take Nov LTS release in their Feb release. Delaying 19.11 may
have an impact on this.
- Not easy to change release at this stage as many things depend on
it (OS distros etc.).
2 Other notes from the panel discussion.
========================================
2.1 Performance as the paramount goal.
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- Some users, would trade performance for better readability and
debug-ability.
- Skepticism that micro-benchmarks reflect real world performance.
2.2 Length of ABI Stability.
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- Some users, questioned if 1 year would be long enough.
- It was clarified that the 1 year period, would be reviewed after the
first year with the intention of lengthening the period.
2.3 Testing ABI Stability
~~~~~~~~~~~~~~~~~~~~~~~~~
- More work for developers and validation teams. Need to validate
multiple paths for symbol versioning.
2.4 Call to action
~~~~~~~~~~~~~~~~~~
- We should do something. So we don’t want to have the same
conversation again in a year.
^ permalink raw reply [relevance 9%]
* Re: [dpdk-dev] [PATCH] mbuf: support dynamic fields and flags
2019-09-23 16:16 3% ` Olivier Matz
@ 2019-09-23 17:14 0% ` Wiles, Keith
0 siblings, 0 replies; 200+ results
From: Wiles, Keith @ 2019-09-23 17:14 UTC (permalink / raw)
To: Olivier Matz
Cc: dev, Thomas Monjalon, Wang, Haiyue, Stephen Hemminger,
Andrew Rybchenko, Jerin Jacob Kollanukkaran
> On Sep 23, 2019, at 11:16 AM, Olivier Matz <olivier.matz@6wind.com> wrote:
>
> Hi,
>
> (reformated the quotes)
>
> On Mon, Sep 23, 2019 at 03:14:51PM +0000, Wiles, Keith wrote:
>>
>>
>> On Sep 23, 2019, at 4:13 AM, Olivier Matz <olivier.matz@6wind.com<mailto:olivier.matz@6wind.com>> wrote:
>>>
>>> Hi Keith,
>>>
>>> On Sat, Sep 21, 2019 at 08:28:32AM +0000, Wiles, Keith wrote:
>>>>
>>>>
>>>> On Sep 18, 2019, at 6:54 PM, Olivier Matz <olivier.matz@6wind.com<mailto:olivier.matz@6wind.com>> wrote:
>>>>
>>>>> Many features require to store data inside the mbuf. As the room in mbuf
>>>>> structure is limited, it is not possible to have a field for each
>>>>> feature. Also, changing fields in the mbuf structure can break the API
>>>>> or ABI.
>>>>>
>>>>> This commit addresses these issues, by enabling the dynamic registration
>>>>> of fields or flags:
>>>>>
>>>>> - a dynamic field is a named area in the rte_mbuf structure, with a
>>>>> given size (>= 1 byte) and alignment constraint.
>>>>> - a dynamic flag is a named bit in the rte_mbuf structure.
>>>>>
>>>>> The typical use case is a PMD that registers space for an offload
>>>>> feature, when the application requests to enable this feature. As
>>>>> the space in mbuf is limited, the space should only be reserved if it
>>>>> is going to be used (i.e when the application explicitly asks for it).
>>>>>
>>>>> The registration can be done at any moment, but it is not possible
>>>>> to unregister fields or flags for now.
>>>>>
>>>>> Signed-off-by: Olivier Matz <olivier.matz@6wind.com<mailto:olivier.matz@6wind.com>>
>>>>> Acked-by: Thomas Monjalon <thomas@monjalon.net<mailto:thomas@monjalon.net>>
>>>> —
>>>>
>>>>
>>>>
>>>> The idea of registration for space in the mbuf I am not a big fan. I did like
>>>> Konstantin’s suggestion of having the compiler help with optimizing the code,
>>>> but with a slight difference. Maybe I misunderstand, but now with this design
>>>> you have to pass the offsets to different parts of the application or place in
>>>> global memory or have each section request the offsets. It seems great if the
>>>> application is one big application or an appliance model application having
>>>> control of the whole design not so good for service chains like designs where
>>>> different parts of the whole application is design by different teams.
>>>
>>> If the global variable storing the offset is defined in the mbuf layer, what
>>> would be the problem?
>>
>> Are you assuming the values are shared between primary/secondary model or
>> between processes using shared memory? If moving the packet data via shared
>> memory to a different application written by a different company you still
>> have to move that metadata.
>
> The dynamic mbuf proposal works with secondary processes. What does that
> change if the application is written by a different company? If you need
> to store a timestamp, you register the timestamp and the offset will be
> the same in primary and secondary.
>
>
>> If the type was carried with the mbuf we can easily convey a small
>> type value or we would need to tell the other side we have all of this
>> registration information to send. I would suggest the number of mbuf
>> types will be small over time and I believe a 4 bit or 8 bit type is
>> reasonable. In many protocols using a type value is used to convey
>> this type of information. We can even tightly control the number of
>> types DPDK controls and then leave some for user defined if we like.
>
> 8 bits means 256 different mbuf layouts.
I also stated 4 bits, but it is a problem with using types we have to allow a number of them, but the smart thing is we restrict DPDK uses to only a few. The developer of other applications using DPDK can use any number they need.
> You did not replied to my previous questions:
Sorry I did not see a question other than a statement wrapped in a question.
The reply to the your question about storing the offsets in the mbuf layer is just extra data and APIs we have to test. The mbuf pool would have to carry this information or some way to associate the metadata of the metadata to the given mbuf. My point is the type is carried with the mbuf and then we have no question as to the type of metadata contained in the mbuf. Having metadata for the metadata for the application to grab or use even more macros or inlines its not going to make it easier for the developer only more complex. A type field will tell you exactly what and were the metadata is located in the mbuf header.
>
> - what happens if you need a field from layout1 and another from layout2?
> (ex: timestamp + ipsec, timestamp + seqn, seqn + ipsec, …)
At this point you need to be smart and use a single type instead of trying to merge two or three types or use cases. The type defines the valid fields, if we are changing the fields from one to another in a single mbuf instance or packet instance that will not happen in the cases you defined above. Multi-mbufs from say a PMD or application is not going to need to combine all of the above examples.
> - how do you implement the rx/tx drivers functions if you have to support
> several layouts, where a field may be at a different offset?
>
>>>> The only things you would have to do is:
>>>>
>>>> 1/ ensure the offset is registered
>>>> rte_mbuf_dyn_timestamp_register()
>>>>
>>>> 2/ use helpers
>>>> rte_mbuf_dyn_timestamp_get(), rte_mbuf_dyn_timestamp_set(), ...
>>
>>> Konstantin’s suggest if I understand it was to use structures to allow the
>>> compiler to optimize the access to the mbuf and I like that idea, but with one
>>> change we add a field in the mbuf to define the mbuf structure type.
>>>
>>> Say 0 is the standard rte_mbuf type then type 1 could be the IPSec offset type
>>> mbuf, type 2 could be something else, … The type 0 looks just like the mbuf we
>>> have today with maybe the optional fields set to reserved or some type of
>>> filler variables to reserve the holes in the structure. Then type 1 is the
>>> IPSec mbuf and in the reserved sections of the mbuf contain the IPSec related
>>> data with the standard mbuf fields still matching the type 0 version.
>>
>> This very look like the "selective layout" in our presentation [1], page 14.
>>
>> Your example talks about IPsec, but someone else will want to use a
>> sequence number, another one a timestamp, and another one will want to
>> use this space for its own application. There are a lot of use cases,
>> and it does not scale to have a layout for each of them. Worst, if
>> someone wants IPsec + a sequence number, how can it work?
>>
>> One of the problem to solve is to avoid mutually exclusive feature (i.e.
>> union of fields that cannot be used together in the mbuf).
>>
>> This allows the mbuf to be used by the developer and the compiler now knows
>> exactly where the fields are located in the structure and does not have to
>> deal with any of the macros and offsets and registration suggested here. Just
>> cast the mbuf pointer into the new type mbuf structure. We just have to make
>> sure the code that needs to use a given mbuf type has access to the structure
>> definitions.
>>
>> With the current proposal, we can imagine an API to ask to register a
>> field at a specific offset. It can then be used in the application, so
>> that accesses are done at no cost compared to a static field, because
>> the offset would be const.
>>
>> In the driver, the same logic could be used, but dynamically:
>>
>> if (offset == PREFERRED_OFFSET) {
>> /* code with static offset */
>> } else {
>> /* generic code */
>> }
>>
>> But I'm not sure it would scale a lot if there are several features
>> using dynamic fields.
>>
>>> If the mbufs it going to be translated from one type mbuf to another mbuf
>>> type, we just have to define that type and then cast the mbuf pointer to that
>>> structure. When an mbuf is received from IPSec PMD then the application needs
>>> to forward that mbuf to the next stage it can reset the type to 0 or to
>>> another type filling in the reserved fields to be used by the next stage in
>>> the pipeline.
>>
>> What you describe is one use case.
>>
>> What could be done with the API mentionned above (but I think it is
>> dangerous), is to allow a user to register 2 different fields at the
>> same offset, using a specific flag. This could work if the user knows
>> that these 2 fields are never used at the same time.
>>
>> The mbuf now contains the type and every point in the application can look at
>> the type to determine how that mbuf is defined. I am sure there are some holes
>> here, but I think it is a better solution then using all of these macros,
>> offset values and registration APIs.
>>
>> I'm not convinced having selective layouts is doable. The layouts cannot
>> fit all possible use cases, and managing the different layouts in the
>> driver looks difficult to me. Additionnaly, it does not solve the
>> problem of mutually exclusive features.
>>
>> I too at one time wanted some type of allocation or registration for
>> private mbuf space and applying to these limited fields in the mbuf
>> header may have been reasonable. The problem is using registration and
>> moving that information between processes is going to be hard to get
>> right. For a single Appliance model application it would work great
>> and not in a non-appliance model applications.
>
> I didn't get why it wouldn't work in a non-appliance model (are you
> talking about primary/secondary processes?). Can you elaborate about
> waht would be the problem?
Let's look at something similar to VPP with nodes in a graph where each node needs to verify the metadata exists in that packet (extra macro/inline calls). This would be needed for every packet processed unless something else make sure only specific mbuf type is used. Why not just use a type field and cast the mbuf into the correct type structure and reject any mbufs that do not match the types that can be handled by this node. The type field would be a simple switch or if/else construct.
>
>> The type/structure
>> method can help and it could have problems too, but using a
>> type/struct design seems to be one of the BKMs (Best Known Methods) in
>> the industry.
>
> Sorry, but this is not a valid argument.
You assumption is not valid IMO. Sorry.
>> To be honest it maybe we just take the hit in performance and add a
>> third cache line as I am sure trying to squeeze metadata into these
>> very limit fields will be a challenge IMO. I am not suggesting we add
>> a cache line to every mbuf only to the pools that require the extra
>> metadata by using the private space if that is reasonable. The
>> applications needing a lot of metadata will just have to take the hit
>> in performance anyway.
>
> If an application wants to attach more data in the mbuf, we already have
> the application private area. This zone is transparent from DPDK point
> of view, it does not impact drivers or libs.
We still have to have all parts of the system which accesses the metadata to know the private data exist and what it's given format. You solved this with offsets and registration to define the format. My suggestion is similar as it defines the location and type of metadata in a type/struct format. I feel it is easier to understand and too process in a high performance way.
>
>> Having to grab a metadata value via a set of macros and inline
>> functions seems like it will consume more cycles then just a
>> type/structure method as the compiler will help optimize the code
>> without having to call any macros or inline functions.
>
> Yes, I know that. This is the price to pay for solving the problems
> (wasted size, exclusive features, avoid abi breakage). I answered in a
> previous mail that the extra cost can be removed at application level if
> we add an API to reserve a known offset. The ability to locate some
> offload fields in the Rx part may also help to gain some cycles compared
> to static fields.
This maybe true, but I do not see how the cycles can be removed unless you create a structure layout in the application to access the metadata. Just saying it can be solved is OK, but proving it can be solved is the real question.
Anyway I am not going to argue with you, the community can decide if your solution solves the problem. In my case I do not see it solving it in the best way and my suggestion may not be the best either.
>
> Regards,
> Olivier
Regards,
Keith
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] RFC: hiding struct rte_eth_dev
2019-09-23 16:19 5% [dpdk-dev] RFC: hiding struct rte_eth_dev Ray Kinsella
@ 2019-09-23 16:35 0% ` Bruce Richardson
2019-09-24 9:07 0% ` Morten Brørup
2019-09-24 16:42 0% ` Jerin Jacob
2 siblings, 0 replies; 200+ results
From: Bruce Richardson @ 2019-09-23 16:35 UTC (permalink / raw)
To: Ray Kinsella
Cc: dpdk-dev, Jerin Jacob Kollanukkaran, Hemant Agrawal,
Thomas Monjalon, Stephen Hemminger, Yigit, Ferruh, Ananyev,
Konstantin, maxime.coquelin, David Marchand, Marcin Zapolski
On Mon, Sep 23, 2019 at 05:19:27PM +0100, Ray Kinsella wrote:
> Hi folks,
>
> The ABI Stability proposals should be pretty well known at this point.
> The latest rev is here ...
>
> http://inbox.dpdk.org/dev/1565864619-17206-1-git-send-email-mdr@ashroe.eu/
>
> As has been discussed public data structure's are risky for ABI
> stability, as any changes to a data structure can change the ABI. As a
> general rule you want to expose as few as possible (ideally none), and
> keep them as small as possible.
>
> One of the key data structures in DPDK is `struct rte_eth_dev`. In this
> case, rte_eth_dev is exposed public-ally, as a side-effect of the
> inlining of the [rx,tx]_burst functions.
>
> Marcin Zapolski has been looking at what to do about it, with no current
> consensus on a path forward. The options on our table is:-
>
> 1. Do nothing, live with the risk to DPDK v20 ABI stability.
>
> 2. Pad rte_eth_dev, add some extra bytes to the structure "in case" we
> need to add a field during the v20 ABI (through to 20.11).
>
> 3. Break rte_eth_dev into public and private structs.
> - See
> http://inbox.dpdk.org/dev/20190906131813.1343-1-marcinx.a.zapolski@intel.com/
> - This ends up quiet an invasive patch, late in the cycle, however it
> does have no performance penalty.
>
> 4. Uninline [rx,tx]_burst functions
> - See
> http://inbox.dpdk.org/dev/20190730124950.1293-1-marcinx.a.zapolski@intel.com/
> - This has a performance penalty of ~2% with testpmd, impact on a "real
> workload" is likely to be in the noise.
>
> We need to agree an approach for v19.11, and that may be we agree to do
> nothing. My personal vote is 4. as the simplest with minimal impact.
>
Thanks for calling out these potential options, Ray.
#4, uninlining, would also be my preference, though I think #1, do nothing,
is probably ok and could live with #2, adding padding, if others like the
idea. While #3, splitting structures, has advantages, I just dislike how
invasive it is, and don't think it's a good candidate for 19.11.
/Bruce
^ permalink raw reply [relevance 0%]
* [dpdk-dev] RFC: hiding struct rte_eth_dev
@ 2019-09-23 16:19 5% Ray Kinsella
2019-09-23 16:35 0% ` Bruce Richardson
` (2 more replies)
0 siblings, 3 replies; 200+ results
From: Ray Kinsella @ 2019-09-23 16:19 UTC (permalink / raw)
To: dpdk-dev
Cc: Richardson, Bruce, Jerin Jacob Kollanukkaran, Hemant Agrawal,
Thomas Monjalon, Stephen Hemminger, Yigit, Ferruh, Ananyev,
Konstantin, maxime.coquelin, David Marchand, Marcin Zapolski
Hi folks,
The ABI Stability proposals should be pretty well known at this point.
The latest rev is here ...
http://inbox.dpdk.org/dev/1565864619-17206-1-git-send-email-mdr@ashroe.eu/
As has been discussed public data structure's are risky for ABI
stability, as any changes to a data structure can change the ABI. As a
general rule you want to expose as few as possible (ideally none), and
keep them as small as possible.
One of the key data structures in DPDK is `struct rte_eth_dev`. In this
case, rte_eth_dev is exposed public-ally, as a side-effect of the
inlining of the [rx,tx]_burst functions.
Marcin Zapolski has been looking at what to do about it, with no current
consensus on a path forward. The options on our table is:-
1. Do nothing, live with the risk to DPDK v20 ABI stability.
2. Pad rte_eth_dev, add some extra bytes to the structure "in case" we
need to add a field during the v20 ABI (through to 20.11).
3. Break rte_eth_dev into public and private structs.
- See
http://inbox.dpdk.org/dev/20190906131813.1343-1-marcinx.a.zapolski@intel.com/
- This ends up quiet an invasive patch, late in the cycle, however it
does have no performance penalty.
4. Uninline [rx,tx]_burst functions
- See
http://inbox.dpdk.org/dev/20190730124950.1293-1-marcinx.a.zapolski@intel.com/
- This has a performance penalty of ~2% with testpmd, impact on a "real
workload" is likely to be in the noise.
We need to agree an approach for v19.11, and that may be we agree to do
nothing. My personal vote is 4. as the simplest with minimal impact.
Thanks,
Ray K
^ permalink raw reply [relevance 5%]
* Re: [dpdk-dev] [PATCH] mbuf: support dynamic fields and flags
2019-09-23 15:14 2% ` Wiles, Keith
@ 2019-09-23 16:16 3% ` Olivier Matz
2019-09-23 17:14 0% ` Wiles, Keith
0 siblings, 1 reply; 200+ results
From: Olivier Matz @ 2019-09-23 16:16 UTC (permalink / raw)
To: Wiles, Keith
Cc: dev, Thomas Monjalon, Wang, Haiyue, Stephen Hemminger,
Andrew Rybchenko, Jerin Jacob Kollanukkaran
Hi,
(reformated the quotes)
On Mon, Sep 23, 2019 at 03:14:51PM +0000, Wiles, Keith wrote:
>
>
> On Sep 23, 2019, at 4:13 AM, Olivier Matz <olivier.matz@6wind.com<mailto:olivier.matz@6wind.com>> wrote:
> >
> > Hi Keith,
> >
> > On Sat, Sep 21, 2019 at 08:28:32AM +0000, Wiles, Keith wrote:
> > >
> > >
> > > On Sep 18, 2019, at 6:54 PM, Olivier Matz <olivier.matz@6wind.com<mailto:olivier.matz@6wind.com>> wrote:
> > >
> > > > Many features require to store data inside the mbuf. As the room in mbuf
> > > > structure is limited, it is not possible to have a field for each
> > > > feature. Also, changing fields in the mbuf structure can break the API
> > > > or ABI.
> > > >
> > > > This commit addresses these issues, by enabling the dynamic registration
> > > > of fields or flags:
> > > >
> > > > - a dynamic field is a named area in the rte_mbuf structure, with a
> > > > given size (>= 1 byte) and alignment constraint.
> > > > - a dynamic flag is a named bit in the rte_mbuf structure.
> > > >
> > > > The typical use case is a PMD that registers space for an offload
> > > > feature, when the application requests to enable this feature. As
> > > > the space in mbuf is limited, the space should only be reserved if it
> > > > is going to be used (i.e when the application explicitly asks for it).
> > > >
> > > > The registration can be done at any moment, but it is not possible
> > > > to unregister fields or flags for now.
> > > >
> > > > Signed-off-by: Olivier Matz <olivier.matz@6wind.com<mailto:olivier.matz@6wind.com>>
> > > > Acked-by: Thomas Monjalon <thomas@monjalon.net<mailto:thomas@monjalon.net>>
> > > —
> > >
> > >
> > >
> > > The idea of registration for space in the mbuf I am not a big fan. I did like
> > > Konstantin’s suggestion of having the compiler help with optimizing the code,
> > > but with a slight difference. Maybe I misunderstand, but now with this design
> > > you have to pass the offsets to different parts of the application or place in
> > > global memory or have each section request the offsets. It seems great if the
> > > application is one big application or an appliance model application having
> > > control of the whole design not so good for service chains like designs where
> > > different parts of the whole application is design by different teams.
> >
> > If the global variable storing the offset is defined in the mbuf layer, what
> > would be the problem?
>
> Are you assuming the values are shared between primary/secondary model or
> between processes using shared memory? If moving the packet data via shared
> memory to a different application written by a different company you still
> have to move that metadata.
The dynamic mbuf proposal works with secondary processes. What does that
change if the application is written by a different company? If you need
to store a timestamp, you register the timestamp and the offset will be
the same in primary and secondary.
> If the type was carried with the mbuf we can easily convey a small
> type value or we would need to tell the other side we have all of this
> registration information to send. I would suggest the number of mbuf
> types will be small over time and I believe a 4 bit or 8 bit type is
> reasonable. In many protocols using a type value is used to convey
> this type of information. We can even tightly control the number of
> types DPDK controls and then leave some for user defined if we like.
8 bits means 256 different mbuf layouts.
You did not replied to my previous questions:
- what happens if you need a field from layout1 and another from layout2?
(ex: timestamp + ipsec, timestamp + seqn, seqn + ipsec, ...)
- how do you implement the rx/tx drivers functions if you have to support
several layouts, where a field may be at a different offset?
> > > The only things you would have to do is:
> > >
> > > 1/ ensure the offset is registered
> > > rte_mbuf_dyn_timestamp_register()
> > >
> > > 2/ use helpers
> > > rte_mbuf_dyn_timestamp_get(), rte_mbuf_dyn_timestamp_set(), ...
>
> > Konstantin’s suggest if I understand it was to use structures to allow the
> > compiler to optimize the access to the mbuf and I like that idea, but with one
> > change we add a field in the mbuf to define the mbuf structure type.
> >
> > Say 0 is the standard rte_mbuf type then type 1 could be the IPSec offset type
> > mbuf, type 2 could be something else, … The type 0 looks just like the mbuf we
> > have today with maybe the optional fields set to reserved or some type of
> > filler variables to reserve the holes in the structure. Then type 1 is the
> > IPSec mbuf and in the reserved sections of the mbuf contain the IPSec related
> > data with the standard mbuf fields still matching the type 0 version.
>
> This very look like the "selective layout" in our presentation [1], page 14.
>
> Your example talks about IPsec, but someone else will want to use a
> sequence number, another one a timestamp, and another one will want to
> use this space for its own application. There are a lot of use cases,
> and it does not scale to have a layout for each of them. Worst, if
> someone wants IPsec + a sequence number, how can it work?
>
> One of the problem to solve is to avoid mutually exclusive feature (i.e.
> union of fields that cannot be used together in the mbuf).
>
> This allows the mbuf to be used by the developer and the compiler now knows
> exactly where the fields are located in the structure and does not have to
> deal with any of the macros and offsets and registration suggested here. Just
> cast the mbuf pointer into the new type mbuf structure. We just have to make
> sure the code that needs to use a given mbuf type has access to the structure
> definitions.
>
> With the current proposal, we can imagine an API to ask to register a
> field at a specific offset. It can then be used in the application, so
> that accesses are done at no cost compared to a static field, because
> the offset would be const.
>
> In the driver, the same logic could be used, but dynamically:
>
> if (offset == PREFERRED_OFFSET) {
> /* code with static offset */
> } else {
> /* generic code */
> }
>
> But I'm not sure it would scale a lot if there are several features
> using dynamic fields.
>
> > If the mbufs it going to be translated from one type mbuf to another mbuf
> > type, we just have to define that type and then cast the mbuf pointer to that
> > structure. When an mbuf is received from IPSec PMD then the application needs
> > to forward that mbuf to the next stage it can reset the type to 0 or to
> > another type filling in the reserved fields to be used by the next stage in
> > the pipeline.
>
> What you describe is one use case.
>
> What could be done with the API mentionned above (but I think it is
> dangerous), is to allow a user to register 2 different fields at the
> same offset, using a specific flag. This could work if the user knows
> that these 2 fields are never used at the same time.
>
> The mbuf now contains the type and every point in the application can look at
> the type to determine how that mbuf is defined. I am sure there are some holes
> here, but I think it is a better solution then using all of these macros,
> offset values and registration APIs.
>
> I'm not convinced having selective layouts is doable. The layouts cannot
> fit all possible use cases, and managing the different layouts in the
> driver looks difficult to me. Additionnaly, it does not solve the
> problem of mutually exclusive features.
>
> I too at one time wanted some type of allocation or registration for
> private mbuf space and applying to these limited fields in the mbuf
> header may have been reasonable. The problem is using registration and
> moving that information between processes is going to be hard to get
> right. For a single Appliance model application it would work great
> and not in a non-appliance model applications.
I didn't get why it wouldn't work in a non-appliance model (are you
talking about primary/secondary processes?). Can you elaborate about
waht would be the problem?
> The type/structure
> method can help and it could have problems too, but using a
> type/struct design seems to be one of the BKMs (Best Known Methods) in
> the industry.
Sorry, but this is not a valid argument.
> To be honest it maybe we just take the hit in performance and add a
> third cache line as I am sure trying to squeeze metadata into these
> very limit fields will be a challenge IMO. I am not suggesting we add
> a cache line to every mbuf only to the pools that require the extra
> metadata by using the private space if that is reasonable. The
> applications needing a lot of metadata will just have to take the hit
> in performance anyway.
If an application wants to attach more data in the mbuf, we already have
the application private area. This zone is transparent from DPDK point
of view, it does not impact drivers or libs.
> Having to grab a metadata value via a set of macros and inline
> functions seems like it will consume more cycles then just a
> type/structure method as the compiler will help optimize the code
> without having to call any macros or inline functions.
Yes, I know that. This is the price to pay for solving the problems
(wasted size, exclusive features, avoid abi breakage). I answered in a
previous mail that the extra cost can be removed at application level if
we add an API to reserve a known offset. The ability to locate some
offload fields in the Rx part may also help to gain some cycles compared
to static fields.
Regards,
Olivier
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] [PATCH] mbuf: support dynamic fields and flags
2019-09-23 9:13 0% ` Olivier Matz
2019-09-23 15:14 2% ` Wiles, Keith
@ 2019-09-23 16:09 0% ` Wiles, Keith
1 sibling, 0 replies; 200+ results
From: Wiles, Keith @ 2019-09-23 16:09 UTC (permalink / raw)
To: Olivier Matz
Cc: dev, Thomas Monjalon, Wang, Haiyue, Stephen Hemminger,
Andrew Rybchenko, Jerin Jacob Kollanukkaran
Sorry, resend in plain text :-(
> On Sep 23, 2019, at 4:13 AM, Olivier Matz <olivier.matz@6wind.com> wrote:
>
> Hi Keith,
>
> On Sat, Sep 21, 2019 at 08:28:32AM +0000, Wiles, Keith wrote:
>>
>>
>>> On Sep 18, 2019, at 6:54 PM, Olivier Matz <olivier.matz@6wind.com> wrote:
>>>
>>> Many features require to store data inside the mbuf. As the room in mbuf
>>> structure is limited, it is not possible to have a field for each
>>> feature. Also, changing fields in the mbuf structure can break the API
>>> or ABI.
>>>
>>> This commit addresses these issues, by enabling the dynamic registration
>>> of fields or flags:
>>>
>>> - a dynamic field is a named area in the rte_mbuf structure, with a
>>> given size (>= 1 byte) and alignment constraint.
>>> - a dynamic flag is a named bit in the rte_mbuf structure.
>>>
>>> The typical use case is a PMD that registers space for an offload
>>> feature, when the application requests to enable this feature. As
>>> the space in mbuf is limited, the space should only be reserved if it
>>> is going to be used (i.e when the application explicitly asks for it).
>>>
>>> The registration can be done at any moment, but it is not possible
>>> to unregister fields or flags for now.
>>>
>>> Signed-off-by: Olivier Matz <olivier.matz@6wind.com>
>>> Acked-by: Thomas Monjalon <thomas@monjalon.net>
>>> —
>>>
>>
>
>> The idea of registration for space in the mbuf I am not a big fan. I did like
>> Konstantin’s suggestion of having the compiler help with optimizing the code,
>> but with a slight difference. Maybe I misunderstand, but now with this design
>> you have to pass the offsets to different parts of the application or place in
>> global memory or have each section request the offsets. It seems great if the
>> application is one big application or an appliance model application having
>> control of the whole design not so good for service chains like designs where
>> different parts of the whole application is design by different teams.
>
> If the global variable storing the offset is defined in the mbuf layer, what
> would be the problem?
>
Are you assuming the values are shared between primary/secondary model or between processes using shared memory? If moving the packet data via shared memory to a different application written by a different company you still have to move that metadata. If the type was carried with the mbuf we can easily convey a small type value or we would need to tell the other side we have all of this registration information to send. I would suggest the number of mbuf types will be small over time and I believe a 4 bit or 8 bit type is reasonable. In many protocols using a type value is used to convey this type of information. We can even tightly control the number of types DPDK controls and then leave some for user defined if we like.
> The only things you would have to do is:
>
> 1/ ensure the offset is registered
> rte_mbuf_dyn_timestamp_register()
>
> 2/ use helpers
> rte_mbuf_dyn_timestamp_get(), rte_mbuf_dyn_timestamp_set(), ...
>
>> Konstantin’s suggest if I understand it was to use structures to allow the
>> compiler to optimize the access to the mbuf and I like that idea, but with one
>> change we add a field in the mbuf to define the mbuf structure type.
>>
>> Say 0 is the standard rte_mbuf type then type 1 could be the IPSec offset type
>> mbuf, type 2 could be something else, … The type 0 looks just like the mbuf we
>> have today with maybe the optional fields set to reserved or some type of
>> filler variables to reserve the holes in the structure. Then type 1 is the
>> IPSec mbuf and in the reserved sections of the mbuf contain the IPSec related
>> data with the standard mbuf fields still matching the type 0 version.
>
> This very look like the "selective layout" in our presentation [1], page 14.
>
> Your example talks about IPsec, but someone else will want to use a
> sequence number, another one a timestamp, and another one will want to
> use this space for its own application. There are a lot of use cases,
> and it does not scale to have a layout for each of them. Worst, if
> someone wants IPsec + a sequence number, how can it work?
>
> One of the problem to solve is to avoid mutually exclusive feature (i.e.
> union of fields that cannot be used together in the mbuf).
>
>> This allows the mbuf to be used by the developer and the compiler now knows
>> exactly where the fields are located in the structure and does not have to
>> deal with any of the macros and offsets and registration suggested here. Just
>> cast the mbuf pointer into the new type mbuf structure. We just have to make
>> sure the code that needs to use a given mbuf type has access to the structure
>> definitions.
>
> With the current proposal, we can imagine an API to ask to register a
> field at a specific offset. It can then be used in the application, so
> that accesses are done at no cost compared to a static field, because
> the offset would be const.
>
> In the driver, the same logic could be used, but dynamically:
>
> if (offset == PREFERRED_OFFSET) {
> /* code with static offset */
> } else {
> /* generic code */
> }
>
> But I'm not sure it would scale a lot if there are several features
> using dynamic fields.
>
>> If the mbufs it going to be translated from one type mbuf to another mbuf
>> type, we just have to define that type and then cast the mbuf pointer to that
>> structure. When an mbuf is received from IPSec PMD then the application needs
>> to forward that mbuf to the next stage it can reset the type to 0 or to
>> another type filling in the reserved fields to be used by the next stage in
>> the pipeline.
>
> What you describe is one use case.
>
> What could be done with the API mentionned above (but I think it is
> dangerous), is to allow a user to register 2 different fields at the
> same offset, using a specific flag. This could work if the user knows
> that these 2 fields are never used at the same time.
>
>> The mbuf now contains the type and every point in the application can look at
>> the type to determine how that mbuf is defined. I am sure there are some holes
>> here, but I think it is a better solution then using all of these macros,
>> offset values and registration APIs.
>
> I'm not convinced having selective layouts is doable. The layouts cannot
> fit all possible use cases, and managing the different layouts in the
> driver looks difficult to me. Additionnaly, it does not solve the
> problem of mutually exclusive features.
>
I too at one time wanted some type of allocation or registration for private mbuf space and applying to these limited fields in the mbuf header may have been reasonable. The problem is using registration and moving that information between processes is going to be hard to get right. For a single Appliance model application it would work great and not in a non-appliance model applications. The type/structure method can help and it could have problems too, but using a type/struct design seems to be one of the BKMs (Best Known Methods) in the industry.
To be honest it maybe we just take the hit in performance and add a third cache line as I am sure trying to squeeze metadata into these very limit fields will be a challenge IMO. I am not suggesting we add a cache line to every mbuf only to the pools that require the extra metadata by using the private space if that is reasonable. The applications needing a lot of metadata will just have to take the hit in performance anyway.
Having to grab a metadata value via a set of macros and inline functions seems like it will consume more cycles then just a type/structure method as the compiler will help optimize the code without having to call any macros or inline functions.
>
> Thanks for the feedback.
> Olivier
>
> [1] https://static.sched.com/hosted_files/dpdkbordeaux2019/2b/dpdk-201909-dyn-mbuf.pdf
Regards,
Keith
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [PATCH] vhost: add __rte_experimental to rte_vhost_va_from_guest_pa
@ 2019-09-23 15:41 3% ` Ferruh Yigit
0 siblings, 0 replies; 200+ results
From: Ferruh Yigit @ 2019-09-23 15:41 UTC (permalink / raw)
To: Maxime Coquelin, Jim Harris, dev, tiwei.bie, zhihong.wang
On 9/18/2019 2:12 PM, Maxime Coquelin wrote:
>
>
> On 8/20/19 11:37 AM, Jim Harris wrote:
>> This function is listed under EXPERIMENTAL in the
>> rte_vhost_version.map, so it needs to be marked
>> with __rte_experimental in the header file as well.
>>
>> Found by check-experimental-syms.sh when trying to compile
>> DPDK with -finstrument-functions. This script didn't
>> catch this in the normal case, since the function is
>> declared __rte_always_inline.
>>
>> Signed-off-by: Jim Harris <james.r.harris@intel.com>
>> ---
>> lib/librte_vhost/rte_vhost.h | 1 +
>> 1 file changed, 1 insertion(+)
>>
>> diff --git a/lib/librte_vhost/rte_vhost.h b/lib/librte_vhost/rte_vhost.h
>> index 7fb172912..fc27bc21e 100644
>> --- a/lib/librte_vhost/rte_vhost.h
>> +++ b/lib/librte_vhost/rte_vhost.h
>> @@ -225,6 +225,7 @@ rte_vhost_gpa_to_vva(struct rte_vhost_memory *mem, uint64_t gpa)
>> * @return
>> * the host virtual address on success, 0 on failure
>> */
>> +__rte_experimental
>> static __rte_always_inline uint64_t
>> rte_vhost_va_from_guest_pa(struct rte_vhost_memory *mem,
>> uint64_t gpa, uint64_t *len)
>>
>
> Fixed commit message to comply with check-git-log tool.
>
> Applied to dpdk-next-virtio/master.
This is breaking the 'vhost_scsi' sample application since it is using this
experimental API [1].
Build system should be updated to say sample application is allowed to using
experimental APIs.
Can you please send a new version?
[1]
.../dpdk/examples/vhost_scsi/vhost_scsi.c: In function ‘gpa_to_vva’:
.../dpdk/examples/vhost_scsi/vhost_scsi.c:61:2: error:
‘rte_vhost_va_from_guest_pa’ is deprecated: Symbol is not yet part of stable ABI
[-Werror=deprecated-declarations]
61 | return rte_vhost_va_from_guest_pa(ctrlr->mem, gpa, len);
| ^~~~~~
In file included from .../dpdk/examples/vhost_scsi/vhost_scsi.c:18:
.../dpdk/build32/include/rte_vhost.h:238:1: note: declared here
238 | rte_vhost_va_from_guest_pa(struct rte_vhost_memory *mem,
| ^~~~~~~~~~~~~~~~~~~~~~~~~~
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] [PATCH] mbuf: support dynamic fields and flags
2019-09-23 9:13 0% ` Olivier Matz
@ 2019-09-23 15:14 2% ` Wiles, Keith
2019-09-23 16:16 3% ` Olivier Matz
2019-09-23 16:09 0% ` Wiles, Keith
1 sibling, 1 reply; 200+ results
From: Wiles, Keith @ 2019-09-23 15:14 UTC (permalink / raw)
To: Olivier Matz
Cc: dev, Thomas Monjalon, Wang, Haiyue, Stephen Hemminger,
Andrew Rybchenko, Jerin Jacob Kollanukkaran
On Sep 23, 2019, at 4:13 AM, Olivier Matz <olivier.matz@6wind.com<mailto:olivier.matz@6wind.com>> wrote:
Hi Keith,
On Sat, Sep 21, 2019 at 08:28:32AM +0000, Wiles, Keith wrote:
On Sep 18, 2019, at 6:54 PM, Olivier Matz <olivier.matz@6wind.com<mailto:olivier.matz@6wind.com>> wrote:
Many features require to store data inside the mbuf. As the room in mbuf
structure is limited, it is not possible to have a field for each
feature. Also, changing fields in the mbuf structure can break the API
or ABI.
This commit addresses these issues, by enabling the dynamic registration
of fields or flags:
- a dynamic field is a named area in the rte_mbuf structure, with a
given size (>= 1 byte) and alignment constraint.
- a dynamic flag is a named bit in the rte_mbuf structure.
The typical use case is a PMD that registers space for an offload
feature, when the application requests to enable this feature. As
the space in mbuf is limited, the space should only be reserved if it
is going to be used (i.e when the application explicitly asks for it).
The registration can be done at any moment, but it is not possible
to unregister fields or flags for now.
Signed-off-by: Olivier Matz <olivier.matz@6wind.com<mailto:olivier.matz@6wind.com>>
Acked-by: Thomas Monjalon <thomas@monjalon.net<mailto:thomas@monjalon.net>>
—
The idea of registration for space in the mbuf I am not a big fan. I did like
Konstantin’s suggestion of having the compiler help with optimizing the code,
but with a slight difference. Maybe I misunderstand, but now with this design
you have to pass the offsets to different parts of the application or place in
global memory or have each section request the offsets. It seems great if the
application is one big application or an appliance model application having
control of the whole design not so good for service chains like designs where
different parts of the whole application is design by different teams.
If the global variable storing the offset is defined in the mbuf layer, what
would be the problem?
Are you assuming the values are shared between primary/secondary model or between processes using shared memory? If moving the packet data via shared memory to a different application written by a different company you still have to move that metadata. If the type was carried with the mbuf we can easily convey a small type value or we would need to tell the other side we have all of this registration information to send. I would suggest the number of mbuf types will be small over time and I believe a 4 bit or 8 bit type is reasonable. In many protocols using a type value is used to convey this type of information. We can even tightly control the number of types DPDK controls and then leave some for user defined if we like.
The only things you would have to do is:
1/ ensure the offset is registered
rte_mbuf_dyn_timestamp_register()
2/ use helpers
rte_mbuf_dyn_timestamp_get(), rte_mbuf_dyn_timestamp_set(), ...
Konstantin’s suggest if I understand it was to use structures to allow the
compiler to optimize the access to the mbuf and I like that idea, but with one
change we add a field in the mbuf to define the mbuf structure type.
Say 0 is the standard rte_mbuf type then type 1 could be the IPSec offset type
mbuf, type 2 could be something else, … The type 0 looks just like the mbuf we
have today with maybe the optional fields set to reserved or some type of
filler variables to reserve the holes in the structure. Then type 1 is the
IPSec mbuf and in the reserved sections of the mbuf contain the IPSec related
data with the standard mbuf fields still matching the type 0 version.
This very look like the "selective layout" in our presentation [1], page 14.
Your example talks about IPsec, but someone else will want to use a
sequence number, another one a timestamp, and another one will want to
use this space for its own application. There are a lot of use cases,
and it does not scale to have a layout for each of them. Worst, if
someone wants IPsec + a sequence number, how can it work?
One of the problem to solve is to avoid mutually exclusive feature (i.e.
union of fields that cannot be used together in the mbuf).
This allows the mbuf to be used by the developer and the compiler now knows
exactly where the fields are located in the structure and does not have to
deal with any of the macros and offsets and registration suggested here. Just
cast the mbuf pointer into the new type mbuf structure. We just have to make
sure the code that needs to use a given mbuf type has access to the structure
definitions.
With the current proposal, we can imagine an API to ask to register a
field at a specific offset. It can then be used in the application, so
that accesses are done at no cost compared to a static field, because
the offset would be const.
In the driver, the same logic could be used, but dynamically:
if (offset == PREFERRED_OFFSET) {
/* code with static offset */
} else {
/* generic code */
}
But I'm not sure it would scale a lot if there are several features
using dynamic fields.
If the mbufs it going to be translated from one type mbuf to another mbuf
type, we just have to define that type and then cast the mbuf pointer to that
structure. When an mbuf is received from IPSec PMD then the application needs
to forward that mbuf to the next stage it can reset the type to 0 or to
another type filling in the reserved fields to be used by the next stage in
the pipeline.
What you describe is one use case.
What could be done with the API mentionned above (but I think it is
dangerous), is to allow a user to register 2 different fields at the
same offset, using a specific flag. This could work if the user knows
that these 2 fields are never used at the same time.
The mbuf now contains the type and every point in the application can look at
the type to determine how that mbuf is defined. I am sure there are some holes
here, but I think it is a better solution then using all of these macros,
offset values and registration APIs.
I'm not convinced having selective layouts is doable. The layouts cannot
fit all possible use cases, and managing the different layouts in the
driver looks difficult to me. Additionnaly, it does not solve the
problem of mutually exclusive features.
I too at one time wanted some type of allocation or registration for private mbuf space and applying to these limited fields in the mbuf header may have been reasonable. The problem is using registration and moving that information between processes is going to be hard to get right. For a single Appliance model application it would work great and not in a non-appliance model applications. The type/structure method can help and it could have problems too, but using a type/struct design seems to be one of the BKMs (Best Known Methods) in the industry.
To be honest it maybe we just take the hit in performance and add a third cache line as I am sure trying to squeeze metadata into these very limit fields will be a challenge IMO. I am not suggesting we add a cache line to every mbuf only to the pools that require the extra metadata by using the private space if that is reasonable. The applications needing a lot of metadata will just have to take the hit in performance anyway.
Having to grab a metadata value via a set of macros and inline functions seems like it will consume more cycles then just a type/structure method as the compiler will help optimize the code without having to call any macros or inline functions.
Thanks for the feedback.
Olivier
[1] https://static.sched.com/hosted_files/dpdkbordeaux2019/2b/dpdk-201909-dyn-mbuf.pdf
Regards,
Keith
^ permalink raw reply [relevance 2%]
* Re: [dpdk-dev] [PATCH] mbuf: support dynamic fields and flags
2019-09-23 8:31 0% ` Olivier Matz
@ 2019-09-23 11:01 0% ` Wang, Haiyue
0 siblings, 0 replies; 200+ results
From: Wang, Haiyue @ 2019-09-23 11:01 UTC (permalink / raw)
To: Olivier Matz
Cc: dev, Thomas Monjalon, Stephen Hemminger, Andrew Rybchenko, Wiles,
Keith, Jerin Jacob Kollanukkaran
> -----Original Message-----
> From: Olivier Matz [mailto:olivier.matz@6wind.com]
> Sent: Monday, September 23, 2019 16:32
> To: Wang, Haiyue <haiyue.wang@intel.com>
> Cc: dev@dpdk.org; Thomas Monjalon <thomas@monjalon.net>; Stephen Hemminger
> <stephen@networkplumber.org>; Andrew Rybchenko <arybchenko@solarflare.com>; Wiles, Keith
> <keith.wiles@intel.com>; Jerin Jacob Kollanukkaran <jerinj@marvell.com>
> Subject: Re: [PATCH] mbuf: support dynamic fields and flags
>
> Hi,
>
> On Sat, Sep 21, 2019 at 04:54:39AM +0000, Wang, Haiyue wrote:
> > > -----Original Message-----
> > > From: Olivier Matz [mailto:olivier.matz@6wind.com]
> > > Sent: Thursday, September 19, 2019 00:55
> > > To: dev@dpdk.org
> > > Cc: Thomas Monjalon <thomas@monjalon.net>; Wang, Haiyue <haiyue.wang@intel.com>; Stephen Hemminger
> > > <stephen@networkplumber.org>; Andrew Rybchenko <arybchenko@solarflare.com>; Wiles, Keith
> > > <keith.wiles@intel.com>; Jerin Jacob Kollanukkaran <jerinj@marvell.com>
> > > Subject: [PATCH] mbuf: support dynamic fields and flags
> > >
> > > Many features require to store data inside the mbuf. As the room in mbuf
> > > structure is limited, it is not possible to have a field for each
> > > feature. Also, changing fields in the mbuf structure can break the API
> > > or ABI.
> > >
> > > This commit addresses these issues, by enabling the dynamic registration
> > > of fields or flags:
> > >
> > > - a dynamic field is a named area in the rte_mbuf structure, with a
> > > given size (>= 1 byte) and alignment constraint.
> > > - a dynamic flag is a named bit in the rte_mbuf structure.
> > >
> > > The typical use case is a PMD that registers space for an offload
> > > feature, when the application requests to enable this feature. As
> > > the space in mbuf is limited, the space should only be reserved if it
> > > is going to be used (i.e when the application explicitly asks for it).
> > >
> > > The registration can be done at any moment, but it is not possible
> > > to unregister fields or flags for now.
> > >
> > > Signed-off-by: Olivier Matz <olivier.matz@6wind.com>
> > > Acked-by: Thomas Monjalon <thomas@monjalon.net>
> > > ---
> > >
> > > rfc -> v1
> > >
> > > * Rebase on top of master
> > > * Change registration API to use a structure instead of
> > > variables, getting rid of #defines (Stephen's comment)
> > > * Update flag registration to use a similar API as fields.
> > > * Change max name length from 32 to 64 (sugg. by Thomas)
> > > * Enhance API documentation (Haiyue's and Andrew's comments)
> > > * Add a debug log at registration
> > > * Add some words in release note
> > > * Did some performance tests (sugg. by Andrew):
> > > On my platform, reading a dynamic field takes ~3 cycles more
> > > than a static field, and ~2 cycles more for writing.
> > >
> > > app/test/test_mbuf.c | 114 ++++++-
> > > doc/guides/rel_notes/release_19_11.rst | 7 +
> > > lib/librte_mbuf/Makefile | 2 +
> > > lib/librte_mbuf/meson.build | 6 +-
> > > lib/librte_mbuf/rte_mbuf.h | 25 +-
> > > lib/librte_mbuf/rte_mbuf_dyn.c | 408 +++++++++++++++++++++++++
> > > lib/librte_mbuf/rte_mbuf_dyn.h | 163 ++++++++++
> > > lib/librte_mbuf/rte_mbuf_version.map | 4 +
> > > 8 files changed, 724 insertions(+), 5 deletions(-)
> > > create mode 100644 lib/librte_mbuf/rte_mbuf_dyn.c
> > > create mode 100644 lib/librte_mbuf/rte_mbuf_dyn.h
> > >
> >
> > [snip]
> >
> > > +/**
> > > + * Helper macro to access to a dynamic field.
> > > + */
> > > +#define RTE_MBUF_DYNFIELD(m, offset, type) ((type)((uintptr_t)(m) + (offset)))
> >
> > How about to change it as: ?
> > #define RTE_MBUF_DYNFIELD(m, offset, type) ((type *)((uintptr_t)(m) + (offset)))
> > ^
> > Then,
> > *RTE_MBUF_DYNFIELD(mb, xxx, uint32_t) = yyy;
> >
> > Since we use 'type' like: sizeof(type), __alignof__(type), this makes 'type' be
> > more consistent, not have to force cast 'type *' when using it.
> >
> > const struct rte_mbuf_dynfield dynfield2 = {
> > .name = "test-dynfield2",
> > .size = sizeof(uint16_t),
> > .align = __alignof__(uint16_t),
> > .flags = 0,
> > };
>
> Yes, I don't see use cases where the '*' is omitted, so it could be in the
> macro. On the other hand, doing like in the patch is more consistent with
> similar macros like rte_pktmbuf_mtod(), so I'll tend to keep it as is.
>
> This is maybe not that important, because this macro will often be hidden
> in a wrapper, like below:
>
> static inline uint64_t rte_mbuf_dyn_timestamp_get(const struct rte_mbuf *m)
> {
> return *RTE_MBUF_DYNFIELD(m, rte_mbuf_dynfield_timestamp_offset,
> uint64_t *);
> }
>
Thanks, yes, the same style as 'rte_pktmbuf_mtod', I didn't notice it.
>
> Thank you for the feedback!
>
> Olivier
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [RFC] Proposal to remove EXPERIMENTAL label from compressdev API
@ 2019-09-23 10:51 3% ` Trahe, Fiona
0 siblings, 0 replies; 200+ results
From: Trahe, Fiona @ 2019-09-23 10:51 UTC (permalink / raw)
To: akhil.goyal, ashish.gupta, shallyv, De Lara Guarch, Pablo, Daly,
Lee, Sunila Sahu, Trybula, ArturX, Dybkowski, AdamX, dev, Luse,
Paul E, Harris, James R
Cc: Trahe, Fiona
Hi all,
Taking into account
- some discussions I had in Bordeaux
- how difficult it will be to API/ABI breakage in non-experimental APIs future
- the proposed ARM compression PMD for LZ4 - which may discover a need for API change
- the absence of anyone crying out for the label to be removed
I'm going to defer this proposal until a later release.
Fiona
From: Trahe, Fiona
Sent: Tuesday, August 27, 2019 6:36 PM
To: akhil.goyal@nxp.com; ashish.gupta@marvell.com; shallyv@marvell.com; De Lara Guarch, Pablo <pablo.de.lara.guarch@intel.com>; Daly, Lee <lee.daly@intel.com>; Sunila Sahu <ssahu@marvell.com>; Trybula, ArturX <arturx.trybula@intel.com>; Dybkowski, AdamX <adamx.dybkowski@intel.com>; Trahe, Fiona <fiona.trahe@intel.com>; dev@dpdk.org; Luse, Paul E <paul.e.luse@intel.com>; Harris, James R <james.r.harris@intel.com>
Cc: Trahe, Fiona <fiona.trahe@intel.com>
Subject: [RFC] Proposal to remove EXPERIMENTAL label from compressdev API
Hi all,
Just putting it out there that the DPDK compressdev API has been available since 18.05, has had minimal change in recent releases and can be considered mature enough to have the experimental label removed.
I'd like to make this change in the 19.11 release.
Opinions?
Regards,
Fiona
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] [PATCH] mbuf: support dynamic fields and flags
2019-09-23 8:56 0% ` Morten Brørup
@ 2019-09-23 9:41 0% ` Olivier Matz
0 siblings, 0 replies; 200+ results
From: Olivier Matz @ 2019-09-23 9:41 UTC (permalink / raw)
To: Morten Brørup
Cc: Wiles, Keith, dev, Thomas Monjalon, Wang, Haiyue,
Stephen Hemminger, Andrew Rybchenko, Jerin Jacob Kollanukkaran,
bruce.richardson
Hi Morten,
On Mon, Sep 23, 2019 at 10:56:01AM +0200, Morten Brørup wrote:
> > -----Original Message-----
> > From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Wiles, Keith
> > Sent: Saturday, September 21, 2019 10:29 AM
> >
> > > On Sep 18, 2019, at 6:54 PM, Olivier Matz <olivier.matz@6wind.com>
> > wrote:
> > >
> > > Many features require to store data inside the mbuf. As the room in
> > mbuf
> > > structure is limited, it is not possible to have a field for each
> > > feature. Also, changing fields in the mbuf structure can break the
> > API
> > > or ABI.
> > >
> > > This commit addresses these issues, by enabling the dynamic
> > registration
> > > of fields or flags:
> > >
> > > - a dynamic field is a named area in the rte_mbuf structure, with a
> > > given size (>= 1 byte) and alignment constraint.
> > > - a dynamic flag is a named bit in the rte_mbuf structure.
> > >
> > > The typical use case is a PMD that registers space for an offload
> > > feature, when the application requests to enable this feature. As
> > > the space in mbuf is limited, the space should only be reserved if it
> > > is going to be used (i.e when the application explicitly asks for
> > it).
> > >
> > > The registration can be done at any moment, but it is not possible
> > > to unregister fields or flags for now.
> > >
> > > Signed-off-by: Olivier Matz <olivier.matz@6wind.com>
> > > Acked-by: Thomas Monjalon <thomas@monjalon.net>
> > > —
> > >
> >
> > The idea of registration for space in the mbuf I am not a big fan. I
> > did like Konstantin’s suggestion of having the compiler help with
> > optimizing the code, but with a slight difference. Maybe I
> > misunderstand, but now with this design you have to pass the offsets to
> > different parts of the application or place in global memory or have
> > each section request the offsets. It seems great if the application is
> > one big application or an appliance model application having control of
> > the whole design not so good for service chains like designs where
> > different parts of the whole application is design by different teams.
> >
> > Konstantin’s suggest if I understand it was to use structures to allow
> > the compiler to optimize the access to the mbuf and I like that idea,
> > but with one change we add a field in the mbuf to define the mbuf
> > structure type.
> >
> > Say 0 is the standard rte_mbuf type then type 1 could be the IPSec
> > offset type mbuf, type 2 could be something else, … The type 0 looks
> > just like the mbuf we have today with maybe the optional fields set to
> > reserved or some type of filler variables to reserve the holes in the
> > structure. Then type 1 is the IPSec mbuf and in the reserved sections
> > of the mbuf contain the IPSec related data with the standard mbuf
> > fields still matching the type 0 version.
> >
> > This allows the mbuf to be used by the developer and the compiler now
> > knows exactly where the fields are located in the structure and does
> > not have to deal with any of the macros and offsets and registration
> > suggested here. Just cast the mbuf pointer into the new type mbuf
> > structure. We just have to make sure the code that needs to use a given
> > mbuf type has access to the structure definitions.
> >
> > If the mbufs it going to be translated from one type mbuf to another
> > mbuf type, we just have to define that type and then cast the mbuf
> > pointer to that structure. When an mbuf is received from IPSec PMD then
> > the application needs to forward that mbuf to the next stage it can
> > reset the type to 0 or to another type filling in the reserved fields
> > to be used by the next stage in the pipeline.
> >
> > The mbuf now contains the type and every point in the application can
> > look at the type to determine how that mbuf is defined. I am sure there
> > are some holes here, but I think it is a better solution then using all
> > of these macros, offset values and registration APIs.
> >
> >
> > Regards,
> > Keith
>
>
> First of all, I applaud the idea of cleaning up the mbuf structure and
> removing the fields only rarely used and/or for special use cases only, as
> mentioned in the presentation, e.g. timestamp, timesync and seqn. It is great
> seeing serious effort put into improving the very core of DPDK!
>
> However, after some hallway discussions at DPDK Userspace and further thinking
> about the details, I can see two additional risks by introducing dynamic
> mbufs, which I would like to share for your consideration:
>
> 1. It may prevent us from adding future solutions not yet considered.
>
> If we were to introduce new functions for more granular handling of mbufs,
> similar to some of the Linux kernel's skbuff handling functions, how should
> such functions handle the dynamic fields? And how is the rte_pktmbuf_clone()
> function supposed to handle the dynamic fields? Some fields may need to be
> copied as-is, some may need to be initialized to zero or some other value, and
> so on. It is apparently not a problem now; but dynamic mbufs may prevent us
> from adding some of such functions in the future.
>
> I admit that I can only imagine the issue on an abstract level, so I can't
> give you a concrete example. Perhaps some of the more experienced Linux
> developers can provide one or debunk my concern. (Stephen: In relation to
> packet capturing we were discussing the reference counter not being respected
> by some applications, and the possible need for more granular mbuf handling.)
For now, the clone copies the fields. If we introduce a copy function, I
think it should do the same. Right now, I cannot find a use case where
the field should be set to another value, but yes, this could be a
limitation.
> 2. In the long run, we might end up adding more fields than we remove.
>
> Dynamic mbufs makes it easier to add specialized fields, which is great. But
> when the barrier to adding specialized fields is reduced, more PMDs and
> libraries may add their own unique fields, rather than going through a
> discussion and consensus for adding them to the fixed mbuf structure or
> finding some other solution. And if a multitude of PMDs each need a
> specialized field, PMDs might end up adding variants of a field, rather than
> reaching consensus for a common standard. It will be much easier to just add
> your own specialized mbuf field rather than having to go through the
> standardization process in the DPDK community.
>
> I will use the timestamp field as a theoretic example: The packet timestamp
> measurement unit differs between vendors, so with dynamic mbufs one vendor's
> PMD might create a timestamp_ns field, counting in nanoseconds, and another
> vendor's PMD might create a timestamp_clocks field, counting in clock
> counts. With the fixed mbuf, this triggered a public discussion, and a
> compromise for the field was reached.
In the mbuf structure, there is not a lot of room to describe some of
the fields, especially the ones that are inside a structure of union of
structure of union of union ;)
The definition of a dynamic field is in a dedicated structure, so there
is more room to describe it. For public dynamic fields, we have to be as
strict as for static fields, because it will be a public API. It has to
be correctly defined. About your timestamp example, we should not allow
2 different timestamp formats in the public API.
For private fields (Lib only, App only, PMD only), it's less critical
because it is not public, even if it's always good to have a clear
description.
> Although I am worried about dynamic mbufs, I don't have a better
> suggestion. And perhaps I just worry too much.
Feedback is always good to have, thanks.
Olivier
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [PATCH] mbuf: support dynamic fields and flags
2019-09-21 8:28 0% ` Wiles, Keith
2019-09-23 8:56 0% ` Morten Brørup
@ 2019-09-23 9:13 0% ` Olivier Matz
2019-09-23 15:14 2% ` Wiles, Keith
2019-09-23 16:09 0% ` Wiles, Keith
1 sibling, 2 replies; 200+ results
From: Olivier Matz @ 2019-09-23 9:13 UTC (permalink / raw)
To: Wiles, Keith
Cc: dev, Thomas Monjalon, Wang, Haiyue, Stephen Hemminger,
Andrew Rybchenko, Jerin Jacob Kollanukkaran
Hi Keith,
On Sat, Sep 21, 2019 at 08:28:32AM +0000, Wiles, Keith wrote:
>
>
> > On Sep 18, 2019, at 6:54 PM, Olivier Matz <olivier.matz@6wind.com> wrote:
> >
> > Many features require to store data inside the mbuf. As the room in mbuf
> > structure is limited, it is not possible to have a field for each
> > feature. Also, changing fields in the mbuf structure can break the API
> > or ABI.
> >
> > This commit addresses these issues, by enabling the dynamic registration
> > of fields or flags:
> >
> > - a dynamic field is a named area in the rte_mbuf structure, with a
> > given size (>= 1 byte) and alignment constraint.
> > - a dynamic flag is a named bit in the rte_mbuf structure.
> >
> > The typical use case is a PMD that registers space for an offload
> > feature, when the application requests to enable this feature. As
> > the space in mbuf is limited, the space should only be reserved if it
> > is going to be used (i.e when the application explicitly asks for it).
> >
> > The registration can be done at any moment, but it is not possible
> > to unregister fields or flags for now.
> >
> > Signed-off-by: Olivier Matz <olivier.matz@6wind.com>
> > Acked-by: Thomas Monjalon <thomas@monjalon.net>
> > —
> >
>
> The idea of registration for space in the mbuf I am not a big fan. I did like
> Konstantin’s suggestion of having the compiler help with optimizing the code,
> but with a slight difference. Maybe I misunderstand, but now with this design
> you have to pass the offsets to different parts of the application or place in
> global memory or have each section request the offsets. It seems great if the
> application is one big application or an appliance model application having
> control of the whole design not so good for service chains like designs where
> different parts of the whole application is design by different teams.
If the global variable storing the offset is defined in the mbuf layer, what
would be the problem?
The only things you would have to do is:
1/ ensure the offset is registered
rte_mbuf_dyn_timestamp_register()
2/ use helpers
rte_mbuf_dyn_timestamp_get(), rte_mbuf_dyn_timestamp_set(), ...
> Konstantin’s suggest if I understand it was to use structures to allow the
> compiler to optimize the access to the mbuf and I like that idea, but with one
> change we add a field in the mbuf to define the mbuf structure type.
>
> Say 0 is the standard rte_mbuf type then type 1 could be the IPSec offset type
> mbuf, type 2 could be something else, … The type 0 looks just like the mbuf we
> have today with maybe the optional fields set to reserved or some type of
> filler variables to reserve the holes in the structure. Then type 1 is the
> IPSec mbuf and in the reserved sections of the mbuf contain the IPSec related
> data with the standard mbuf fields still matching the type 0 version.
This very look like the "selective layout" in our presentation [1], page 14.
Your example talks about IPsec, but someone else will want to use a
sequence number, another one a timestamp, and another one will want to
use this space for its own application. There are a lot of use cases,
and it does not scale to have a layout for each of them. Worst, if
someone wants IPsec + a sequence number, how can it work?
One of the problem to solve is to avoid mutually exclusive feature (i.e.
union of fields that cannot be used together in the mbuf).
> This allows the mbuf to be used by the developer and the compiler now knows
> exactly where the fields are located in the structure and does not have to
> deal with any of the macros and offsets and registration suggested here. Just
> cast the mbuf pointer into the new type mbuf structure. We just have to make
> sure the code that needs to use a given mbuf type has access to the structure
> definitions.
With the current proposal, we can imagine an API to ask to register a
field at a specific offset. It can then be used in the application, so
that accesses are done at no cost compared to a static field, because
the offset would be const.
In the driver, the same logic could be used, but dynamically:
if (offset == PREFERRED_OFFSET) {
/* code with static offset */
} else {
/* generic code */
}
But I'm not sure it would scale a lot if there are several features
using dynamic fields.
> If the mbufs it going to be translated from one type mbuf to another mbuf
> type, we just have to define that type and then cast the mbuf pointer to that
> structure. When an mbuf is received from IPSec PMD then the application needs
> to forward that mbuf to the next stage it can reset the type to 0 or to
> another type filling in the reserved fields to be used by the next stage in
> the pipeline.
What you describe is one use case.
What could be done with the API mentionned above (but I think it is
dangerous), is to allow a user to register 2 different fields at the
same offset, using a specific flag. This could work if the user knows
that these 2 fields are never used at the same time.
> The mbuf now contains the type and every point in the application can look at
> the type to determine how that mbuf is defined. I am sure there are some holes
> here, but I think it is a better solution then using all of these macros,
> offset values and registration APIs.
I'm not convinced having selective layouts is doable. The layouts cannot
fit all possible use cases, and managing the different layouts in the
driver looks difficult to me. Additionnaly, it does not solve the
problem of mutually exclusive features.
Thanks for the feedback.
Olivier
[1] https://static.sched.com/hosted_files/dpdkbordeaux2019/2b/dpdk-201909-dyn-mbuf.pdf
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [PATCH] mbuf: support dynamic fields and flags
2019-09-21 8:28 0% ` Wiles, Keith
@ 2019-09-23 8:56 0% ` Morten Brørup
2019-09-23 9:41 0% ` Olivier Matz
2019-09-23 9:13 0% ` Olivier Matz
1 sibling, 1 reply; 200+ results
From: Morten Brørup @ 2019-09-23 8:56 UTC (permalink / raw)
To: Wiles, Keith, Olivier Matz
Cc: dev, Thomas Monjalon, Wang, Haiyue, Stephen Hemminger,
Andrew Rybchenko, Jerin Jacob Kollanukkaran, bruce.richardson
> -----Original Message-----
> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Wiles, Keith
> Sent: Saturday, September 21, 2019 10:29 AM
>
> > On Sep 18, 2019, at 6:54 PM, Olivier Matz <olivier.matz@6wind.com>
> wrote:
> >
> > Many features require to store data inside the mbuf. As the room in
> mbuf
> > structure is limited, it is not possible to have a field for each
> > feature. Also, changing fields in the mbuf structure can break the
> API
> > or ABI.
> >
> > This commit addresses these issues, by enabling the dynamic
> registration
> > of fields or flags:
> >
> > - a dynamic field is a named area in the rte_mbuf structure, with a
> > given size (>= 1 byte) and alignment constraint.
> > - a dynamic flag is a named bit in the rte_mbuf structure.
> >
> > The typical use case is a PMD that registers space for an offload
> > feature, when the application requests to enable this feature. As
> > the space in mbuf is limited, the space should only be reserved if it
> > is going to be used (i.e when the application explicitly asks for
> it).
> >
> > The registration can be done at any moment, but it is not possible
> > to unregister fields or flags for now.
> >
> > Signed-off-by: Olivier Matz <olivier.matz@6wind.com>
> > Acked-by: Thomas Monjalon <thomas@monjalon.net>
> > —
> >
>
> The idea of registration for space in the mbuf I am not a big fan. I
> did like Konstantin’s suggestion of having the compiler help with
> optimizing the code, but with a slight difference. Maybe I
> misunderstand, but now with this design you have to pass the offsets to
> different parts of the application or place in global memory or have
> each section request the offsets. It seems great if the application is
> one big application or an appliance model application having control of
> the whole design not so good for service chains like designs where
> different parts of the whole application is design by different teams.
>
> Konstantin’s suggest if I understand it was to use structures to allow
> the compiler to optimize the access to the mbuf and I like that idea,
> but with one change we add a field in the mbuf to define the mbuf
> structure type.
>
> Say 0 is the standard rte_mbuf type then type 1 could be the IPSec
> offset type mbuf, type 2 could be something else, … The type 0 looks
> just like the mbuf we have today with maybe the optional fields set to
> reserved or some type of filler variables to reserve the holes in the
> structure. Then type 1 is the IPSec mbuf and in the reserved sections
> of the mbuf contain the IPSec related data with the standard mbuf
> fields still matching the type 0 version.
>
> This allows the mbuf to be used by the developer and the compiler now
> knows exactly where the fields are located in the structure and does
> not have to deal with any of the macros and offsets and registration
> suggested here. Just cast the mbuf pointer into the new type mbuf
> structure. We just have to make sure the code that needs to use a given
> mbuf type has access to the structure definitions.
>
> If the mbufs it going to be translated from one type mbuf to another
> mbuf type, we just have to define that type and then cast the mbuf
> pointer to that structure. When an mbuf is received from IPSec PMD then
> the application needs to forward that mbuf to the next stage it can
> reset the type to 0 or to another type filling in the reserved fields
> to be used by the next stage in the pipeline.
>
> The mbuf now contains the type and every point in the application can
> look at the type to determine how that mbuf is defined. I am sure there
> are some holes here, but I think it is a better solution then using all
> of these macros, offset values and registration APIs.
>
>
> Regards,
> Keith
First of all, I applaud the idea of cleaning up the mbuf structure and removing the fields only rarely used and/or for special use cases only, as mentioned in the presentation, e.g. timestamp, timesync and seqn. It is great seeing serious effort put into improving the very core of DPDK!
However, after some hallway discussions at DPDK Userspace and further thinking about the details, I can see two additional risks by introducing dynamic mbufs, which I would like to share for your consideration:
1. It may prevent us from adding future solutions not yet considered.
If we were to introduce new functions for more granular handling of mbufs, similar to some of the Linux kernel's skbuff handling functions, how should such functions handle the dynamic fields? And how is the rte_pktmbuf_clone() function supposed to handle the dynamic fields? Some fields may need to be copied as-is, some may need to be initialized to zero or some other value, and so on. It is apparently not a problem now; but dynamic mbufs may prevent us from adding some of such functions in the future.
I admit that I can only imagine the issue on an abstract level, so I can't give you a concrete example. Perhaps some of the more experienced Linux developers can provide one or debunk my concern. (Stephen: In relation to packet capturing we were discussing the reference counter not being respected by some applications, and the possible need for more granular mbuf handling.)
2. In the long run, we might end up adding more fields than we remove.
Dynamic mbufs makes it easier to add specialized fields, which is great. But when the barrier to adding specialized fields is reduced, more PMDs and libraries may add their own unique fields, rather than going through a discussion and consensus for adding them to the fixed mbuf structure or finding some other solution. And if a multitude of PMDs each need a specialized field, PMDs might end up adding variants of a field, rather than reaching consensus for a common standard. It will be much easier to just add your own specialized mbuf field rather than having to go through the standardization process in the DPDK community.
I will use the timestamp field as a theoretic example: The packet timestamp measurement unit differs between vendors, so with dynamic mbufs one vendor's PMD might create a timestamp_ns field, counting in nanoseconds, and another vendor's PMD might create a timestamp_clocks field, counting in clock counts. With the fixed mbuf, this triggered a public discussion, and a compromise for the field was reached.
Although I am worried about dynamic mbufs, I don't have a better suggestion. And perhaps I just worry too much.
Med venlig hilsen / kind regards
- Morten Brørup
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [PATCH] mbuf: support dynamic fields and flags
2019-09-21 4:54 0% ` Wang, Haiyue
@ 2019-09-23 8:31 0% ` Olivier Matz
2019-09-23 11:01 0% ` Wang, Haiyue
0 siblings, 1 reply; 200+ results
From: Olivier Matz @ 2019-09-23 8:31 UTC (permalink / raw)
To: Wang, Haiyue
Cc: dev, Thomas Monjalon, Stephen Hemminger, Andrew Rybchenko, Wiles,
Keith, Jerin Jacob Kollanukkaran
Hi,
On Sat, Sep 21, 2019 at 04:54:39AM +0000, Wang, Haiyue wrote:
> > -----Original Message-----
> > From: Olivier Matz [mailto:olivier.matz@6wind.com]
> > Sent: Thursday, September 19, 2019 00:55
> > To: dev@dpdk.org
> > Cc: Thomas Monjalon <thomas@monjalon.net>; Wang, Haiyue <haiyue.wang@intel.com>; Stephen Hemminger
> > <stephen@networkplumber.org>; Andrew Rybchenko <arybchenko@solarflare.com>; Wiles, Keith
> > <keith.wiles@intel.com>; Jerin Jacob Kollanukkaran <jerinj@marvell.com>
> > Subject: [PATCH] mbuf: support dynamic fields and flags
> >
> > Many features require to store data inside the mbuf. As the room in mbuf
> > structure is limited, it is not possible to have a field for each
> > feature. Also, changing fields in the mbuf structure can break the API
> > or ABI.
> >
> > This commit addresses these issues, by enabling the dynamic registration
> > of fields or flags:
> >
> > - a dynamic field is a named area in the rte_mbuf structure, with a
> > given size (>= 1 byte) and alignment constraint.
> > - a dynamic flag is a named bit in the rte_mbuf structure.
> >
> > The typical use case is a PMD that registers space for an offload
> > feature, when the application requests to enable this feature. As
> > the space in mbuf is limited, the space should only be reserved if it
> > is going to be used (i.e when the application explicitly asks for it).
> >
> > The registration can be done at any moment, but it is not possible
> > to unregister fields or flags for now.
> >
> > Signed-off-by: Olivier Matz <olivier.matz@6wind.com>
> > Acked-by: Thomas Monjalon <thomas@monjalon.net>
> > ---
> >
> > rfc -> v1
> >
> > * Rebase on top of master
> > * Change registration API to use a structure instead of
> > variables, getting rid of #defines (Stephen's comment)
> > * Update flag registration to use a similar API as fields.
> > * Change max name length from 32 to 64 (sugg. by Thomas)
> > * Enhance API documentation (Haiyue's and Andrew's comments)
> > * Add a debug log at registration
> > * Add some words in release note
> > * Did some performance tests (sugg. by Andrew):
> > On my platform, reading a dynamic field takes ~3 cycles more
> > than a static field, and ~2 cycles more for writing.
> >
> > app/test/test_mbuf.c | 114 ++++++-
> > doc/guides/rel_notes/release_19_11.rst | 7 +
> > lib/librte_mbuf/Makefile | 2 +
> > lib/librte_mbuf/meson.build | 6 +-
> > lib/librte_mbuf/rte_mbuf.h | 25 +-
> > lib/librte_mbuf/rte_mbuf_dyn.c | 408 +++++++++++++++++++++++++
> > lib/librte_mbuf/rte_mbuf_dyn.h | 163 ++++++++++
> > lib/librte_mbuf/rte_mbuf_version.map | 4 +
> > 8 files changed, 724 insertions(+), 5 deletions(-)
> > create mode 100644 lib/librte_mbuf/rte_mbuf_dyn.c
> > create mode 100644 lib/librte_mbuf/rte_mbuf_dyn.h
> >
>
> [snip]
>
> > +/**
> > + * Helper macro to access to a dynamic field.
> > + */
> > +#define RTE_MBUF_DYNFIELD(m, offset, type) ((type)((uintptr_t)(m) + (offset)))
>
> How about to change it as: ?
> #define RTE_MBUF_DYNFIELD(m, offset, type) ((type *)((uintptr_t)(m) + (offset)))
> ^
> Then,
> *RTE_MBUF_DYNFIELD(mb, xxx, uint32_t) = yyy;
>
> Since we use 'type' like: sizeof(type), __alignof__(type), this makes 'type' be
> more consistent, not have to force cast 'type *' when using it.
>
> const struct rte_mbuf_dynfield dynfield2 = {
> .name = "test-dynfield2",
> .size = sizeof(uint16_t),
> .align = __alignof__(uint16_t),
> .flags = 0,
> };
Yes, I don't see use cases where the '*' is omitted, so it could be in the
macro. On the other hand, doing like in the patch is more consistent with
similar macros like rte_pktmbuf_mtod(), so I'll tend to keep it as is.
This is maybe not that important, because this macro will often be hidden
in a wrapper, like below:
static inline uint64_t rte_mbuf_dyn_timestamp_get(const struct rte_mbuf *m)
{
return *RTE_MBUF_DYNFIELD(m, rte_mbuf_dynfield_timestamp_offset,
uint64_t *);
}
> And also, when I'm trying to use the dynamic flag, found a macro will be better
> for making code align with dynamic field. Just a small suggestion. ;-)
> mb->ol_flags |= RTE_MBUF_DYNFLAG(ol_offset);
>
> /**
> * Helper macro to access to a dynamic flag.
> */
> #define RTE_MBUF_DYNFLAG(offset) (1ULL << (offset))
OK, I will add it in next version.
Thank you for the feedback!
Olivier
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [PATCH v2 2/3] vhost: call vDPA callback at the end of vring enable handler
@ 2019-09-23 8:12 3% ` Tiwei Bie
0 siblings, 0 replies; 200+ results
From: Tiwei Bie @ 2019-09-23 8:12 UTC (permalink / raw)
To: Andy Pei
Cc: dev, rosen.xu, xiaolong.ye, xiao.w.wang, maxime.coquelin, zhihong.wang
On Tue, Sep 17, 2019 at 05:09:47PM +0800, Andy Pei wrote:
> vDPA's set_vring_state callback would need to know the virtqueues'
> enable status to configure the hardware.
>
> Signed-off-by: Xiaolong Ye <xiaolong.ye@intel.com>
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> ---
> v2:
> add nr_active_vring as a parameter to ops function set_vring_state in
> case of callback in set_vring_state() and avoid exposing new API.
>
> lib/librte_vhost/rte_vdpa.h | 4 ++--
> lib/librte_vhost/vhost_user.c | 27 +++++++++++++++++++++++++--
> 2 files changed, 27 insertions(+), 4 deletions(-)
>
> diff --git a/lib/librte_vhost/rte_vdpa.h b/lib/librte_vhost/rte_vdpa.h
> index 9a3deb3..6e55d4d 100644
> --- a/lib/librte_vhost/rte_vdpa.h
> +++ b/lib/librte_vhost/rte_vdpa.h
> @@ -54,8 +54,8 @@ struct rte_vdpa_dev_ops {
> int (*dev_conf)(int vid);
> int (*dev_close)(int vid);
>
> - /** Enable/disable this vring */
> - int (*set_vring_state)(int vid, int vring, int state);
> + /** Enable/disable vring queue pairs */
> + int (*set_vring_state)(int vid, int nr_active_vring);
We should avoid changing the API/ABI unless we have a very good
justification.
With the existing API, it should be easy to get the number of
active rings by maintaining a bitmap or something similar in
ifc driver.
Besides, please keep other maintainers got from get-maintainer.sh
in the Cc list as well.
Thanks,
Tiwei
>
> /** Set features when changed */
> int (*set_features)(int vid);
> diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c
> index 0b72648..4d7de44 100644
> --- a/lib/librte_vhost/vhost_user.c
> +++ b/lib/librte_vhost/vhost_user.c
> @@ -1325,6 +1325,25 @@ static int vhost_user_set_vring_err(struct virtio_net **pdev __rte_unused,
> return RTE_VHOST_MSG_RESULT_REPLY;
> }
>
> +static uint16_t
> +vhost_get_active_vring_num(int vid)
> +{
> + struct virtio_net *dev = get_device(vid);
> + struct vhost_virtqueue *vq;
> + uint16_t qid;
> +
> + if (dev == NULL)
> + return 0;
> +
> + for (qid = 0; qid < dev->nr_vring; qid++) {
> + vq = dev->virtqueue[qid];
> + if (!vq->enabled)
> + break;
> + }
> +
> + return qid;
> +}
> +
> /*
> * when virtio queues are ready to work, qemu will send us to
> * enable the virtio queue pair.
> @@ -1339,6 +1358,7 @@ static int vhost_user_set_vring_err(struct virtio_net **pdev __rte_unused,
> int index = (int)msg->payload.state.index;
> struct rte_vdpa_device *vdpa_dev;
> int did = -1;
> + int nr_active_vring;
>
> RTE_LOG(INFO, VHOST_CONFIG,
> "set queue enable: %d to qp idx: %d\n",
> @@ -1346,8 +1366,6 @@ static int vhost_user_set_vring_err(struct virtio_net **pdev __rte_unused,
>
> did = dev->vdpa_dev_id;
> vdpa_dev = rte_vdpa_get_device(did);
> - if (vdpa_dev && vdpa_dev->ops->set_vring_state)
> - vdpa_dev->ops->set_vring_state(dev->vid, index, enable);
>
> if (dev->notify_ops->vring_state_changed)
> dev->notify_ops->vring_state_changed(dev->vid,
> @@ -1359,6 +1377,11 @@ static int vhost_user_set_vring_err(struct virtio_net **pdev __rte_unused,
>
> dev->virtqueue[index]->enabled = enable;
>
> + if (vdpa_dev && vdpa_dev->ops->set_vring_state) {
> + nr_active_vring = vhost_get_active_vring_num(dev->vid);
> + vdpa_dev->ops->set_vring_state(dev->vid, nr_active_vring);
> + }
> +
> return RTE_VHOST_MSG_RESULT_OK;
> }
>
> --
> 1.8.3.1
>
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] [PATCH] mbuf: support dynamic fields and flags
2019-09-18 16:54 4% ` [dpdk-dev] [PATCH] " Olivier Matz
2019-09-21 4:54 0% ` Wang, Haiyue
@ 2019-09-21 8:28 0% ` Wiles, Keith
2019-09-23 8:56 0% ` Morten Brørup
2019-09-23 9:13 0% ` Olivier Matz
1 sibling, 2 replies; 200+ results
From: Wiles, Keith @ 2019-09-21 8:28 UTC (permalink / raw)
To: Olivier Matz
Cc: dev, Thomas Monjalon, Wang, Haiyue, Stephen Hemminger,
Andrew Rybchenko, Jerin Jacob Kollanukkaran
> On Sep 18, 2019, at 6:54 PM, Olivier Matz <olivier.matz@6wind.com> wrote:
>
> Many features require to store data inside the mbuf. As the room in mbuf
> structure is limited, it is not possible to have a field for each
> feature. Also, changing fields in the mbuf structure can break the API
> or ABI.
>
> This commit addresses these issues, by enabling the dynamic registration
> of fields or flags:
>
> - a dynamic field is a named area in the rte_mbuf structure, with a
> given size (>= 1 byte) and alignment constraint.
> - a dynamic flag is a named bit in the rte_mbuf structure.
>
> The typical use case is a PMD that registers space for an offload
> feature, when the application requests to enable this feature. As
> the space in mbuf is limited, the space should only be reserved if it
> is going to be used (i.e when the application explicitly asks for it).
>
> The registration can be done at any moment, but it is not possible
> to unregister fields or flags for now.
>
> Signed-off-by: Olivier Matz <olivier.matz@6wind.com>
> Acked-by: Thomas Monjalon <thomas@monjalon.net>
> —
>
The idea of registration for space in the mbuf I am not a big fan. I did like Konstantin’s suggestion of having the compiler help with optimizing the code, but with a slight difference. Maybe I misunderstand, but now with this design you have to pass the offsets to different parts of the application or place in global memory or have each section request the offsets. It seems great if the application is one big application or an appliance model application having control of the whole design not so good for service chains like designs where different parts of the whole application is design by different teams.
Konstantin’s suggest if I understand it was to use structures to allow the compiler to optimize the access to the mbuf and I like that idea, but with one change we add a field in the mbuf to define the mbuf structure type.
Say 0 is the standard rte_mbuf type then type 1 could be the IPSec offset type mbuf, type 2 could be something else, … The type 0 looks just like the mbuf we have today with maybe the optional fields set to reserved or some type of filler variables to reserve the holes in the structure. Then type 1 is the IPSec mbuf and in the reserved sections of the mbuf contain the IPSec related data with the standard mbuf fields still matching the type 0 version.
This allows the mbuf to be used by the developer and the compiler now knows exactly where the fields are located in the structure and does not have to deal with any of the macros and offsets and registration suggested here. Just cast the mbuf pointer into the new type mbuf structure. We just have to make sure the code that needs to use a given mbuf type has access to the structure definitions.
If the mbufs it going to be translated from one type mbuf to another mbuf type, we just have to define that type and then cast the mbuf pointer to that structure. When an mbuf is received from IPSec PMD then the application needs to forward that mbuf to the next stage it can reset the type to 0 or to another type filling in the reserved fields to be used by the next stage in the pipeline.
The mbuf now contains the type and every point in the application can look at the type to determine how that mbuf is defined. I am sure there are some holes here, but I think it is a better solution then using all of these macros, offset values and registration APIs.
Regards,
Keith
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [PATCH] mbuf: support dynamic fields and flags
2019-09-18 16:54 4% ` [dpdk-dev] [PATCH] " Olivier Matz
@ 2019-09-21 4:54 0% ` Wang, Haiyue
2019-09-23 8:31 0% ` Olivier Matz
2019-09-21 8:28 0% ` Wiles, Keith
1 sibling, 1 reply; 200+ results
From: Wang, Haiyue @ 2019-09-21 4:54 UTC (permalink / raw)
To: Olivier Matz, dev
Cc: Thomas Monjalon, Stephen Hemminger, Andrew Rybchenko, Wiles,
Keith, Jerin Jacob Kollanukkaran
> -----Original Message-----
> From: Olivier Matz [mailto:olivier.matz@6wind.com]
> Sent: Thursday, September 19, 2019 00:55
> To: dev@dpdk.org
> Cc: Thomas Monjalon <thomas@monjalon.net>; Wang, Haiyue <haiyue.wang@intel.com>; Stephen Hemminger
> <stephen@networkplumber.org>; Andrew Rybchenko <arybchenko@solarflare.com>; Wiles, Keith
> <keith.wiles@intel.com>; Jerin Jacob Kollanukkaran <jerinj@marvell.com>
> Subject: [PATCH] mbuf: support dynamic fields and flags
>
> Many features require to store data inside the mbuf. As the room in mbuf
> structure is limited, it is not possible to have a field for each
> feature. Also, changing fields in the mbuf structure can break the API
> or ABI.
>
> This commit addresses these issues, by enabling the dynamic registration
> of fields or flags:
>
> - a dynamic field is a named area in the rte_mbuf structure, with a
> given size (>= 1 byte) and alignment constraint.
> - a dynamic flag is a named bit in the rte_mbuf structure.
>
> The typical use case is a PMD that registers space for an offload
> feature, when the application requests to enable this feature. As
> the space in mbuf is limited, the space should only be reserved if it
> is going to be used (i.e when the application explicitly asks for it).
>
> The registration can be done at any moment, but it is not possible
> to unregister fields or flags for now.
>
> Signed-off-by: Olivier Matz <olivier.matz@6wind.com>
> Acked-by: Thomas Monjalon <thomas@monjalon.net>
> ---
>
> rfc -> v1
>
> * Rebase on top of master
> * Change registration API to use a structure instead of
> variables, getting rid of #defines (Stephen's comment)
> * Update flag registration to use a similar API as fields.
> * Change max name length from 32 to 64 (sugg. by Thomas)
> * Enhance API documentation (Haiyue's and Andrew's comments)
> * Add a debug log at registration
> * Add some words in release note
> * Did some performance tests (sugg. by Andrew):
> On my platform, reading a dynamic field takes ~3 cycles more
> than a static field, and ~2 cycles more for writing.
>
> app/test/test_mbuf.c | 114 ++++++-
> doc/guides/rel_notes/release_19_11.rst | 7 +
> lib/librte_mbuf/Makefile | 2 +
> lib/librte_mbuf/meson.build | 6 +-
> lib/librte_mbuf/rte_mbuf.h | 25 +-
> lib/librte_mbuf/rte_mbuf_dyn.c | 408 +++++++++++++++++++++++++
> lib/librte_mbuf/rte_mbuf_dyn.h | 163 ++++++++++
> lib/librte_mbuf/rte_mbuf_version.map | 4 +
> 8 files changed, 724 insertions(+), 5 deletions(-)
> create mode 100644 lib/librte_mbuf/rte_mbuf_dyn.c
> create mode 100644 lib/librte_mbuf/rte_mbuf_dyn.h
>
[snip]
> +/**
> + * Helper macro to access to a dynamic field.
> + */
> +#define RTE_MBUF_DYNFIELD(m, offset, type) ((type)((uintptr_t)(m) + (offset)))
How about to change it as: ?
#define RTE_MBUF_DYNFIELD(m, offset, type) ((type *)((uintptr_t)(m) + (offset)))
^
Then,
*RTE_MBUF_DYNFIELD(mb, xxx, uint32_t) = yyy;
Since we use 'type' like: sizeof(type), __alignof__(type), this makes 'type' be
more consistent, not have to force cast 'type *' when using it.
const struct rte_mbuf_dynfield dynfield2 = {
.name = "test-dynfield2",
.size = sizeof(uint16_t),
.align = __alignof__(uint16_t),
.flags = 0,
};
And also, when I'm trying to use the dynamic flag, found a macro will be better
for making code align with dynamic field. Just a small suggestion. ;-)
mb->ol_flags |= RTE_MBUF_DYNFLAG(ol_offset);
/**
* Helper macro to access to a dynamic flag.
*/
#define RTE_MBUF_DYNFLAG(offset) (1ULL << (offset))
> +
> +#endif
> diff --git a/lib/librte_mbuf/rte_mbuf_version.map b/lib/librte_mbuf/rte_mbuf_version.map
> index 2662a37bf..a98310570 100644
> --- a/lib/librte_mbuf/rte_mbuf_version.map
> +++ b/lib/librte_mbuf/rte_mbuf_version.map
> @@ -50,4 +50,8 @@ EXPERIMENTAL {
> global:
>
> rte_mbuf_check;
> + rte_mbuf_dynfield_lookup;
> + rte_mbuf_dynfield_register;
> + rte_mbuf_dynflag_lookup;
> + rte_mbuf_dynflag_register;
> } DPDK_18.08;
> --
> 2.20.1
^ permalink raw reply [relevance 0%]
* [dpdk-dev] [PATCH] mbuf: support dynamic fields and flags
@ 2019-09-18 16:54 4% ` Olivier Matz
2019-09-21 4:54 0% ` Wang, Haiyue
2019-09-21 8:28 0% ` Wiles, Keith
0 siblings, 2 replies; 200+ results
From: Olivier Matz @ 2019-09-18 16:54 UTC (permalink / raw)
To: dev
Cc: Thomas Monjalon, Haiyue Wang, Stephen Hemminger,
Andrew Rybchenko, Keith Wiles, Jerin Jacob Kollanukkaran
Many features require to store data inside the mbuf. As the room in mbuf
structure is limited, it is not possible to have a field for each
feature. Also, changing fields in the mbuf structure can break the API
or ABI.
This commit addresses these issues, by enabling the dynamic registration
of fields or flags:
- a dynamic field is a named area in the rte_mbuf structure, with a
given size (>= 1 byte) and alignment constraint.
- a dynamic flag is a named bit in the rte_mbuf structure.
The typical use case is a PMD that registers space for an offload
feature, when the application requests to enable this feature. As
the space in mbuf is limited, the space should only be reserved if it
is going to be used (i.e when the application explicitly asks for it).
The registration can be done at any moment, but it is not possible
to unregister fields or flags for now.
Signed-off-by: Olivier Matz <olivier.matz@6wind.com>
Acked-by: Thomas Monjalon <thomas@monjalon.net>
---
rfc -> v1
* Rebase on top of master
* Change registration API to use a structure instead of
variables, getting rid of #defines (Stephen's comment)
* Update flag registration to use a similar API as fields.
* Change max name length from 32 to 64 (sugg. by Thomas)
* Enhance API documentation (Haiyue's and Andrew's comments)
* Add a debug log at registration
* Add some words in release note
* Did some performance tests (sugg. by Andrew):
On my platform, reading a dynamic field takes ~3 cycles more
than a static field, and ~2 cycles more for writing.
app/test/test_mbuf.c | 114 ++++++-
doc/guides/rel_notes/release_19_11.rst | 7 +
lib/librte_mbuf/Makefile | 2 +
lib/librte_mbuf/meson.build | 6 +-
lib/librte_mbuf/rte_mbuf.h | 25 +-
lib/librte_mbuf/rte_mbuf_dyn.c | 408 +++++++++++++++++++++++++
lib/librte_mbuf/rte_mbuf_dyn.h | 163 ++++++++++
lib/librte_mbuf/rte_mbuf_version.map | 4 +
8 files changed, 724 insertions(+), 5 deletions(-)
create mode 100644 lib/librte_mbuf/rte_mbuf_dyn.c
create mode 100644 lib/librte_mbuf/rte_mbuf_dyn.h
diff --git a/app/test/test_mbuf.c b/app/test/test_mbuf.c
index 2a97afe20..96acfc4b2 100644
--- a/app/test/test_mbuf.c
+++ b/app/test/test_mbuf.c
@@ -28,6 +28,7 @@
#include <rte_random.h>
#include <rte_cycles.h>
#include <rte_malloc.h>
+#include <rte_mbuf_dyn.h>
#include "test.h"
@@ -502,7 +503,6 @@ test_attach_from_different_pool(struct rte_mempool *pktmbuf_pool,
rte_pktmbuf_free(clone2);
return -1;
}
-#undef GOTO_FAIL
/*
* test allocation and free of mbufs
@@ -1121,6 +1121,112 @@ test_tx_offload(void)
return (v1 == v2) ? 0 : -EINVAL;
}
+static int
+test_mbuf_dyn(struct rte_mempool *pktmbuf_pool)
+{
+ const struct rte_mbuf_dynfield dynfield = {
+ .name = "test-dynfield",
+ .size = sizeof(uint8_t),
+ .align = __alignof__(uint8_t),
+ .flags = 0,
+ };
+ const struct rte_mbuf_dynfield dynfield2 = {
+ .name = "test-dynfield2",
+ .size = sizeof(uint16_t),
+ .align = __alignof__(uint16_t),
+ .flags = 0,
+ };
+ const struct rte_mbuf_dynfield dynfield_fail_big = {
+ .name = "test-dynfield-fail-big",
+ .size = 256,
+ .align = 1,
+ .flags = 0,
+ };
+ const struct rte_mbuf_dynfield dynfield_fail_align = {
+ .name = "test-dynfield-fail-align",
+ .size = 1,
+ .align = 3,
+ .flags = 0,
+ };
+ const struct rte_mbuf_dynflag dynflag = {
+ .name = "test-dynflag",
+ .flags = 0,
+ };
+ const struct rte_mbuf_dynflag dynflag2 = {
+ .name = "test-dynflag2",
+ .flags = 0,
+ };
+ struct rte_mbuf *m = NULL;
+ int offset, offset2;
+ int flag, flag2;
+
+ printf("Test mbuf dynamic fields and flags\n");
+
+ offset = rte_mbuf_dynfield_register(&dynfield);
+ if (offset == -1)
+ GOTO_FAIL("failed to register dynamic field, offset=%d: %s",
+ offset, strerror(errno));
+
+ offset2 = rte_mbuf_dynfield_register(&dynfield);
+ if (offset2 != offset)
+ GOTO_FAIL("failed to lookup dynamic field, offset=%d, offset2=%d: %s",
+ offset, offset2, strerror(errno));
+
+ offset2 = rte_mbuf_dynfield_register(&dynfield2);
+ if (offset2 == -1 || offset2 == offset || (offset & 1))
+ GOTO_FAIL("failed to register dynfield field 2, offset=%d, offset2=%d: %s",
+ offset, offset2, strerror(errno));
+
+ printf("dynfield: offset = %d, offset2 = %d\n", offset, offset2);
+
+ offset = rte_mbuf_dynfield_register(&dynfield_fail_big);
+ if (offset != -1)
+ GOTO_FAIL("dynamic field creation should fail (too big)");
+
+ offset = rte_mbuf_dynfield_register(&dynfield_fail_align);
+ if (offset != -1)
+ GOTO_FAIL("dynamic field creation should fail (bad alignment)");
+
+ flag = rte_mbuf_dynflag_register(&dynflag);
+ if (flag == -1)
+ GOTO_FAIL("failed to register dynamic field, flag=%d: %s",
+ flag, strerror(errno));
+
+ flag2 = rte_mbuf_dynflag_register(&dynflag);
+ if (flag2 != flag)
+ GOTO_FAIL("failed to lookup dynamic field, flag=%d, flag2=%d: %s",
+ flag, flag2, strerror(errno));
+
+ flag2 = rte_mbuf_dynflag_register(&dynflag2);
+ if (flag2 == -1 || flag2 == flag)
+ GOTO_FAIL("failed to register dynflag field 2, flag=%d, flag2=%d: %s",
+ flag, flag2, strerror(errno));
+
+ printf("dynflag: flag = %d, flag2 = %d\n", flag, flag2);
+
+ /* set, get dynamic field */
+ m = rte_pktmbuf_alloc(pktmbuf_pool);
+ if (m == NULL)
+ GOTO_FAIL("Cannot allocate mbuf");
+
+ *RTE_MBUF_DYNFIELD(m, offset, uint8_t *) = 1;
+ if (*RTE_MBUF_DYNFIELD(m, offset, uint8_t *) != 1)
+ GOTO_FAIL("failed to read dynamic field");
+ *RTE_MBUF_DYNFIELD(m, offset2, uint16_t *) = 1000;
+ if (*RTE_MBUF_DYNFIELD(m, offset2, uint16_t *) != 1000)
+ GOTO_FAIL("failed to read dynamic field");
+
+ /* set a dynamic flag */
+ m->ol_flags |= (1ULL << flag);
+
+ rte_pktmbuf_free(m);
+ return 0;
+fail:
+ rte_pktmbuf_free(m);
+ return -1;
+}
+#undef GOTO_FAIL
+
static int
test_mbuf(void)
{
@@ -1140,6 +1246,12 @@ test_mbuf(void)
goto err;
}
+ /* test registration of dynamic fields and flags */
+ if (test_mbuf_dyn(pktmbuf_pool) < 0) {
+ printf("mbuf dynflag test failed\n");
+ goto err;
+ }
+
/* create a specific pktmbuf pool with a priv_size != 0 and no data
* room size */
pktmbuf_pool2 = rte_pktmbuf_pool_create("test_pktmbuf_pool2",
diff --git a/doc/guides/rel_notes/release_19_11.rst b/doc/guides/rel_notes/release_19_11.rst
index 27cfbd9e3..0fcb76f76 100644
--- a/doc/guides/rel_notes/release_19_11.rst
+++ b/doc/guides/rel_notes/release_19_11.rst
@@ -56,6 +56,13 @@ New Features
Also, make sure to start the actual text at the margin.
=========================================================
+* **Add support of support dynamic fields and flags in mbuf.**
+
+ This new feature adds the ability to dynamically register some room
+ for a field or a flag in the mbuf structure. This is typically used
+ for specific offload features, where adding a static field or flag
+ in the mbuf is not justified.
+
Removed Items
-------------
diff --git a/lib/librte_mbuf/Makefile b/lib/librte_mbuf/Makefile
index c8f6d2689..5a9bcee73 100644
--- a/lib/librte_mbuf/Makefile
+++ b/lib/librte_mbuf/Makefile
@@ -17,8 +17,10 @@ LIBABIVER := 5
# all source are stored in SRCS-y
SRCS-$(CONFIG_RTE_LIBRTE_MBUF) := rte_mbuf.c rte_mbuf_ptype.c rte_mbuf_pool_ops.c
+SRCS-$(CONFIG_RTE_LIBRTE_MBUF) += rte_mbuf_dyn.c
# install includes
SYMLINK-$(CONFIG_RTE_LIBRTE_MBUF)-include := rte_mbuf.h rte_mbuf_ptype.h rte_mbuf_pool_ops.h
+SYMLINK-$(CONFIG_RTE_LIBRTE_MBUF)-include += rte_mbuf_dyn.h
include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/lib/librte_mbuf/meson.build b/lib/librte_mbuf/meson.build
index 6cc11ebb4..9137e8f26 100644
--- a/lib/librte_mbuf/meson.build
+++ b/lib/librte_mbuf/meson.build
@@ -2,8 +2,10 @@
# Copyright(c) 2017 Intel Corporation
version = 5
-sources = files('rte_mbuf.c', 'rte_mbuf_ptype.c', 'rte_mbuf_pool_ops.c')
-headers = files('rte_mbuf.h', 'rte_mbuf_ptype.h', 'rte_mbuf_pool_ops.h')
+sources = files('rte_mbuf.c', 'rte_mbuf_ptype.c', 'rte_mbuf_pool_ops.c',
+ 'rte_mbuf_dyn.c')
+headers = files('rte_mbuf.h', 'rte_mbuf_ptype.h', 'rte_mbuf_pool_ops.h',
+ 'rte_mbuf_dyn.h')
deps += ['mempool']
allow_experimental_apis = true
diff --git a/lib/librte_mbuf/rte_mbuf.h b/lib/librte_mbuf/rte_mbuf.h
index 98225ec80..ef588cd54 100644
--- a/lib/librte_mbuf/rte_mbuf.h
+++ b/lib/librte_mbuf/rte_mbuf.h
@@ -198,9 +198,12 @@ extern "C" {
#define PKT_RX_OUTER_L4_CKSUM_GOOD (1ULL << 22)
#define PKT_RX_OUTER_L4_CKSUM_INVALID ((1ULL << 21) | (1ULL << 22))
-/* add new RX flags here */
+/* add new RX flags here, don't forget to update PKT_FIRST_FREE */
-/* add new TX flags here */
+#define PKT_FIRST_FREE (1ULL << 23)
+#define PKT_LAST_FREE (1ULL << 39)
+
+/* add new TX flags here, don't forget to update PKT_LAST_FREE */
/**
* Indicate that the metadata field in the mbuf is in use.
@@ -738,6 +741,8 @@ struct rte_mbuf {
*/
struct rte_mbuf_ext_shared_info *shinfo;
+ uint64_t dynfield1; /**< Reserved for dynamic fields. */
+ uint64_t dynfield2; /**< Reserved for dynamic fields. */
} __rte_cache_aligned;
/**
@@ -1684,6 +1689,21 @@ rte_pktmbuf_attach_extbuf(struct rte_mbuf *m, void *buf_addr,
*/
#define rte_pktmbuf_detach_extbuf(m) rte_pktmbuf_detach(m)
+/**
+ * Copy dynamic fields from m_src to m_dst.
+ *
+ * @param m_dst
+ * The destination mbuf.
+ * @param m_src
+ * The source mbuf.
+ */
+static inline void
+rte_mbuf_dynfield_copy(struct rte_mbuf *m_dst, const struct rte_mbuf *m_src)
+{
+ m_dst->dynfield1 = m_src->dynfield1;
+ m_dst->dynfield2 = m_src->dynfield2;
+}
+
/**
* Attach packet mbuf to another packet mbuf.
*
@@ -1732,6 +1752,7 @@ static inline void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m)
mi->vlan_tci_outer = m->vlan_tci_outer;
mi->tx_offload = m->tx_offload;
mi->hash = m->hash;
+ rte_mbuf_dynfield_copy(mi, m);
mi->next = NULL;
mi->pkt_len = mi->data_len;
diff --git a/lib/librte_mbuf/rte_mbuf_dyn.c b/lib/librte_mbuf/rte_mbuf_dyn.c
new file mode 100644
index 000000000..13b8742d0
--- /dev/null
+++ b/lib/librte_mbuf/rte_mbuf_dyn.c
@@ -0,0 +1,408 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2019 6WIND S.A.
+ */
+
+#include <sys/queue.h>
+
+#include <rte_common.h>
+#include <rte_eal.h>
+#include <rte_eal_memconfig.h>
+#include <rte_tailq.h>
+#include <rte_errno.h>
+#include <rte_malloc.h>
+#include <rte_string_fns.h>
+#include <rte_mbuf.h>
+#include <rte_mbuf_dyn.h>
+
+#define RTE_MBUF_DYN_MZNAME "rte_mbuf_dyn"
+
+struct mbuf_dynfield_elt {
+ TAILQ_ENTRY(mbuf_dynfield_elt) next;
+ struct rte_mbuf_dynfield params;
+ int offset;
+};
+TAILQ_HEAD(mbuf_dynfield_list, rte_tailq_entry);
+
+static struct rte_tailq_elem mbuf_dynfield_tailq = {
+ .name = "RTE_MBUF_DYNFIELD",
+};
+EAL_REGISTER_TAILQ(mbuf_dynfield_tailq);
+
+struct mbuf_dynflag_elt {
+ TAILQ_ENTRY(mbuf_dynflag_elt) next;
+ struct rte_mbuf_dynflag params;
+ int bitnum;
+};
+TAILQ_HEAD(mbuf_dynflag_list, rte_tailq_entry);
+
+static struct rte_tailq_elem mbuf_dynflag_tailq = {
+ .name = "RTE_MBUF_DYNFLAG",
+};
+EAL_REGISTER_TAILQ(mbuf_dynflag_tailq);
+
+struct mbuf_dyn_shm {
+ /** For each mbuf byte, free_space[i] == 1 if space is free. */
+ uint8_t free_space[sizeof(struct rte_mbuf)];
+ /** Bitfield of available flags. */
+ uint64_t free_flags;
+};
+static struct mbuf_dyn_shm *shm;
+
+/* allocate and initialize the shared memory */
+static int
+init_shared_mem(void)
+{
+ const struct rte_memzone *mz;
+ uint64_t mask;
+
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+ mz = rte_memzone_reserve_aligned(RTE_MBUF_DYN_MZNAME,
+ sizeof(struct mbuf_dyn_shm),
+ SOCKET_ID_ANY, 0,
+ RTE_CACHE_LINE_SIZE);
+ } else {
+ mz = rte_memzone_lookup(RTE_MBUF_DYN_MZNAME);
+ }
+ if (mz == NULL)
+ return -1;
+
+ shm = mz->addr;
+
+#define mark_free(field) \
+ memset(&shm->free_space[offsetof(struct rte_mbuf, field)], \
+ 0xff, sizeof(((struct rte_mbuf *)0)->field))
+
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+ /* init free_space, keep it sync'd with
+ * rte_mbuf_dynfield_copy().
+ */
+ memset(shm, 0, sizeof(*shm));
+ mark_free(dynfield1);
+ mark_free(dynfield2);
+
+ /* init free_flags */
+ for (mask = PKT_FIRST_FREE; mask <= PKT_LAST_FREE; mask <<= 1)
+ shm->free_flags |= mask;
+ }
+#undef mark_free
+
+ return 0;
+}
+
+/* check if this offset can be used */
+static int
+check_offset(size_t offset, size_t size, size_t align, unsigned int flags)
+{
+ size_t i;
+
+ (void)flags;
+
+ if ((offset & (align - 1)) != 0)
+ return -1;
+ if (offset + size > sizeof(struct rte_mbuf))
+ return -1;
+
+ for (i = 0; i < size; i++) {
+ if (!shm->free_space[i + offset])
+ return -1;
+ }
+
+ return 0;
+}
+
+/* assume tailq is locked */
+static struct mbuf_dynfield_elt *
+__mbuf_dynfield_lookup(const char *name)
+{
+ struct mbuf_dynfield_list *mbuf_dynfield_list;
+ struct mbuf_dynfield_elt *mbuf_dynfield;
+ struct rte_tailq_entry *te;
+
+ mbuf_dynfield_list = RTE_TAILQ_CAST(
+ mbuf_dynfield_tailq.head, mbuf_dynfield_list);
+
+ TAILQ_FOREACH(te, mbuf_dynfield_list, next) {
+ mbuf_dynfield = (struct mbuf_dynfield_elt *)te->data;
+ if (strcmp(name, mbuf_dynfield->params.name) == 0)
+ break;
+ }
+
+ if (te == NULL) {
+ rte_errno = ENOENT;
+ return NULL;
+ }
+
+ return mbuf_dynfield;
+}
+
+int
+rte_mbuf_dynfield_lookup(const char *name, struct rte_mbuf_dynfield *params)
+{
+ struct mbuf_dynfield_elt *mbuf_dynfield;
+
+ if (shm == NULL) {
+ rte_errno = ENOENT;
+ return -1;
+ }
+
+ rte_mcfg_tailq_read_lock();
+ mbuf_dynfield = __mbuf_dynfield_lookup(name);
+ rte_mcfg_tailq_read_unlock();
+
+ if (mbuf_dynfield == NULL) {
+ rte_errno = ENOENT;
+ return -1;
+ }
+
+ if (params != NULL)
+ memcpy(params, &mbuf_dynfield->params, sizeof(*params));
+
+ return mbuf_dynfield->offset;
+}
+
+static int mbuf_dynfield_cmp(const struct rte_mbuf_dynfield *params1,
+ const struct rte_mbuf_dynfield *params2)
+{
+ if (strcmp(params1->name, params2->name))
+ return -1;
+ if (params1->size != params2->size)
+ return -1;
+ if (params1->align != params2->align)
+ return -1;
+ if (params1->flags != params2->flags)
+ return -1;
+ return 0;
+}
+
+int
+rte_mbuf_dynfield_register(const struct rte_mbuf_dynfield *params)
+{
+ struct mbuf_dynfield_list *mbuf_dynfield_list;
+ struct mbuf_dynfield_elt *mbuf_dynfield = NULL;
+ struct rte_tailq_entry *te = NULL;
+ int offset, ret;
+ size_t i;
+
+ if (shm == NULL && init_shared_mem() < 0)
+ goto fail;
+ if (params->size >= sizeof(struct rte_mbuf)) {
+ rte_errno = EINVAL;
+ goto fail;
+ }
+ if (!rte_is_power_of_2(params->align)) {
+ rte_errno = EINVAL;
+ goto fail;
+ }
+ if (params->flags != 0) {
+ rte_errno = EINVAL;
+ goto fail;
+ }
+
+ rte_mcfg_tailq_write_lock();
+
+ mbuf_dynfield = __mbuf_dynfield_lookup(params->name);
+ if (mbuf_dynfield != NULL) {
+ if (mbuf_dynfield_cmp(params, &mbuf_dynfield->params) < 0) {
+ rte_errno = EEXIST;
+ goto fail_unlock;
+ }
+ offset = mbuf_dynfield->offset;
+ goto out_unlock;
+ }
+
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ rte_errno = EPERM;
+ goto fail_unlock;
+ }
+
+ for (offset = 0;
+ offset < (int)sizeof(struct rte_mbuf);
+ offset++) {
+ if (check_offset(offset, params->size, params->align,
+ params->flags) == 0)
+ break;
+ }
+
+ if (offset == sizeof(struct rte_mbuf)) {
+ rte_errno = ENOENT;
+ goto fail_unlock;
+ }
+
+ mbuf_dynfield_list = RTE_TAILQ_CAST(
+ mbuf_dynfield_tailq.head, mbuf_dynfield_list);
+
+ te = rte_zmalloc("MBUF_DYNFIELD_TAILQ_ENTRY", sizeof(*te), 0);
+ if (te == NULL)
+ goto fail_unlock;
+
+ mbuf_dynfield = rte_zmalloc("mbuf_dynfield", sizeof(*mbuf_dynfield), 0);
+ if (mbuf_dynfield == NULL)
+ goto fail_unlock;
+
+ ret = strlcpy(mbuf_dynfield->params.name, params->name,
+ sizeof(mbuf_dynfield->params.name));
+ if (ret < 0 || ret >= (int)sizeof(mbuf_dynfield->params.name)) {
+ rte_errno = ENAMETOOLONG;
+ goto fail_unlock;
+ }
+ memcpy(&mbuf_dynfield->params, params, sizeof(mbuf_dynfield->params));
+ mbuf_dynfield->offset = offset;
+ te->data = mbuf_dynfield;
+
+ TAILQ_INSERT_TAIL(mbuf_dynfield_list, te, next);
+
+ for (i = offset; i < offset + params->size; i++)
+ shm->free_space[i] = 0;
+
+ RTE_LOG(DEBUG, MBUF, "Registered dynamic field %s (sz=%zu, al=%zu, fl=0x%x) -> %d\n",
+ params->name, params->size, params->align, params->flags,
+ offset);
+
+out_unlock:
+ rte_mcfg_tailq_write_unlock();
+
+ return offset;
+
+fail_unlock:
+ rte_mcfg_tailq_write_unlock();
+fail:
+ rte_free(mbuf_dynfield);
+ rte_free(te);
+ return -1;
+}
+
+/* assume tailq is locked */
+static struct mbuf_dynflag_elt *
+__mbuf_dynflag_lookup(const char *name)
+{
+ struct mbuf_dynflag_list *mbuf_dynflag_list;
+ struct mbuf_dynflag_elt *mbuf_dynflag;
+ struct rte_tailq_entry *te;
+
+ mbuf_dynflag_list = RTE_TAILQ_CAST(
+ mbuf_dynflag_tailq.head, mbuf_dynflag_list);
+
+ TAILQ_FOREACH(te, mbuf_dynflag_list, next) {
+ mbuf_dynflag = (struct mbuf_dynflag_elt *)te->data;
+ if (strncmp(name, mbuf_dynflag->params.name,
+ RTE_MBUF_DYN_NAMESIZE) == 0)
+ break;
+ }
+
+ if (te == NULL) {
+ rte_errno = ENOENT;
+ return NULL;
+ }
+
+ return mbuf_dynflag;
+}
+
+int
+rte_mbuf_dynflag_lookup(const char *name,
+ struct rte_mbuf_dynflag *params)
+{
+ struct mbuf_dynflag_elt *mbuf_dynflag;
+
+ if (shm == NULL) {
+ rte_errno = ENOENT;
+ return -1;
+ }
+
+ rte_mcfg_tailq_read_lock();
+ mbuf_dynflag = __mbuf_dynflag_lookup(name);
+ rte_mcfg_tailq_read_unlock();
+
+ if (mbuf_dynflag == NULL) {
+ rte_errno = ENOENT;
+ return -1;
+ }
+
+ if (params != NULL)
+ memcpy(params, &mbuf_dynflag->params, sizeof(*params));
+
+ return mbuf_dynflag->bitnum;
+}
+
+static int mbuf_dynflag_cmp(const struct rte_mbuf_dynflag *params1,
+ const struct rte_mbuf_dynflag *params2)
+{
+ if (strcmp(params1->name, params2->name))
+ return -1;
+ if (params1->flags != params2->flags)
+ return -1;
+ return 0;
+}
+
+int
+rte_mbuf_dynflag_register(const struct rte_mbuf_dynflag *params)
+{
+ struct mbuf_dynflag_list *mbuf_dynflag_list;
+ struct mbuf_dynflag_elt *mbuf_dynflag = NULL;
+ struct rte_tailq_entry *te = NULL;
+ int bitnum, ret;
+
+ if (shm == NULL && init_shared_mem() < 0)
+ goto fail;
+
+ rte_mcfg_tailq_write_lock();
+
+ mbuf_dynflag = __mbuf_dynflag_lookup(params->name);
+ if (mbuf_dynflag != NULL) {
+ if (mbuf_dynflag_cmp(params, &mbuf_dynflag->params) < 0) {
+ rte_errno = EEXIST;
+ goto fail_unlock;
+ }
+ bitnum = mbuf_dynflag->bitnum;
+ goto out_unlock;
+ }
+
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ rte_errno = EPERM;
+ goto fail_unlock;
+ }
+
+ if (shm->free_flags == 0) {
+ rte_errno = ENOENT;
+ goto fail_unlock;
+ }
+ bitnum = rte_bsf64(shm->free_flags);
+
+ mbuf_dynflag_list = RTE_TAILQ_CAST(
+ mbuf_dynflag_tailq.head, mbuf_dynflag_list);
+
+ te = rte_zmalloc("MBUF_DYNFLAG_TAILQ_ENTRY", sizeof(*te), 0);
+ if (te == NULL)
+ goto fail_unlock;
+
+ mbuf_dynflag = rte_zmalloc("mbuf_dynflag", sizeof(*mbuf_dynflag), 0);
+ if (mbuf_dynflag == NULL)
+ goto fail_unlock;
+
+ ret = strlcpy(mbuf_dynflag->params.name, params->name,
+ sizeof(mbuf_dynflag->params.name));
+ if (ret < 0 || ret >= (int)sizeof(mbuf_dynflag->params.name)) {
+ rte_errno = ENAMETOOLONG;
+ goto fail_unlock;
+ }
+ mbuf_dynflag->bitnum = bitnum;
+ te->data = mbuf_dynflag;
+
+ TAILQ_INSERT_TAIL(mbuf_dynflag_list, te, next);
+
+ shm->free_flags &= ~(1ULL << bitnum);
+
+ RTE_LOG(DEBUG, MBUF, "Registered dynamic flag %s (fl=0x%x) -> %u\n",
+ params->name, params->flags, bitnum);
+
+out_unlock:
+ rte_mcfg_tailq_write_unlock();
+
+ return bitnum;
+
+fail_unlock:
+ rte_mcfg_tailq_write_unlock();
+fail:
+ rte_free(mbuf_dynflag);
+ rte_free(te);
+ return -1;
+}
diff --git a/lib/librte_mbuf/rte_mbuf_dyn.h b/lib/librte_mbuf/rte_mbuf_dyn.h
new file mode 100644
index 000000000..6e2c81654
--- /dev/null
+++ b/lib/librte_mbuf/rte_mbuf_dyn.h
@@ -0,0 +1,163 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2019 6WIND S.A.
+ */
+
+#ifndef _RTE_MBUF_DYN_H_
+#define _RTE_MBUF_DYN_H_
+
+/**
+ * @file
+ * RTE Mbuf dynamic fields and flags
+ *
+ * Many features require to store data inside the mbuf. As the room in
+ * mbuf structure is limited, it is not possible to have a field for
+ * each feature. Also, changing fields in the mbuf structure can break
+ * the API or ABI.
+ *
+ * This module addresses this issue, by enabling the dynamic
+ * registration of fields or flags:
+ *
+ * - a dynamic field is a named area in the rte_mbuf structure, with a
+ * given size (>= 1 byte) and alignment constraint.
+ * - a dynamic flag is a named bit in the rte_mbuf structure, stored
+ * in mbuf->ol_flags.
+ *
+ * The typical use case is when a specific offload feature requires to
+ * register a dedicated offload field in the mbuf structure, and adding
+ * a static field or flag is not justified.
+ *
+ * Example of use:
+ *
+ * - A rte_mbuf_dynfield structure is defined, containing the parameters
+ * of the dynamic field to be registered:
+ * const struct rte_mbuf_dynfield rte_dynfield_my_feature = { ... };
+ * - The application initializes the PMD, and asks for this feature
+ * at port initialization by passing DEV_RX_OFFLOAD_MY_FEATURE in
+ * rxconf. This will make the PMD to register the field by calling
+ * rte_mbuf_dynfield_register(&rte_dynfield_my_feature). The PMD
+ * stores the returned offset.
+ * - The application that uses the offload feature also registers
+ * the field to retrieve the same offset.
+ * - When the PMD receives a packet, it can set the field:
+ * *RTE_MBUF_DYNFIELD(m, offset, <type *>) = value;
+ * - In the main loop, the application can retrieve the value with
+ * the same macro.
+ *
+ * To avoid wasting space, the dynamic fields or flags must only be
+ * reserved on demand, when an application asks for the related feature.
+ *
+ * The registration can be done at any moment, but it is not possible
+ * to unregister fields or flags for now.
+ *
+ * A dynamic field can be reserved and used by an application only.
+ * It can for instance be a packet mark.
+ */
+
+#include <sys/types.h>
+/**
+ * Maximum length of the dynamic field or flag string.
+ */
+#define RTE_MBUF_DYN_NAMESIZE 64
+
+/**
+ * Structure describing the parameters of a mbuf dynamic field.
+ */
+struct rte_mbuf_dynfield {
+ char name[RTE_MBUF_DYN_NAMESIZE]; /**< Name of the field. */
+ size_t size; /**< The number of bytes to reserve. */
+ size_t align; /**< The alignment constraint (power of 2). */
+ unsigned int flags; /**< Reserved for future use, must be 0. */
+};
+
+/**
+ * Structure describing the parameters of a mbuf dynamic flag.
+ */
+struct rte_mbuf_dynflag {
+ char name[RTE_MBUF_DYN_NAMESIZE]; /**< Name of the dynamic flag. */
+ unsigned int flags; /**< Reserved for future use, must be 0. */
+};
+
+/**
+ * Register space for a dynamic field in the mbuf structure.
+ *
+ * If the field is already registered (same name and parameters), its
+ * offset is returned.
+ *
+ * @param params
+ * A structure containing the requested parameters (name, size,
+ * alignment constraint and flags).
+ * @return
+ * The offset in the mbuf structure, or -1 on error.
+ * Possible values for rte_errno:
+ * - EINVAL: invalid parameters (size, align, or flags).
+ * - EEXIST: this name is already register with different parameters.
+ * - EPERM: called from a secondary process.
+ * - ENOENT: not enough room in mbuf.
+ * - ENOMEM: allocation failure.
+ * - ENAMETOOLONG: name does not ends with \0.
+ */
+__rte_experimental
+int rte_mbuf_dynfield_register(const struct rte_mbuf_dynfield *params);
+
+/**
+ * Lookup for a registered dynamic mbuf field.
+ *
+ * @param name
+ * A string identifying the dynamic field.
+ * @param params
+ * If not NULL, and if the lookup is successful, the structure is
+ * filled with the parameters of the dynamic field.
+ * @return
+ * The offset of this field in the mbuf structure, or -1 on error.
+ * Possible values for rte_errno:
+ * - ENOENT: no dynamic field matches this name.
+ */
+__rte_experimental
+int rte_mbuf_dynfield_lookup(const char *name,
+ struct rte_mbuf_dynfield *params);
+
+/**
+ * Register a dynamic flag in the mbuf structure.
+ *
+ * If the flag is already registered (same name and parameters), its
+ * offset is returned.
+ *
+ * @param params
+ * A structure containing the requested parameters of the dynamic
+ * flag (name and options).
+ * @return
+ * The number of the reserved bit, or -1 on error.
+ * Possible values for rte_errno:
+ * - EINVAL: invalid parameters (size, align, or flags).
+ * - EEXIST: this name is already register with different parameters.
+ * - EPERM: called from a secondary process.
+ * - ENOENT: no more flag available.
+ * - ENOMEM: allocation failure.
+ * - ENAMETOOLONG: name is longer than RTE_MBUF_DYN_NAMESIZE - 1.
+ */
+__rte_experimental
+int rte_mbuf_dynflag_register(const struct rte_mbuf_dynflag *params);
+
+/**
+ * Lookup for a registered dynamic mbuf flag.
+ *
+ * @param name
+ * A string identifying the dynamic flag.
+ * @param params
+ * If not NULL, and if the lookup is successful, the structure is
+ * filled with the parameters of the dynamic flag.
+ * @return
+ * The offset of this flag in the mbuf structure, or -1 on error.
+ * Possible values for rte_errno:
+ * - ENOENT: no dynamic flag matches this name.
+ */
+__rte_experimental
+int rte_mbuf_dynflag_lookup(const char *name,
+ struct rte_mbuf_dynflag *params);
+
+/**
+ * Helper macro to access to a dynamic field.
+ */
+#define RTE_MBUF_DYNFIELD(m, offset, type) ((type)((uintptr_t)(m) + (offset)))
+
+#endif
diff --git a/lib/librte_mbuf/rte_mbuf_version.map b/lib/librte_mbuf/rte_mbuf_version.map
index 2662a37bf..a98310570 100644
--- a/lib/librte_mbuf/rte_mbuf_version.map
+++ b/lib/librte_mbuf/rte_mbuf_version.map
@@ -50,4 +50,8 @@ EXPERIMENTAL {
global:
rte_mbuf_check;
+ rte_mbuf_dynfield_lookup;
+ rte_mbuf_dynfield_register;
+ rte_mbuf_dynflag_lookup;
+ rte_mbuf_dynflag_register;
} DPDK_18.08;
--
2.20.1
^ permalink raw reply [relevance 4%]
* Re: [dpdk-dev] [RFC PATCH 1/9] security: introduce CPU Crypto action type and API
2019-09-17 6:02 3% ` Akhil Goyal
@ 2019-09-18 7:44 4% ` Ananyev, Konstantin
2019-09-25 18:24 4% ` Ananyev, Konstantin
0 siblings, 1 reply; 200+ results
From: Ananyev, Konstantin @ 2019-09-18 7:44 UTC (permalink / raw)
To: Akhil Goyal, dev, De Lara Guarch, Pablo, Thomas Monjalon
Cc: Zhang, Roy Fan, Doherty, Declan, Anoob Joseph
Hi Akhil,
> > > > > > > > This action type allows the burst of symmetric crypto workload using
> > the
> > > > > > same
> > > > > > > > algorithm, key, and direction being processed by CPU cycles
> > > > synchronously.
> > > > > > > > This flexible action type does not require external hardware
> > involvement,
> > > > > > > > having the crypto workload processed synchronously, and is more
> > > > > > performant
> > > > > > > > than Cryptodev SW PMD due to the saved cycles on removed "async
> > > > mode
> > > > > > > > simulation" as well as 3 cacheline access of the crypto ops.
> > > > > > >
> > > > > > > Does that mean application will not call the cryptodev_enqueue_burst
> > and
> > > > > > corresponding dequeue burst.
> > > > > >
> > > > > > Yes, instead it just call rte_security_process_cpu_crypto_bulk(...)
> > > > > >
> > > > > > > It would be a new API something like process_packets and it will have
> > the
> > > > > > crypto processed packets while returning from the API?
> > > > > >
> > > > > > Yes, though the plan is that API will operate on raw data buffers, not
> > mbufs.
> > > > > >
> > > > > > >
> > > > > > > I still do not understand why we cannot do with the conventional
> > crypto lib
> > > > > > only.
> > > > > > > As far as I can understand, you are not doing any protocol processing
> > or
> > > > any
> > > > > > value add
> > > > > > > To the crypto processing. IMO, you just need a synchronous crypto
> > > > processing
> > > > > > API which
> > > > > > > Can be defined in cryptodev, you don't need to re-create a crypto
> > session
> > > > in
> > > > > > the name of
> > > > > > > Security session in the driver just to do a synchronous processing.
> > > > > >
> > > > > > I suppose your question is why not to have
> > > > > > rte_crypot_process_cpu_crypto_bulk(...) instead?
> > > > > > The main reason is that would require disruptive changes in existing
> > > > cryptodev
> > > > > > API
> > > > > > (would cause ABI/API breakage).
> > > > > > Session for RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO need some
> > extra
> > > > > > information
> > > > > > that normal crypto_sym_xform doesn't contain
> > > > > > (cipher offset from the start of the buffer, might be something extra in
> > > > future).
> > > > >
> > > > > Cipher offset will be part of rte_crypto_op.
> > > >
> > > > fill/read (+ alloc/free) is one of the main things that slowdown current
> > crypto-op
> > > > approach.
> > > > That's why the general idea - have all data that wouldn't change from packet
> > to
> > > > packet
> > > > included into the session and setup it once at session_init().
> > >
> > > I agree that you cannot use crypto-op.
> > > You can have the new API in crypto.
> > > As per the current patch, you only need cipher_offset which you can have it as
> > a parameter until
> > > You get it approved in the crypto xform. I believe it will be beneficial in case of
> > other crypto cases as well.
> > > We can have cipher offset at both places(crypto-op and cipher_xform). It will
> > give flexibility to the user to
> > > override it.
> >
> > After having another thought on your proposal:
> > Probably we can introduce new rte_crypto_sym_xform_types for CPU related
> > stuff here?
>
> I also thought of adding new xforms, but that wont serve the purpose for may be all the cases.
> You would be needing all information currently available in the current xforms.
> So if you are adding new fields in the new xform, the size will be more than that of the union of xforms.
> ABI breakage would still be there.
>
> If you think a valid compression of the AEAD xform can be done, then that can be done for each of the
> Xforms and we can have a solution to this issue.
I think that we can re-use iv.offset for our purposes (for crypto offset).
So for now we can make that path work without any ABI breakage.
Fan, please feel free to correct me here, if I missed something.
If in future we would need to add some extra information it might
require ABI breakage, though by now I don't envision anything particular to add.
Anyway, if there is no objection to go that way, we can try to make
these changes for v2.
>
> > Let say we can have :
> > num rte_crypto_sym_xform_type {
> > RTE_CRYPTO_SYM_XFORM_NOT_SPECIFIED = 0, /**< No xform specified
> > */
> > RTE_CRYPTO_SYM_XFORM_AUTH, /**< Authentication xform */
> > RTE_CRYPTO_SYM_XFORM_CIPHER, /**< Cipher xform */
> > RTE_CRYPTO_SYM_XFORM_AEAD /**< AEAD xform */
> > + RTE_CRYPTO_SYM_XFORM_CPU = INT32_MIN,
> > + RTE_CRYPTO_SYM_XFORM_CPU_AEAD = (RTE_CRYPTO_SYM_XFORM_CPU |
> > RTE_CRYPTO_SYM_XFORM_CPU),
>
> Instead of CPU I believe SYNC would be better.
I don't mind to name it to SYNC, but I'd like to outline,
that it's not really more CPU then generic SYNC API
(it doesn't pass IOVA for data buffers, etc., only VA).
>
> > /* same for auth and crypto xforms */
> > };
> >
> > Then we either can re-define some values in struct rte_crypto_aead_xform (via
> > unions),
> > or even have new struct rte_crypto_cpu_aead_xform (same for crypto and auth
> > xforms).
> > Then if PMD wants to support new sync API it would need to recognize new
> > xform types
> > and internally it might end up with different session structure (one for sync,
> > another for async mode).
> > That I think should allow us to introduce cpu_crypto as part of crypto-dev API
> > without ABI breakage.
> > What do you think?
> > Konstantin
> >
> > >
> > > >
> > > > > If you intend not to use rte_crypto_op
> > > > > You can pass this as an argument in the new cryptodev API.
> > > >
> > > > You mean extra parameter in rte_security_process_cpu_crypto_bulk()?
> > > > It can be in theory, but that solution looks a bit ugly:
> > > > why to pass for each call something that would be constant per session?
> > > > Again having that value constant per session might allow some extra
> > > > optimisations
> > > > That would be hard to achieve for dynamic case.
> > > > and not extendable:
> > > > Suppose tomorrow will need to add something extra (some new algorithm
> > > > support or so).
> > > > With what you proposing will need to new parameter to the function,
> > > > which means API breakage.
> > > >
> > > > > Something extra will also cause ABI breakage in security as well.
> > > > > So it will be same.
> > > >
> > > > I don't think it would.
> > > > AFAIK, right now this patch doesn't introduce any API/ABI breakage.
> > > > Iinside struct rte_security_session_conf we have a union of xforms
> > > > depending on session type.
> > > > So as long as cpu_crypto_xform wouldn't exceed sizes of other xform -
> > > > I believe no ABI breakage will appear.
> > > Agreed, it will not break ABI in case of security till we do not exceed current
> > size.
> > >
> > > Saving an ABI/API breakage is more important or placing the code at the
> > correct place.
> > > We need to find a tradeoff. Others can comment on this.
> > > @Thomas Monjalon, @De Lara Guarch, Pablo Any comments?
> > >
> > > >
> > > >
> > > > >
> > > > > > Also right now there is no way to add new type of crypto_sym_session
> > > > without
> > > > > > either breaking existing crypto-dev ABI/API or introducing new structure
> > > > > > (rte_crypto_sym_cpu_session or so) for that.
> > > > >
> > > > > What extra info is required in rte_cryptodev_sym_session to get the
> > > > rte_crypto_sym_cpu_session.
> > > >
> > > > Right now - just cipher_offset (see above).
> > > > What else in future (if any) - don't know.
> > > >
> > > > > I don't think there is any.
> > > > > I believe the same crypto session will be able to work synchronously as well.
> > > >
> > > > Exactly the same - problematically, see above.
> > > >
> > > > > We would only need a new API to perform synchronous actions.
> > > > > That will reduce the duplication code significantly
> > > > > in the driver to support 2 different kind of APIs with similar code inside.
> > > > > Please correct me in case I am missing something.
> > > >
> > > > To add new API into crypto-dev would also require changes in the PMD,
> > > > it wouldn't come totally free and I believe would require roughly the same
> > > > amount of changes.
> > >
> > > It will be required only in the PMDs which support it and would be minimal.
> > > You would need a feature flag, support for that synchronous API. Session
> > information will
> > > already be there in the session. The changes wrt cipher_offset need to be
> > added
> > > but with some default value to identify override will be done or not.
> > >
> > > >
> > > > >
> > > > >
> > > > > > While rte_security is designed in a way that we can add new session
> > types
> > > > and
> > > > > > related parameters without causing API/ABI breakage.
> > > > >
> > > > > Yes the intent is to add new sessions based on various protocols that can
> > be
> > > > supported by the driver.
> > > >
> > > > Various protocols and different types of sessions (and devices they belong
> > to).
> > > > Let say right now we have INLINE_CRYPTO, INLINE_PROTO,
> > LOOKASIDE_PROTO,
> > > > etc.
> > > > Here we introduce new type of session.
> > >
> > > What is the new value add to the existing sessions. The changes that we are
> > doing
> > > here is just to avoid an API/ABI breakage. The synchronous processing can
> > happen on both
> > > crypto and security session. This would mean, only the processing API should
> > be defined,
> > > rest all should be already there in the sessions.
> > > In All other cases, INLINE - eth device was not having any format to perform
> > crypto op
> > > LOOKASIDE - PROTO - add protocol specific sessions which is not available in
> > crypto.
> > >
> > > >
> > > > > It is not that we should find it as an alternative to cryptodev and using it
> > just
> > > > because it will not cause
> > > > > ABI/API breakage.
> > > >
> > > > I am considering this new API as an alternative to existing ones, but as an
> > > > extension.
> > > > Existing crypto-op API has its own advantages (generic), and I think we
> > should
> > > > keep it supported by all crypto-devs.
> > > > From other side rte_security is an extendable framework that suits the
> > purpose:
> > > > allows easily (and yes without ABI breakage) introduce new API for special
> > type
> > > > of crypto-dev (SW based).
> > > >
> > > >
> > >
> > > Adding a synchronous processing API is understandable and can be added in
> > both
> > > Crypto as well as Security, but a new action type for it is not required.
> > > Now whether to support that, we have ABI/API breakage, that is a different
> > issue.
> > > And we may have to deal with it if no other option is there.
> > >
> > > >
> > > >
> > > >
> > > > > IMO the code should be placed where its intent is.
> > > > >
> > > > > >
> > > > > > BTW, what is your concern with proposed approach (via rte_security)?
> > > > > > From my perspective it is a lightweight change and it is totally optional
> > > > > > for the crypto PMDs to support it or not.
> > > > > > Konstantin
> > > > > >
> > > > > > > >
> > > > > > > > AESNI-GCM and AESNI-MB PMDs are updated with this support.
> > There is
> > > > a
> > > > > > small
> > > > > > > > performance test app under
> > app/test/security_aesni_gcm(mb)_perftest
> > > > to
> > > > > > > > prove.
> > > > > > > >
> > > > > > > > For the new API
> > > > > > > > The packet is sent to the crypto device for symmetric crypto
> > > > > > > > processing. The device will encrypt or decrypt the buffer based on the
> > > > > > session
> > > > > > > > data specified and preprocessed in the security session. Different
> > > > > > > > than the inline or lookaside modes, when the function exits, the user
> > will
> > > > > > > > expect the buffers are either processed successfully, or having the
> > error
> > > > > > number
> > > > > > > > assigned to the appropriate index of the status array.
> > > > > > > >
> > > > > > > > Will update the program's guide in the v1 patch.
> > > > > > > >
> > > > > > > > Regards,
> > > > > > > > Fan
> > > > > > > >
> > > > > > > > > -----Original Message-----
> > > > > > > > > From: Akhil Goyal [mailto:akhil.goyal@nxp.com]
> > > > > > > > > Sent: Wednesday, September 4, 2019 11:33 AM
> > > > > > > > > To: Zhang, Roy Fan <roy.fan.zhang@intel.com>; dev@dpdk.org
> > > > > > > > > Cc: Ananyev, Konstantin <konstantin.ananyev@intel.com>; Doherty,
> > > > > > Declan
> > > > > > > > > <declan.doherty@intel.com>; De Lara Guarch, Pablo
> > > > > > > > > <pablo.de.lara.guarch@intel.com>
> > > > > > > > > Subject: RE: [RFC PATCH 1/9] security: introduce CPU Crypto action
> > > > type
> > > > > > and
> > > > > > > > > API
> > > > > > > > >
> > > > > > > > > Hi Fan,
> > > > > > > > >
> > > > > > > > > >
> > > > > > > > > > This patch introduce new
> > > > RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO
> > > > > > > > > action
> > > > > > > > > > type to security library. The type represents performing crypto
> > > > > > > > > > operation with CPU cycles. The patch also includes a new API to
> > > > > > > > > > process crypto operations in bulk and the function pointers for
> > PMDs.
> > > > > > > > > >
> > > > > > > > > I am not able to get the flow of execution for this action type.
> > Could
> > > > you
> > > > > > > > > please elaborate the flow in the documentation. If not in
> > > > documentation
> > > > > > > > > right now, then please elaborate the flow in cover letter.
> > > > > > > > > Also I see that there are new APIs for processing crypto operations
> > in
> > > > bulk.
> > > > > > > > > What does that mean. How are they different from the existing APIs
> > > > which
> > > > > > > > > are also handling bulk crypto ops depending on the budget.
> > > > > > > > >
> > > > > > > > >
> > > > > > > > > -Akhil
^ permalink raw reply [relevance 4%]
* Re: [dpdk-dev] [RFC PATCH 1/9] security: introduce CPU Crypto action type and API
2019-09-16 14:53 3% ` Ananyev, Konstantin
2019-09-16 15:08 0% ` Ananyev, Konstantin
@ 2019-09-17 6:02 3% ` Akhil Goyal
2019-09-18 7:44 4% ` Ananyev, Konstantin
1 sibling, 1 reply; 200+ results
From: Akhil Goyal @ 2019-09-17 6:02 UTC (permalink / raw)
To: Ananyev, Konstantin, dev, De Lara Guarch, Pablo, Thomas Monjalon
Cc: Zhang, Roy Fan, Doherty, Declan, Anoob Joseph
Hi Konstantin,
>
> Hi Akhil,
>
> > > > > > > This action type allows the burst of symmetric crypto workload using
> the
> > > > > same
> > > > > > > algorithm, key, and direction being processed by CPU cycles
> > > synchronously.
> > > > > > > This flexible action type does not require external hardware
> involvement,
> > > > > > > having the crypto workload processed synchronously, and is more
> > > > > performant
> > > > > > > than Cryptodev SW PMD due to the saved cycles on removed "async
> > > mode
> > > > > > > simulation" as well as 3 cacheline access of the crypto ops.
> > > > > >
> > > > > > Does that mean application will not call the cryptodev_enqueue_burst
> and
> > > > > corresponding dequeue burst.
> > > > >
> > > > > Yes, instead it just call rte_security_process_cpu_crypto_bulk(...)
> > > > >
> > > > > > It would be a new API something like process_packets and it will have
> the
> > > > > crypto processed packets while returning from the API?
> > > > >
> > > > > Yes, though the plan is that API will operate on raw data buffers, not
> mbufs.
> > > > >
> > > > > >
> > > > > > I still do not understand why we cannot do with the conventional
> crypto lib
> > > > > only.
> > > > > > As far as I can understand, you are not doing any protocol processing
> or
> > > any
> > > > > value add
> > > > > > To the crypto processing. IMO, you just need a synchronous crypto
> > > processing
> > > > > API which
> > > > > > Can be defined in cryptodev, you don't need to re-create a crypto
> session
> > > in
> > > > > the name of
> > > > > > Security session in the driver just to do a synchronous processing.
> > > > >
> > > > > I suppose your question is why not to have
> > > > > rte_crypot_process_cpu_crypto_bulk(...) instead?
> > > > > The main reason is that would require disruptive changes in existing
> > > cryptodev
> > > > > API
> > > > > (would cause ABI/API breakage).
> > > > > Session for RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO need some
> extra
> > > > > information
> > > > > that normal crypto_sym_xform doesn't contain
> > > > > (cipher offset from the start of the buffer, might be something extra in
> > > future).
> > > >
> > > > Cipher offset will be part of rte_crypto_op.
> > >
> > > fill/read (+ alloc/free) is one of the main things that slowdown current
> crypto-op
> > > approach.
> > > That's why the general idea - have all data that wouldn't change from packet
> to
> > > packet
> > > included into the session and setup it once at session_init().
> >
> > I agree that you cannot use crypto-op.
> > You can have the new API in crypto.
> > As per the current patch, you only need cipher_offset which you can have it as
> a parameter until
> > You get it approved in the crypto xform. I believe it will be beneficial in case of
> other crypto cases as well.
> > We can have cipher offset at both places(crypto-op and cipher_xform). It will
> give flexibility to the user to
> > override it.
>
> After having another thought on your proposal:
> Probably we can introduce new rte_crypto_sym_xform_types for CPU related
> stuff here?
I also thought of adding new xforms, but that wont serve the purpose for may be all the cases.
You would be needing all information currently available in the current xforms.
So if you are adding new fields in the new xform, the size will be more than that of the union of xforms.
ABI breakage would still be there.
If you think a valid compression of the AEAD xform can be done, then that can be done for each of the
Xforms and we can have a solution to this issue.
> Let say we can have :
> num rte_crypto_sym_xform_type {
> RTE_CRYPTO_SYM_XFORM_NOT_SPECIFIED = 0, /**< No xform specified
> */
> RTE_CRYPTO_SYM_XFORM_AUTH, /**< Authentication xform */
> RTE_CRYPTO_SYM_XFORM_CIPHER, /**< Cipher xform */
> RTE_CRYPTO_SYM_XFORM_AEAD /**< AEAD xform */
> + RTE_CRYPTO_SYM_XFORM_CPU = INT32_MIN,
> + RTE_CRYPTO_SYM_XFORM_CPU_AEAD = (RTE_CRYPTO_SYM_XFORM_CPU |
> RTE_CRYPTO_SYM_XFORM_CPU),
Instead of CPU I believe SYNC would be better.
> /* same for auth and crypto xforms */
> };
>
> Then we either can re-define some values in struct rte_crypto_aead_xform (via
> unions),
> or even have new struct rte_crypto_cpu_aead_xform (same for crypto and auth
> xforms).
> Then if PMD wants to support new sync API it would need to recognize new
> xform types
> and internally it might end up with different session structure (one for sync,
> another for async mode).
> That I think should allow us to introduce cpu_crypto as part of crypto-dev API
> without ABI breakage.
> What do you think?
> Konstantin
>
> >
> > >
> > > > If you intend not to use rte_crypto_op
> > > > You can pass this as an argument in the new cryptodev API.
> > >
> > > You mean extra parameter in rte_security_process_cpu_crypto_bulk()?
> > > It can be in theory, but that solution looks a bit ugly:
> > > why to pass for each call something that would be constant per session?
> > > Again having that value constant per session might allow some extra
> > > optimisations
> > > That would be hard to achieve for dynamic case.
> > > and not extendable:
> > > Suppose tomorrow will need to add something extra (some new algorithm
> > > support or so).
> > > With what you proposing will need to new parameter to the function,
> > > which means API breakage.
> > >
> > > > Something extra will also cause ABI breakage in security as well.
> > > > So it will be same.
> > >
> > > I don't think it would.
> > > AFAIK, right now this patch doesn't introduce any API/ABI breakage.
> > > Iinside struct rte_security_session_conf we have a union of xforms
> > > depending on session type.
> > > So as long as cpu_crypto_xform wouldn't exceed sizes of other xform -
> > > I believe no ABI breakage will appear.
> > Agreed, it will not break ABI in case of security till we do not exceed current
> size.
> >
> > Saving an ABI/API breakage is more important or placing the code at the
> correct place.
> > We need to find a tradeoff. Others can comment on this.
> > @Thomas Monjalon, @De Lara Guarch, Pablo Any comments?
> >
> > >
> > >
> > > >
> > > > > Also right now there is no way to add new type of crypto_sym_session
> > > without
> > > > > either breaking existing crypto-dev ABI/API or introducing new structure
> > > > > (rte_crypto_sym_cpu_session or so) for that.
> > > >
> > > > What extra info is required in rte_cryptodev_sym_session to get the
> > > rte_crypto_sym_cpu_session.
> > >
> > > Right now - just cipher_offset (see above).
> > > What else in future (if any) - don't know.
> > >
> > > > I don't think there is any.
> > > > I believe the same crypto session will be able to work synchronously as well.
> > >
> > > Exactly the same - problematically, see above.
> > >
> > > > We would only need a new API to perform synchronous actions.
> > > > That will reduce the duplication code significantly
> > > > in the driver to support 2 different kind of APIs with similar code inside.
> > > > Please correct me in case I am missing something.
> > >
> > > To add new API into crypto-dev would also require changes in the PMD,
> > > it wouldn't come totally free and I believe would require roughly the same
> > > amount of changes.
> >
> > It will be required only in the PMDs which support it and would be minimal.
> > You would need a feature flag, support for that synchronous API. Session
> information will
> > already be there in the session. The changes wrt cipher_offset need to be
> added
> > but with some default value to identify override will be done or not.
> >
> > >
> > > >
> > > >
> > > > > While rte_security is designed in a way that we can add new session
> types
> > > and
> > > > > related parameters without causing API/ABI breakage.
> > > >
> > > > Yes the intent is to add new sessions based on various protocols that can
> be
> > > supported by the driver.
> > >
> > > Various protocols and different types of sessions (and devices they belong
> to).
> > > Let say right now we have INLINE_CRYPTO, INLINE_PROTO,
> LOOKASIDE_PROTO,
> > > etc.
> > > Here we introduce new type of session.
> >
> > What is the new value add to the existing sessions. The changes that we are
> doing
> > here is just to avoid an API/ABI breakage. The synchronous processing can
> happen on both
> > crypto and security session. This would mean, only the processing API should
> be defined,
> > rest all should be already there in the sessions.
> > In All other cases, INLINE - eth device was not having any format to perform
> crypto op
> > LOOKASIDE - PROTO - add protocol specific sessions which is not available in
> crypto.
> >
> > >
> > > > It is not that we should find it as an alternative to cryptodev and using it
> just
> > > because it will not cause
> > > > ABI/API breakage.
> > >
> > > I am considering this new API as an alternative to existing ones, but as an
> > > extension.
> > > Existing crypto-op API has its own advantages (generic), and I think we
> should
> > > keep it supported by all crypto-devs.
> > > From other side rte_security is an extendable framework that suits the
> purpose:
> > > allows easily (and yes without ABI breakage) introduce new API for special
> type
> > > of crypto-dev (SW based).
> > >
> > >
> >
> > Adding a synchronous processing API is understandable and can be added in
> both
> > Crypto as well as Security, but a new action type for it is not required.
> > Now whether to support that, we have ABI/API breakage, that is a different
> issue.
> > And we may have to deal with it if no other option is there.
> >
> > >
> > >
> > >
> > > > IMO the code should be placed where its intent is.
> > > >
> > > > >
> > > > > BTW, what is your concern with proposed approach (via rte_security)?
> > > > > From my perspective it is a lightweight change and it is totally optional
> > > > > for the crypto PMDs to support it or not.
> > > > > Konstantin
> > > > >
> > > > > > >
> > > > > > > AESNI-GCM and AESNI-MB PMDs are updated with this support.
> There is
> > > a
> > > > > small
> > > > > > > performance test app under
> app/test/security_aesni_gcm(mb)_perftest
> > > to
> > > > > > > prove.
> > > > > > >
> > > > > > > For the new API
> > > > > > > The packet is sent to the crypto device for symmetric crypto
> > > > > > > processing. The device will encrypt or decrypt the buffer based on the
> > > > > session
> > > > > > > data specified and preprocessed in the security session. Different
> > > > > > > than the inline or lookaside modes, when the function exits, the user
> will
> > > > > > > expect the buffers are either processed successfully, or having the
> error
> > > > > number
> > > > > > > assigned to the appropriate index of the status array.
> > > > > > >
> > > > > > > Will update the program's guide in the v1 patch.
> > > > > > >
> > > > > > > Regards,
> > > > > > > Fan
> > > > > > >
> > > > > > > > -----Original Message-----
> > > > > > > > From: Akhil Goyal [mailto:akhil.goyal@nxp.com]
> > > > > > > > Sent: Wednesday, September 4, 2019 11:33 AM
> > > > > > > > To: Zhang, Roy Fan <roy.fan.zhang@intel.com>; dev@dpdk.org
> > > > > > > > Cc: Ananyev, Konstantin <konstantin.ananyev@intel.com>; Doherty,
> > > > > Declan
> > > > > > > > <declan.doherty@intel.com>; De Lara Guarch, Pablo
> > > > > > > > <pablo.de.lara.guarch@intel.com>
> > > > > > > > Subject: RE: [RFC PATCH 1/9] security: introduce CPU Crypto action
> > > type
> > > > > and
> > > > > > > > API
> > > > > > > >
> > > > > > > > Hi Fan,
> > > > > > > >
> > > > > > > > >
> > > > > > > > > This patch introduce new
> > > RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO
> > > > > > > > action
> > > > > > > > > type to security library. The type represents performing crypto
> > > > > > > > > operation with CPU cycles. The patch also includes a new API to
> > > > > > > > > process crypto operations in bulk and the function pointers for
> PMDs.
> > > > > > > > >
> > > > > > > > I am not able to get the flow of execution for this action type.
> Could
> > > you
> > > > > > > > please elaborate the flow in the documentation. If not in
> > > documentation
> > > > > > > > right now, then please elaborate the flow in cover letter.
> > > > > > > > Also I see that there are new APIs for processing crypto operations
> in
> > > bulk.
> > > > > > > > What does that mean. How are they different from the existing APIs
> > > which
> > > > > > > > are also handling bulk crypto ops depending on the budget.
> > > > > > > >
> > > > > > > >
> > > > > > > > -Akhil
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] [RFC PATCH 1/9] security: introduce CPU Crypto action type and API
2019-09-16 14:53 3% ` Ananyev, Konstantin
@ 2019-09-16 15:08 0% ` Ananyev, Konstantin
2019-09-17 6:02 3% ` Akhil Goyal
1 sibling, 0 replies; 200+ results
From: Ananyev, Konstantin @ 2019-09-16 15:08 UTC (permalink / raw)
To: Ananyev, Konstantin, Akhil Goyal, dev, De Lara Guarch, Pablo,
Thomas Monjalon
Cc: Zhang, Roy Fan, Doherty, Declan, Anoob Joseph
> Hi Akhil,
>
> > > > > > > This action type allows the burst of symmetric crypto workload using the
> > > > > same
> > > > > > > algorithm, key, and direction being processed by CPU cycles
> > > synchronously.
> > > > > > > This flexible action type does not require external hardware involvement,
> > > > > > > having the crypto workload processed synchronously, and is more
> > > > > performant
> > > > > > > than Cryptodev SW PMD due to the saved cycles on removed "async
> > > mode
> > > > > > > simulation" as well as 3 cacheline access of the crypto ops.
> > > > > >
> > > > > > Does that mean application will not call the cryptodev_enqueue_burst and
> > > > > corresponding dequeue burst.
> > > > >
> > > > > Yes, instead it just call rte_security_process_cpu_crypto_bulk(...)
> > > > >
> > > > > > It would be a new API something like process_packets and it will have the
> > > > > crypto processed packets while returning from the API?
> > > > >
> > > > > Yes, though the plan is that API will operate on raw data buffers, not mbufs.
> > > > >
> > > > > >
> > > > > > I still do not understand why we cannot do with the conventional crypto lib
> > > > > only.
> > > > > > As far as I can understand, you are not doing any protocol processing or
> > > any
> > > > > value add
> > > > > > To the crypto processing. IMO, you just need a synchronous crypto
> > > processing
> > > > > API which
> > > > > > Can be defined in cryptodev, you don't need to re-create a crypto session
> > > in
> > > > > the name of
> > > > > > Security session in the driver just to do a synchronous processing.
> > > > >
> > > > > I suppose your question is why not to have
> > > > > rte_crypot_process_cpu_crypto_bulk(...) instead?
> > > > > The main reason is that would require disruptive changes in existing
> > > cryptodev
> > > > > API
> > > > > (would cause ABI/API breakage).
> > > > > Session for RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO need some extra
> > > > > information
> > > > > that normal crypto_sym_xform doesn't contain
> > > > > (cipher offset from the start of the buffer, might be something extra in
> > > future).
> > > >
> > > > Cipher offset will be part of rte_crypto_op.
> > >
> > > fill/read (+ alloc/free) is one of the main things that slowdown current crypto-op
> > > approach.
> > > That's why the general idea - have all data that wouldn't change from packet to
> > > packet
> > > included into the session and setup it once at session_init().
> >
> > I agree that you cannot use crypto-op.
> > You can have the new API in crypto.
> > As per the current patch, you only need cipher_offset which you can have it as a parameter until
> > You get it approved in the crypto xform. I believe it will be beneficial in case of other crypto cases as well.
> > We can have cipher offset at both places(crypto-op and cipher_xform). It will give flexibility to the user to
> > override it.
>
> After having another thought on your proposal:
> Probably we can introduce new rte_crypto_sym_xform_types for CPU related stuff here?
> Let say we can have :
> num rte_crypto_sym_xform_type {
> RTE_CRYPTO_SYM_XFORM_NOT_SPECIFIED = 0, /**< No xform specified */
> RTE_CRYPTO_SYM_XFORM_AUTH, /**< Authentication xform */
> RTE_CRYPTO_SYM_XFORM_CIPHER, /**< Cipher xform */
> RTE_CRYPTO_SYM_XFORM_AEAD /**< AEAD xform */
> + RTE_CRYPTO_SYM_XFORM_CPU = INT32_MIN,
> + RTE_CRYPTO_SYM_XFORM_CPU_AEAD = (RTE_CRYPTO_SYM_XFORM_CPU | RTE_CRYPTO_SYM_XFORM_CPU),
Meant
RTE_CRYPTO_SYM_XFORM_CPU_AEAD = (RTE_CRYPTO_SYM_XFORM_CPU | RTE_CRYPTO_SYM_XFORM_AEAD),
of course.
> /* same for auth and crypto xforms */
> };
>
> Then we either can re-define some values in struct rte_crypto_aead_xform (via unions),
> or even have new struct rte_crypto_cpu_aead_xform (same for crypto and auth xforms).
> Then if PMD wants to support new sync API it would need to recognize new xform types
> and internally it might end up with different session structure (one for sync, another for async mode).
> That I think should allow us to introduce cpu_crypto as part of crypto-dev API without ABI breakage.
> What do you think?
> Konstantin
>
> >
> > >
> > > > If you intend not to use rte_crypto_op
> > > > You can pass this as an argument in the new cryptodev API.
> > >
> > > You mean extra parameter in rte_security_process_cpu_crypto_bulk()?
> > > It can be in theory, but that solution looks a bit ugly:
> > > why to pass for each call something that would be constant per session?
> > > Again having that value constant per session might allow some extra
> > > optimisations
> > > That would be hard to achieve for dynamic case.
> > > and not extendable:
> > > Suppose tomorrow will need to add something extra (some new algorithm
> > > support or so).
> > > With what you proposing will need to new parameter to the function,
> > > which means API breakage.
> > >
> > > > Something extra will also cause ABI breakage in security as well.
> > > > So it will be same.
> > >
> > > I don't think it would.
> > > AFAIK, right now this patch doesn't introduce any API/ABI breakage.
> > > Iinside struct rte_security_session_conf we have a union of xforms
> > > depending on session type.
> > > So as long as cpu_crypto_xform wouldn't exceed sizes of other xform -
> > > I believe no ABI breakage will appear.
> > Agreed, it will not break ABI in case of security till we do not exceed current size.
> >
> > Saving an ABI/API breakage is more important or placing the code at the correct place.
> > We need to find a tradeoff. Others can comment on this.
> > @Thomas Monjalon, @De Lara Guarch, Pablo Any comments?
> >
> > >
> > >
> > > >
> > > > > Also right now there is no way to add new type of crypto_sym_session
> > > without
> > > > > either breaking existing crypto-dev ABI/API or introducing new structure
> > > > > (rte_crypto_sym_cpu_session or so) for that.
> > > >
> > > > What extra info is required in rte_cryptodev_sym_session to get the
> > > rte_crypto_sym_cpu_session.
> > >
> > > Right now - just cipher_offset (see above).
> > > What else in future (if any) - don't know.
> > >
> > > > I don't think there is any.
> > > > I believe the same crypto session will be able to work synchronously as well.
> > >
> > > Exactly the same - problematically, see above.
> > >
> > > > We would only need a new API to perform synchronous actions.
> > > > That will reduce the duplication code significantly
> > > > in the driver to support 2 different kind of APIs with similar code inside.
> > > > Please correct me in case I am missing something.
> > >
> > > To add new API into crypto-dev would also require changes in the PMD,
> > > it wouldn't come totally free and I believe would require roughly the same
> > > amount of changes.
> >
> > It will be required only in the PMDs which support it and would be minimal.
> > You would need a feature flag, support for that synchronous API. Session information will
> > already be there in the session. The changes wrt cipher_offset need to be added
> > but with some default value to identify override will be done or not.
> >
> > >
> > > >
> > > >
> > > > > While rte_security is designed in a way that we can add new session types
> > > and
> > > > > related parameters without causing API/ABI breakage.
> > > >
> > > > Yes the intent is to add new sessions based on various protocols that can be
> > > supported by the driver.
> > >
> > > Various protocols and different types of sessions (and devices they belong to).
> > > Let say right now we have INLINE_CRYPTO, INLINE_PROTO, LOOKASIDE_PROTO,
> > > etc.
> > > Here we introduce new type of session.
> >
> > What is the new value add to the existing sessions. The changes that we are doing
> > here is just to avoid an API/ABI breakage. The synchronous processing can happen on both
> > crypto and security session. This would mean, only the processing API should be defined,
> > rest all should be already there in the sessions.
> > In All other cases, INLINE - eth device was not having any format to perform crypto op
> > LOOKASIDE - PROTO - add protocol specific sessions which is not available in crypto.
> >
> > >
> > > > It is not that we should find it as an alternative to cryptodev and using it just
> > > because it will not cause
> > > > ABI/API breakage.
> > >
> > > I am considering this new API as an alternative to existing ones, but as an
> > > extension.
> > > Existing crypto-op API has its own advantages (generic), and I think we should
> > > keep it supported by all crypto-devs.
> > > From other side rte_security is an extendable framework that suits the purpose:
> > > allows easily (and yes without ABI breakage) introduce new API for special type
> > > of crypto-dev (SW based).
> > >
> > >
> >
> > Adding a synchronous processing API is understandable and can be added in both
> > Crypto as well as Security, but a new action type for it is not required.
> > Now whether to support that, we have ABI/API breakage, that is a different issue.
> > And we may have to deal with it if no other option is there.
> >
> > >
> > >
> > >
> > > > IMO the code should be placed where its intent is.
> > > >
> > > > >
> > > > > BTW, what is your concern with proposed approach (via rte_security)?
> > > > > From my perspective it is a lightweight change and it is totally optional
> > > > > for the crypto PMDs to support it or not.
> > > > > Konstantin
> > > > >
> > > > > > >
> > > > > > > AESNI-GCM and AESNI-MB PMDs are updated with this support. There is
> > > a
> > > > > small
> > > > > > > performance test app under app/test/security_aesni_gcm(mb)_perftest
> > > to
> > > > > > > prove.
> > > > > > >
> > > > > > > For the new API
> > > > > > > The packet is sent to the crypto device for symmetric crypto
> > > > > > > processing. The device will encrypt or decrypt the buffer based on the
> > > > > session
> > > > > > > data specified and preprocessed in the security session. Different
> > > > > > > than the inline or lookaside modes, when the function exits, the user will
> > > > > > > expect the buffers are either processed successfully, or having the error
> > > > > number
> > > > > > > assigned to the appropriate index of the status array.
> > > > > > >
> > > > > > > Will update the program's guide in the v1 patch.
> > > > > > >
> > > > > > > Regards,
> > > > > > > Fan
> > > > > > >
> > > > > > > > -----Original Message-----
> > > > > > > > From: Akhil Goyal [mailto:akhil.goyal@nxp.com]
> > > > > > > > Sent: Wednesday, September 4, 2019 11:33 AM
> > > > > > > > To: Zhang, Roy Fan <roy.fan.zhang@intel.com>; dev@dpdk.org
> > > > > > > > Cc: Ananyev, Konstantin <konstantin.ananyev@intel.com>; Doherty,
> > > > > Declan
> > > > > > > > <declan.doherty@intel.com>; De Lara Guarch, Pablo
> > > > > > > > <pablo.de.lara.guarch@intel.com>
> > > > > > > > Subject: RE: [RFC PATCH 1/9] security: introduce CPU Crypto action
> > > type
> > > > > and
> > > > > > > > API
> > > > > > > >
> > > > > > > > Hi Fan,
> > > > > > > >
> > > > > > > > >
> > > > > > > > > This patch introduce new
> > > RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO
> > > > > > > > action
> > > > > > > > > type to security library. The type represents performing crypto
> > > > > > > > > operation with CPU cycles. The patch also includes a new API to
> > > > > > > > > process crypto operations in bulk and the function pointers for PMDs.
> > > > > > > > >
> > > > > > > > I am not able to get the flow of execution for this action type. Could
> > > you
> > > > > > > > please elaborate the flow in the documentation. If not in
> > > documentation
> > > > > > > > right now, then please elaborate the flow in cover letter.
> > > > > > > > Also I see that there are new APIs for processing crypto operations in
> > > bulk.
> > > > > > > > What does that mean. How are they different from the existing APIs
> > > which
> > > > > > > > are also handling bulk crypto ops depending on the budget.
> > > > > > > >
> > > > > > > >
> > > > > > > > -Akhil
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [RFC PATCH 1/9] security: introduce CPU Crypto action type and API
2019-09-12 14:12 5% ` Akhil Goyal
@ 2019-09-16 14:53 3% ` Ananyev, Konstantin
2019-09-16 15:08 0% ` Ananyev, Konstantin
2019-09-17 6:02 3% ` Akhil Goyal
0 siblings, 2 replies; 200+ results
From: Ananyev, Konstantin @ 2019-09-16 14:53 UTC (permalink / raw)
To: Akhil Goyal, dev, De Lara Guarch, Pablo, Thomas Monjalon
Cc: Zhang, Roy Fan, Doherty, Declan, Anoob Joseph
Hi Akhil,
> > > > > > This action type allows the burst of symmetric crypto workload using the
> > > > same
> > > > > > algorithm, key, and direction being processed by CPU cycles
> > synchronously.
> > > > > > This flexible action type does not require external hardware involvement,
> > > > > > having the crypto workload processed synchronously, and is more
> > > > performant
> > > > > > than Cryptodev SW PMD due to the saved cycles on removed "async
> > mode
> > > > > > simulation" as well as 3 cacheline access of the crypto ops.
> > > > >
> > > > > Does that mean application will not call the cryptodev_enqueue_burst and
> > > > corresponding dequeue burst.
> > > >
> > > > Yes, instead it just call rte_security_process_cpu_crypto_bulk(...)
> > > >
> > > > > It would be a new API something like process_packets and it will have the
> > > > crypto processed packets while returning from the API?
> > > >
> > > > Yes, though the plan is that API will operate on raw data buffers, not mbufs.
> > > >
> > > > >
> > > > > I still do not understand why we cannot do with the conventional crypto lib
> > > > only.
> > > > > As far as I can understand, you are not doing any protocol processing or
> > any
> > > > value add
> > > > > To the crypto processing. IMO, you just need a synchronous crypto
> > processing
> > > > API which
> > > > > Can be defined in cryptodev, you don't need to re-create a crypto session
> > in
> > > > the name of
> > > > > Security session in the driver just to do a synchronous processing.
> > > >
> > > > I suppose your question is why not to have
> > > > rte_crypot_process_cpu_crypto_bulk(...) instead?
> > > > The main reason is that would require disruptive changes in existing
> > cryptodev
> > > > API
> > > > (would cause ABI/API breakage).
> > > > Session for RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO need some extra
> > > > information
> > > > that normal crypto_sym_xform doesn't contain
> > > > (cipher offset from the start of the buffer, might be something extra in
> > future).
> > >
> > > Cipher offset will be part of rte_crypto_op.
> >
> > fill/read (+ alloc/free) is one of the main things that slowdown current crypto-op
> > approach.
> > That's why the general idea - have all data that wouldn't change from packet to
> > packet
> > included into the session and setup it once at session_init().
>
> I agree that you cannot use crypto-op.
> You can have the new API in crypto.
> As per the current patch, you only need cipher_offset which you can have it as a parameter until
> You get it approved in the crypto xform. I believe it will be beneficial in case of other crypto cases as well.
> We can have cipher offset at both places(crypto-op and cipher_xform). It will give flexibility to the user to
> override it.
After having another thought on your proposal:
Probably we can introduce new rte_crypto_sym_xform_types for CPU related stuff here?
Let say we can have :
num rte_crypto_sym_xform_type {
RTE_CRYPTO_SYM_XFORM_NOT_SPECIFIED = 0, /**< No xform specified */
RTE_CRYPTO_SYM_XFORM_AUTH, /**< Authentication xform */
RTE_CRYPTO_SYM_XFORM_CIPHER, /**< Cipher xform */
RTE_CRYPTO_SYM_XFORM_AEAD /**< AEAD xform */
+ RTE_CRYPTO_SYM_XFORM_CPU = INT32_MIN,
+ RTE_CRYPTO_SYM_XFORM_CPU_AEAD = (RTE_CRYPTO_SYM_XFORM_CPU | RTE_CRYPTO_SYM_XFORM_CPU),
/* same for auth and crypto xforms */
};
Then we either can re-define some values in struct rte_crypto_aead_xform (via unions),
or even have new struct rte_crypto_cpu_aead_xform (same for crypto and auth xforms).
Then if PMD wants to support new sync API it would need to recognize new xform types
and internally it might end up with different session structure (one for sync, another for async mode).
That I think should allow us to introduce cpu_crypto as part of crypto-dev API without ABI breakage.
What do you think?
Konstantin
>
> >
> > > If you intend not to use rte_crypto_op
> > > You can pass this as an argument in the new cryptodev API.
> >
> > You mean extra parameter in rte_security_process_cpu_crypto_bulk()?
> > It can be in theory, but that solution looks a bit ugly:
> > why to pass for each call something that would be constant per session?
> > Again having that value constant per session might allow some extra
> > optimisations
> > That would be hard to achieve for dynamic case.
> > and not extendable:
> > Suppose tomorrow will need to add something extra (some new algorithm
> > support or so).
> > With what you proposing will need to new parameter to the function,
> > which means API breakage.
> >
> > > Something extra will also cause ABI breakage in security as well.
> > > So it will be same.
> >
> > I don't think it would.
> > AFAIK, right now this patch doesn't introduce any API/ABI breakage.
> > Iinside struct rte_security_session_conf we have a union of xforms
> > depending on session type.
> > So as long as cpu_crypto_xform wouldn't exceed sizes of other xform -
> > I believe no ABI breakage will appear.
> Agreed, it will not break ABI in case of security till we do not exceed current size.
>
> Saving an ABI/API breakage is more important or placing the code at the correct place.
> We need to find a tradeoff. Others can comment on this.
> @Thomas Monjalon, @De Lara Guarch, Pablo Any comments?
>
> >
> >
> > >
> > > > Also right now there is no way to add new type of crypto_sym_session
> > without
> > > > either breaking existing crypto-dev ABI/API or introducing new structure
> > > > (rte_crypto_sym_cpu_session or so) for that.
> > >
> > > What extra info is required in rte_cryptodev_sym_session to get the
> > rte_crypto_sym_cpu_session.
> >
> > Right now - just cipher_offset (see above).
> > What else in future (if any) - don't know.
> >
> > > I don't think there is any.
> > > I believe the same crypto session will be able to work synchronously as well.
> >
> > Exactly the same - problematically, see above.
> >
> > > We would only need a new API to perform synchronous actions.
> > > That will reduce the duplication code significantly
> > > in the driver to support 2 different kind of APIs with similar code inside.
> > > Please correct me in case I am missing something.
> >
> > To add new API into crypto-dev would also require changes in the PMD,
> > it wouldn't come totally free and I believe would require roughly the same
> > amount of changes.
>
> It will be required only in the PMDs which support it and would be minimal.
> You would need a feature flag, support for that synchronous API. Session information will
> already be there in the session. The changes wrt cipher_offset need to be added
> but with some default value to identify override will be done or not.
>
> >
> > >
> > >
> > > > While rte_security is designed in a way that we can add new session types
> > and
> > > > related parameters without causing API/ABI breakage.
> > >
> > > Yes the intent is to add new sessions based on various protocols that can be
> > supported by the driver.
> >
> > Various protocols and different types of sessions (and devices they belong to).
> > Let say right now we have INLINE_CRYPTO, INLINE_PROTO, LOOKASIDE_PROTO,
> > etc.
> > Here we introduce new type of session.
>
> What is the new value add to the existing sessions. The changes that we are doing
> here is just to avoid an API/ABI breakage. The synchronous processing can happen on both
> crypto and security session. This would mean, only the processing API should be defined,
> rest all should be already there in the sessions.
> In All other cases, INLINE - eth device was not having any format to perform crypto op
> LOOKASIDE - PROTO - add protocol specific sessions which is not available in crypto.
>
> >
> > > It is not that we should find it as an alternative to cryptodev and using it just
> > because it will not cause
> > > ABI/API breakage.
> >
> > I am considering this new API as an alternative to existing ones, but as an
> > extension.
> > Existing crypto-op API has its own advantages (generic), and I think we should
> > keep it supported by all crypto-devs.
> > From other side rte_security is an extendable framework that suits the purpose:
> > allows easily (and yes without ABI breakage) introduce new API for special type
> > of crypto-dev (SW based).
> >
> >
>
> Adding a synchronous processing API is understandable and can be added in both
> Crypto as well as Security, but a new action type for it is not required.
> Now whether to support that, we have ABI/API breakage, that is a different issue.
> And we may have to deal with it if no other option is there.
>
> >
> >
> >
> > > IMO the code should be placed where its intent is.
> > >
> > > >
> > > > BTW, what is your concern with proposed approach (via rte_security)?
> > > > From my perspective it is a lightweight change and it is totally optional
> > > > for the crypto PMDs to support it or not.
> > > > Konstantin
> > > >
> > > > > >
> > > > > > AESNI-GCM and AESNI-MB PMDs are updated with this support. There is
> > a
> > > > small
> > > > > > performance test app under app/test/security_aesni_gcm(mb)_perftest
> > to
> > > > > > prove.
> > > > > >
> > > > > > For the new API
> > > > > > The packet is sent to the crypto device for symmetric crypto
> > > > > > processing. The device will encrypt or decrypt the buffer based on the
> > > > session
> > > > > > data specified and preprocessed in the security session. Different
> > > > > > than the inline or lookaside modes, when the function exits, the user will
> > > > > > expect the buffers are either processed successfully, or having the error
> > > > number
> > > > > > assigned to the appropriate index of the status array.
> > > > > >
> > > > > > Will update the program's guide in the v1 patch.
> > > > > >
> > > > > > Regards,
> > > > > > Fan
> > > > > >
> > > > > > > -----Original Message-----
> > > > > > > From: Akhil Goyal [mailto:akhil.goyal@nxp.com]
> > > > > > > Sent: Wednesday, September 4, 2019 11:33 AM
> > > > > > > To: Zhang, Roy Fan <roy.fan.zhang@intel.com>; dev@dpdk.org
> > > > > > > Cc: Ananyev, Konstantin <konstantin.ananyev@intel.com>; Doherty,
> > > > Declan
> > > > > > > <declan.doherty@intel.com>; De Lara Guarch, Pablo
> > > > > > > <pablo.de.lara.guarch@intel.com>
> > > > > > > Subject: RE: [RFC PATCH 1/9] security: introduce CPU Crypto action
> > type
> > > > and
> > > > > > > API
> > > > > > >
> > > > > > > Hi Fan,
> > > > > > >
> > > > > > > >
> > > > > > > > This patch introduce new
> > RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO
> > > > > > > action
> > > > > > > > type to security library. The type represents performing crypto
> > > > > > > > operation with CPU cycles. The patch also includes a new API to
> > > > > > > > process crypto operations in bulk and the function pointers for PMDs.
> > > > > > > >
> > > > > > > I am not able to get the flow of execution for this action type. Could
> > you
> > > > > > > please elaborate the flow in the documentation. If not in
> > documentation
> > > > > > > right now, then please elaborate the flow in cover letter.
> > > > > > > Also I see that there are new APIs for processing crypto operations in
> > bulk.
> > > > > > > What does that mean. How are they different from the existing APIs
> > which
> > > > > > > are also handling bulk crypto ops depending on the budget.
> > > > > > >
> > > > > > >
> > > > > > > -Akhil
^ permalink raw reply [relevance 3%]
* [dpdk-dev] [PATCH v3 01/13] ethdev: change promiscuous mode controllers to return errors
@ 2019-09-14 11:37 3% ` Andrew Rybchenko
0 siblings, 0 replies; 200+ results
From: Andrew Rybchenko @ 2019-09-14 11:37 UTC (permalink / raw)
To: Neil Horman, John McNamara, Marko Kovacevic, Bernard Iremonger,
Ori Kam, Bruce Richardson, Radu Nicolau, Akhil Goyal,
Tomasz Kantecki, Harry van Haaren, Xiaoyun Li, Thomas Monjalon,
Ferruh Yigit
Cc: dev, Ivan Ilchenko
From: Ivan Ilchenko <Ivan.Ilchenko@oktetlabs.ru>
Change rte_eth_promiscuous_enable()/rte_eth_promiscuous_disable()
return value from void to int and return negative errno values
in case of error conditions.
Modify usage of these functions across the ethdev according
to new return type.
Signed-off-by: Ivan Ilchenko <Ivan.Ilchenko@oktetlabs.ru>
Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
---
doc/guides/rel_notes/deprecation.rst | 1 -
doc/guides/rel_notes/release_19_11.rst | 4 ++
doc/guides/sample_app_ug/flow_classify.rst | 6 ++-
doc/guides/sample_app_ug/flow_filtering.rst | 15 +++++-
doc/guides/sample_app_ug/rxtx_callbacks.rst | 5 +-
doc/guides/sample_app_ug/skeleton.rst | 6 ++-
lib/librte_ethdev/rte_ethdev.c | 52 ++++++++++++++++-----
lib/librte_ethdev/rte_ethdev.h | 14 +++++-
8 files changed, 80 insertions(+), 23 deletions(-)
diff --git a/doc/guides/rel_notes/deprecation.rst b/doc/guides/rel_notes/deprecation.rst
index cbb4c34ef..b2e0a1fc7 100644
--- a/doc/guides/rel_notes/deprecation.rst
+++ b/doc/guides/rel_notes/deprecation.rst
@@ -88,7 +88,6 @@ Deprecation Notices
negative errno values to indicate various error conditions (e.g.
invalid port ID, unsupported operation, failed operation):
- - ``rte_eth_promiscuous_enable`` and ``rte_eth_promiscuous_disable``
- ``rte_eth_allmulticast_enable`` and ``rte_eth_allmulticast_disable``
- ``rte_eth_link_get`` and ``rte_eth_link_get_nowait``
- ``rte_eth_dev_stop``
diff --git a/doc/guides/rel_notes/release_19_11.rst b/doc/guides/rel_notes/release_19_11.rst
index c8d97f16e..90c03cb8f 100644
--- a/doc/guides/rel_notes/release_19_11.rst
+++ b/doc/guides/rel_notes/release_19_11.rst
@@ -97,6 +97,10 @@ API Changes
* ethdev: changed ``rte_eth_dev_infos_get`` return value from ``void`` to
``int`` to provide a way to report various error conditions.
+* ethdev: changed ``rte_eth_promiscuous_enable`` and
+ ``rte_eth_promiscuous_disable`` return value from ``void`` to ``int`` to
+ provide a way to report various error conditions.
+
ABI Changes
-----------
diff --git a/doc/guides/sample_app_ug/flow_classify.rst b/doc/guides/sample_app_ug/flow_classify.rst
index 96a5c66d0..7c2b6dcf8 100644
--- a/doc/guides/sample_app_ug/flow_classify.rst
+++ b/doc/guides/sample_app_ug/flow_classify.rst
@@ -315,7 +315,9 @@ Forwarding application is shown below:
addr.addr_bytes[4], addr.addr_bytes[5]);
/* Enable RX in promiscuous mode for the Ethernet device. */
- rte_eth_promiscuous_enable(port);
+ retval = rte_eth_promiscuous_enable(port);
+ if (retval != 0)
+ return retval;
return 0;
}
@@ -343,7 +345,7 @@ Finally the RX port is set in promiscuous mode:
.. code-block:: c
- rte_eth_promiscuous_enable(port);
+ retval = rte_eth_promiscuous_enable(port);
The Add Rules function
~~~~~~~~~~~~~~~~~~~~~~
diff --git a/doc/guides/sample_app_ug/flow_filtering.rst b/doc/guides/sample_app_ug/flow_filtering.rst
index 02fc67550..de3e4ab0b 100644
--- a/doc/guides/sample_app_ug/flow_filtering.rst
+++ b/doc/guides/sample_app_ug/flow_filtering.rst
@@ -193,7 +193,13 @@ application is shown below:
}
}
- rte_eth_promiscuous_enable(port_id);
+ ret = rte_eth_promiscuous_enable(port_id);
+ if (ret != 0) {
+ rte_exit(EXIT_FAILURE,
+ ":: cannot enable promiscuous mode: err=%d, port=%u\n",
+ ret, port_id);
+ }
+
ret = rte_eth_dev_start(port_id);
if (ret < 0) {
rte_exit(EXIT_FAILURE,
@@ -278,7 +284,12 @@ We are setting the RX port to promiscuous mode:
.. code-block:: c
- rte_eth_promiscuous_enable(port_id);
+ ret = rte_eth_promiscuous_enable(port_id);
+ if (ret != 0) {
+ rte_exit(EXIT_FAILURE,
+ ":: cannot enable promiscuous mode: err=%d, port=%u\n",
+ ret, port_id);
+ }
The last step is to start the port.
diff --git a/doc/guides/sample_app_ug/rxtx_callbacks.rst b/doc/guides/sample_app_ug/rxtx_callbacks.rst
index 32c120992..0a69ec71a 100644
--- a/doc/guides/sample_app_ug/rxtx_callbacks.rst
+++ b/doc/guides/sample_app_ug/rxtx_callbacks.rst
@@ -117,8 +117,9 @@ comments:
return retval;
/* Enable RX in promiscuous mode for the Ethernet device. */
- rte_eth_promiscuous_enable(port);
-
+ retval = rte_eth_promiscuous_enable(port);
+ if (retval != 0)
+ return retval;
/* Add the callbacks for RX and TX.*/
rte_eth_add_rx_callback(port, 0, add_timestamps, NULL);
diff --git a/doc/guides/sample_app_ug/skeleton.rst b/doc/guides/sample_app_ug/skeleton.rst
index 59ca511d3..1d0a2760d 100644
--- a/doc/guides/sample_app_ug/skeleton.rst
+++ b/doc/guides/sample_app_ug/skeleton.rst
@@ -149,7 +149,9 @@ Forwarding application is shown below:
return retval;
/* Enable RX in promiscuous mode for the Ethernet device. */
- rte_eth_promiscuous_enable(port);
+ retval = rte_eth_promiscuous_enable(port);
+ if (retval != 0)
+ return retval;
return 0;
}
@@ -177,7 +179,7 @@ Finally the RX port is set in promiscuous mode:
.. code-block:: c
- rte_eth_promiscuous_enable(port);
+ retval = rte_eth_promiscuous_enable(port);
The Lcores Main
diff --git a/lib/librte_ethdev/rte_ethdev.c b/lib/librte_ethdev/rte_ethdev.c
index 30b0c7803..b97dd8aa8 100644
--- a/lib/librte_ethdev/rte_ethdev.c
+++ b/lib/librte_ethdev/rte_ethdev.c
@@ -1381,24 +1381,41 @@ rte_eth_dev_mac_restore(struct rte_eth_dev *dev,
}
}
-static void
+static int
rte_eth_dev_config_restore(struct rte_eth_dev *dev,
struct rte_eth_dev_info *dev_info, uint16_t port_id)
{
+ int ret;
+
if (!(*dev_info->dev_flags & RTE_ETH_DEV_NOLIVE_MAC_ADDR))
rte_eth_dev_mac_restore(dev, dev_info);
/* replay promiscuous configuration */
- if (rte_eth_promiscuous_get(port_id) == 1)
- rte_eth_promiscuous_enable(port_id);
- else if (rte_eth_promiscuous_get(port_id) == 0)
- rte_eth_promiscuous_disable(port_id);
+ if (rte_eth_promiscuous_get(port_id) == 1) {
+ ret = rte_eth_promiscuous_enable(port_id);
+ if (ret != 0 && ret != -ENOTSUP) {
+ RTE_ETHDEV_LOG(ERR,
+ "Failed to enable promiscuous mode for device (port %u): %s\n",
+ port_id, rte_strerror(-ret));
+ return ret;
+ }
+ } else if (rte_eth_promiscuous_get(port_id) == 0) {
+ ret = rte_eth_promiscuous_disable(port_id);
+ if (ret != 0 && ret != -ENOTSUP) {
+ RTE_ETHDEV_LOG(ERR,
+ "Failed to disable promiscuous mode for device (port %u): %s\n",
+ port_id, rte_strerror(-ret));
+ return ret;
+ }
+ }
/* replay all multicast configuration */
if (rte_eth_allmulticast_get(port_id) == 1)
rte_eth_allmulticast_enable(port_id);
else if (rte_eth_allmulticast_get(port_id) == 0)
rte_eth_allmulticast_disable(port_id);
+
+ return 0;
}
int
@@ -1436,7 +1453,14 @@ rte_eth_dev_start(uint16_t port_id)
else
return eth_err(port_id, diag);
- rte_eth_dev_config_restore(dev, &dev_info, port_id);
+ ret = rte_eth_dev_config_restore(dev, &dev_info, port_id);
+ if (ret != 0) {
+ RTE_ETHDEV_LOG(ERR,
+ "Error during restoring configuration for device (port %u): %s\n",
+ port_id, rte_strerror(-ret));
+ rte_eth_dev_stop(port_id);
+ return ret;
+ }
if (dev->data->dev_conf.intr_conf.lsc == 0) {
RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->link_update, -ENOTSUP);
@@ -1864,30 +1888,34 @@ rte_eth_tx_done_cleanup(uint16_t port_id, uint16_t queue_id, uint32_t free_cnt)
return eth_err(port_id, ret);
}
-void
+int
rte_eth_promiscuous_enable(uint16_t port_id)
{
struct rte_eth_dev *dev;
- RTE_ETH_VALID_PORTID_OR_RET(port_id);
+ RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
dev = &rte_eth_devices[port_id];
- RTE_FUNC_PTR_OR_RET(*dev->dev_ops->promiscuous_enable);
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->promiscuous_enable, -ENOTSUP);
(*dev->dev_ops->promiscuous_enable)(dev);
dev->data->promiscuous = 1;
+
+ return 0;
}
-void
+int
rte_eth_promiscuous_disable(uint16_t port_id)
{
struct rte_eth_dev *dev;
- RTE_ETH_VALID_PORTID_OR_RET(port_id);
+ RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
dev = &rte_eth_devices[port_id];
- RTE_FUNC_PTR_OR_RET(*dev->dev_ops->promiscuous_disable);
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->promiscuous_disable, -ENOTSUP);
dev->data->promiscuous = 0;
(*dev->dev_ops->promiscuous_disable)(dev);
+
+ return 0;
}
int
diff --git a/lib/librte_ethdev/rte_ethdev.h b/lib/librte_ethdev/rte_ethdev.h
index 475dbdae1..f07a829b2 100644
--- a/lib/librte_ethdev/rte_ethdev.h
+++ b/lib/librte_ethdev/rte_ethdev.h
@@ -2022,16 +2022,26 @@ int rte_eth_dev_reset(uint16_t port_id);
*
* @param port_id
* The port identifier of the Ethernet device.
+ * @return
+ * - (0) if successful.
+ * - (-ENOTSUP) if support for promiscuous_enable() does not exist
+ * for the device.
+ * - (-ENODEV) if *port_id* invalid.
*/
-void rte_eth_promiscuous_enable(uint16_t port_id);
+int rte_eth_promiscuous_enable(uint16_t port_id);
/**
* Disable receipt in promiscuous mode for an Ethernet device.
*
* @param port_id
* The port identifier of the Ethernet device.
+ * @return
+ * - (0) if successful.
+ * - (-ENOTSUP) if support for promiscuous_disable() does not exist
+ * for the device.
+ * - (-ENODEV) if *port_id* invalid.
*/
-void rte_eth_promiscuous_disable(uint16_t port_id);
+int rte_eth_promiscuous_disable(uint16_t port_id);
/**
* Return the value of promiscuous mode for an Ethernet device.
--
2.17.1
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] The type string in the malloc library is unused
[not found] ` <CAOaVG17_bhBOfJdvHZOzoVO3t7CXCZCa7f=_Q=_Pi5jgnY3GzA@mail.gmail.com>
@ 2019-09-14 8:29 3% ` Morten Brørup
0 siblings, 0 replies; 200+ results
From: Morten Brørup @ 2019-09-14 8:29 UTC (permalink / raw)
To: Stephen Hemminger, Anatoly Burakov; +Cc: Olivier Matz, Andrew Rybchenko, dev
I tend to agree with Stephen here, although it will break the API/ABI.
Another argument supporting its removal is the fact that it has remained non-implemented for many years, so a function for dumping information about rte_malloc'ed memory using the "type" string for debugging purposes has apparently not been in very high demand.
Med venlig hilsen / kind regards
- Morten Brørup
From: Stephen Hemminger [mailto:stephen@networkplumber.org]
Sent: Saturday, September 14, 2019 8:24 AM
To: Morten Brørup
Subject: Re: [dpdk-dev] The type string in the malloc library is unused
I would vote for removing it.
It is too late to implement it without breaking existing code.
On Fri, Sep 13, 2019, 3:32 PM Morten Brørup <mb@smartsharesystems.com> wrote:
Hi Anatoly,
The functions in the DPDK malloc library takes a "type" parameter (a string, supposedly for debug purposes), but the underlying malloc_heap functions (which take the same string parameter) don't store or use this string for anything.
Is the intention to implement this sometime in the future, or should it be considered for removal?
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] [PATCH v4 02/54] ethdev: change rte_eth_dev_info_get() return value to int
2019-09-12 16:42 3% ` [dpdk-dev] [PATCH v4 02/54] " Andrew Rybchenko
@ 2019-09-13 10:18 0% ` Iremonger, Bernard
0 siblings, 0 replies; 200+ results
From: Iremonger, Bernard @ 2019-09-13 10:18 UTC (permalink / raw)
To: Andrew Rybchenko, Neil Horman, Mcnamara, John, Kovacevic, Marko,
Thomas Monjalon, Yigit, Ferruh
Cc: dev, Ivan Ilchenko
Hi Ivan,
> -----Original Message-----
> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Andrew Rybchenko
> Sent: Thursday, September 12, 2019 5:42 PM
> To: Neil Horman <nhorman@tuxdriver.com>; Mcnamara, John
> <john.mcnamara@intel.com>; Kovacevic, Marko
> <marko.kovacevic@intel.com>; Thomas Monjalon <thomas@monjalon.net>;
> Yigit, Ferruh <ferruh.yigit@intel.com>
> Cc: dev@dpdk.org; Ivan Ilchenko <Ivan.Ilchenko@oktetlabs.com>
> Subject: [dpdk-dev] [PATCH v4 02/54] ethdev: change
> rte_eth_dev_info_get() return value to int
>
> From: Ivan Ilchenko <Ivan.Ilchenko@oktetlabs.com>
>
> Change rte_eth_dev_info_get() return value from void to int and return
> negative errno values in case of error conditions.
> Modify rte_eth_dev_info_get() usage across the ethdev according to new
> return type.
>
> Signed-off-by: Ivan Ilchenko <Ivan.Ilchenko@oktetlabs.com>
> Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
> Reviewed-by: Ferruh Yigit <ferruh.yigit@intel.com>
./check-git-log.sh -1
Wrong headline format:
ethdev: change rte_eth_dev_info_get() return value to int
> ---
> doc/guides/rel_notes/deprecation.rst | 1 -
> doc/guides/rel_notes/release_19_11.rst | 5 +-
> lib/librte_ethdev/rte_ethdev.c | 69 ++++++++++++++++++--------
> lib/librte_ethdev/rte_ethdev.h | 6 ++-
> 4 files changed, 57 insertions(+), 24 deletions(-)
>
> diff --git a/doc/guides/rel_notes/deprecation.rst
> b/doc/guides/rel_notes/deprecation.rst
> index 0ee8533b1..cbb4c34ef 100644
> --- a/doc/guides/rel_notes/deprecation.rst
> +++ b/doc/guides/rel_notes/deprecation.rst
> @@ -88,7 +88,6 @@ Deprecation Notices
> negative errno values to indicate various error conditions (e.g.
> invalid port ID, unsupported operation, failed operation):
>
> - - ``rte_eth_dev_info_get``
> - ``rte_eth_promiscuous_enable`` and ``rte_eth_promiscuous_disable``
> - ``rte_eth_allmulticast_enable`` and ``rte_eth_allmulticast_disable``
> - ``rte_eth_link_get`` and ``rte_eth_link_get_nowait`` diff --git
> a/doc/guides/rel_notes/release_19_11.rst
> b/doc/guides/rel_notes/release_19_11.rst
> index 66297d8f3..c8d97f16e 100644
> --- a/doc/guides/rel_notes/release_19_11.rst
> +++ b/doc/guides/rel_notes/release_19_11.rst
> @@ -94,6 +94,9 @@ API Changes
> Also, make sure to start the actual text at the margin.
> =========================================================
>
> +* ethdev: changed ``rte_eth_dev_infos_get`` return value from ``void``
> +to
> + ``int`` to provide a way to report various error conditions.
> +
>
> ABI Changes
> -----------
> @@ -145,7 +148,7 @@ The libraries prepended with a plus sign were
> incremented in this version.
> librte_distributor.so.1
> librte_eal.so.11
> librte_efd.so.1
> - librte_ethdev.so.12
> + + librte_ethdev.so.13
> librte_eventdev.so.7
> librte_flow_classify.so.1
> librte_gro.so.1
> diff --git a/lib/librte_ethdev/rte_ethdev.c b/lib/librte_ethdev/rte_ethdev.c
> index 17d183e1f..42b1d6e30 100644
> --- a/lib/librte_ethdev/rte_ethdev.c
> +++ b/lib/librte_ethdev/rte_ethdev.c
> @@ -1125,7 +1125,6 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t
> nb_rx_q, uint16_t nb_tx_q,
>
> dev = &rte_eth_devices[port_id];
>
> - RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -
> ENOTSUP);
> RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -
> ENOTSUP);
>
> if (dev->data->dev_started) {
> @@ -1144,7 +1143,9 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t
> nb_rx_q, uint16_t nb_tx_q,
> */
> memcpy(&dev->data->dev_conf, dev_conf, sizeof(dev->data-
> >dev_conf));
>
> - rte_eth_dev_info_get(port_id, &dev_info);
> + ret = rte_eth_dev_info_get(port_id, &dev_info);
> + if (ret != 0)
> + goto rollback;
>
> /* If number of queues specified by application for both Rx and Tx is
> * zero, use driver preferred values. This cannot be done individually
> @@ -1406,6 +1407,7 @@ rte_eth_dev_start(uint16_t port_id)
> struct rte_eth_dev *dev;
> struct rte_eth_dev_info dev_info;
> int diag;
> + int ret;
>
> RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
>
> @@ -1420,7 +1422,9 @@ rte_eth_dev_start(uint16_t port_id)
> return 0;
> }
>
> - rte_eth_dev_info_get(port_id, &dev_info);
> + ret = rte_eth_dev_info_get(port_id, &dev_info);
> + if (ret != 0)
> + return ret;
>
> /* Lets restore MAC now if device does not support live change */
> if (*dev_info.dev_flags & RTE_ETH_DEV_NOLIVE_MAC_ADDR) @@ -
> 1584,7 +1588,6 @@ rte_eth_rx_queue_setup(uint16_t port_id, uint16_t
> rx_queue_id,
> return -EINVAL;
> }
>
> - RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -
> ENOTSUP);
> RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_setup, -
> ENOTSUP);
>
> /*
> @@ -1592,7 +1595,10 @@ rte_eth_rx_queue_setup(uint16_t port_id,
> uint16_t rx_queue_id,
> * This value must be provided in the private data of the memory
> pool.
> * First check that the memory pool has a valid private data.
> */
> - rte_eth_dev_info_get(port_id, &dev_info);
> + ret = rte_eth_dev_info_get(port_id, &dev_info);
> + if (ret != 0)
> + return ret;
> +
> if (mp->private_data_size < sizeof(struct
> rte_pktmbuf_pool_private)) {
> RTE_ETHDEV_LOG(ERR, "%s private_data_size %d < %d\n",
> mp->name, (int)mp->private_data_size, @@ -1703,6
> +1709,7 @@ rte_eth_tx_queue_setup(uint16_t port_id, uint16_t
> tx_queue_id,
> struct rte_eth_dev_info dev_info;
> struct rte_eth_txconf local_conf;
> void **txq;
> + int ret;
>
> RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
>
> @@ -1712,10 +1719,11 @@ rte_eth_tx_queue_setup(uint16_t port_id,
> uint16_t tx_queue_id,
> return -EINVAL;
> }
>
> - RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -
> ENOTSUP);
> RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_setup, -
> ENOTSUP);
>
> - rte_eth_dev_info_get(port_id, &dev_info);
> + ret = rte_eth_dev_info_get(port_id, &dev_info);
> + if (ret != 0)
> + return ret;
>
> /* Use default specified by driver, if nb_tx_desc is zero */
> if (nb_tx_desc == 0) {
> @@ -2540,7 +2548,7 @@ rte_eth_dev_fw_version_get(uint16_t port_id,
> char *fw_version, size_t fw_size)
> fw_version,
> fw_size));
> }
>
> -void
> +int
> rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info
> *dev_info) {
> struct rte_eth_dev *dev;
> @@ -2558,7 +2566,7 @@ rte_eth_dev_info_get(uint16_t port_id, struct
> rte_eth_dev_info *dev_info)
> */
> memset(dev_info, 0, sizeof(struct rte_eth_dev_info));
>
> - RTE_ETH_VALID_PORTID_OR_RET(port_id);
> + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
> dev = &rte_eth_devices[port_id];
>
> dev_info->rx_desc_lim = lim;
> @@ -2567,13 +2575,15 @@ rte_eth_dev_info_get(uint16_t port_id, struct
> rte_eth_dev_info *dev_info)
> dev_info->min_mtu = RTE_ETHER_MIN_MTU;
> dev_info->max_mtu = UINT16_MAX;
>
> - RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_infos_get);
> + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -
> ENOTSUP);
> (*dev->dev_ops->dev_infos_get)(dev, dev_info);
> dev_info->driver_name = dev->device->driver->name;
> dev_info->nb_rx_queues = dev->data->nb_rx_queues;
> dev_info->nb_tx_queues = dev->data->nb_tx_queues;
>
> dev_info->dev_flags = &dev->data->dev_flags;
> +
> + return 0;
> }
>
> int
> @@ -2643,7 +2653,10 @@ rte_eth_dev_set_mtu(uint16_t port_id, uint16_t
> mtu)
> * which relies on dev->dev_ops->dev_infos_get.
> */
> if (*dev->dev_ops->dev_infos_get != NULL) {
> - rte_eth_dev_info_get(port_id, &dev_info);
> + ret = rte_eth_dev_info_get(port_id, &dev_info);
> + if (ret != 0)
> + return ret;
> +
> if (mtu < dev_info.min_mtu || mtu > dev_info.max_mtu)
> return -EINVAL;
> }
> @@ -2991,10 +3004,15 @@ rte_eth_dev_rss_hash_update(uint16_t port_id,
> {
> struct rte_eth_dev *dev;
> struct rte_eth_dev_info dev_info = { .flow_type_rss_offloads = 0, };
> + int ret;
>
> RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
> +
> + ret = rte_eth_dev_info_get(port_id, &dev_info);
> + if (ret != 0)
> + return ret;
> +
> dev = &rte_eth_devices[port_id];
> - rte_eth_dev_info_get(port_id, &dev_info);
> if ((dev_info.flow_type_rss_offloads | rss_conf->rss_hf) !=
> dev_info.flow_type_rss_offloads) {
> RTE_ETHDEV_LOG(ERR,
> @@ -3100,9 +3118,11 @@ get_mac_addr_index(uint16_t port_id, const
> struct rte_ether_addr *addr)
> struct rte_eth_dev_info dev_info;
> struct rte_eth_dev *dev = &rte_eth_devices[port_id];
> unsigned i;
> + int ret;
>
> - RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
> - rte_eth_dev_info_get(port_id, &dev_info);
> + ret = rte_eth_dev_info_get(port_id, &dev_info);
> + if (ret != 0)
> + return -1;
>
> for (i = 0; i < dev_info.max_mac_addrs; i++)
> if (memcmp(addr, &dev->data->mac_addrs[i], @@ -3233,8
> +3253,12 @@ get_hash_mac_addr_index(uint16_t port_id, const struct
> rte_ether_addr *addr)
> struct rte_eth_dev_info dev_info;
> struct rte_eth_dev *dev = &rte_eth_devices[port_id];
> unsigned i;
> + int ret;
> +
> + ret = rte_eth_dev_info_get(port_id, &dev_info);
> + if (ret != 0)
> + return -1;
>
> - rte_eth_dev_info_get(port_id, &dev_info);
> if (!dev->data->hash_mac_addrs)
> return -1;
>
> @@ -3319,11 +3343,15 @@ int rte_eth_set_queue_rate_limit(uint16_t
> port_id, uint16_t queue_idx,
> struct rte_eth_dev *dev;
> struct rte_eth_dev_info dev_info;
> struct rte_eth_link link;
> + int ret;
>
> RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
>
> + ret = rte_eth_dev_info_get(port_id, &dev_info);
> + if (ret != 0)
> + return ret;
> +
> dev = &rte_eth_devices[port_id];
> - rte_eth_dev_info_get(port_id, &dev_info);
> link = dev->data->dev_link;
>
> if (queue_idx > dev_info.max_tx_queues) { @@ -4363,15 +4391,14
> @@ rte_eth_dev_adjust_nb_rx_tx_desc(uint16_t port_id,
> uint16_t *nb_rx_desc,
> uint16_t *nb_tx_desc)
> {
> - struct rte_eth_dev *dev;
> struct rte_eth_dev_info dev_info;
> + int ret;
>
> RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
>
> - dev = &rte_eth_devices[port_id];
> - RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -
> ENOTSUP);
> -
> - rte_eth_dev_info_get(port_id, &dev_info);
> + ret = rte_eth_dev_info_get(port_id, &dev_info);
> + if (ret != 0)
> + return ret;
>
> if (nb_rx_desc != NULL)
> rte_eth_dev_adjust_nb_desc(nb_rx_desc,
> &dev_info.rx_desc_lim); diff --git a/lib/librte_ethdev/rte_ethdev.h
> b/lib/librte_ethdev/rte_ethdev.h index d9871782e..475dbdae1 100644
> --- a/lib/librte_ethdev/rte_ethdev.h
> +++ b/lib/librte_ethdev/rte_ethdev.h
> @@ -2366,8 +2366,12 @@ void rte_eth_macaddr_get(uint16_t port_id,
> struct rte_ether_addr *mac_addr);
> * @param dev_info
> * A pointer to a structure of type *rte_eth_dev_info* to be filled with
> * the contextual information of the Ethernet device.
> + * @return
> + * - (0) if successful.
> + * - (-ENOTSUP) if support for dev_infos_get() does not exist for the
> device.
> + * - (-ENODEV) if *port_id* invalid.
> */
> -void rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info
> *dev_info);
> +int rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info
> +*dev_info);
>
> /**
> * Retrieve the firmware version of a device.
> --
> 2.17.1
Regards,
Bernard.
^ permalink raw reply [relevance 0%]
* [dpdk-dev] DPDK techboard minutes of September 11
@ 2019-09-12 16:46 5% Thomas Monjalon
0 siblings, 0 replies; 200+ results
From: Thomas Monjalon @ 2019-09-12 16:46 UTC (permalink / raw)
To: dev; +Cc: techboard
Meeting notes for the DPDK technical board meeting held on 2019-05-08
Attendees: 9/9
- Bruce Richardson
- Ferruh Yigit
- Hemant Agrawal
- Jerin Jacob
- Konstantin Ananyev
- Maxime Coquelin
- Olivier Matz
- Stephen Hemminger
- Thomas Monjalon
1/ Review of last meeting minutes
Starting now the meeting minutes should be drafted
in the next 24 hours after the meeting.
It has been decided that meeting minutes are agreed
if no objection one week after the draft is shared.
2/ Examples removal
Bruce and Hemant will start removing few useless examples
during 19.11 release cycle.
3/ ABI compatibility
The new ABI policy has been discussed for months
and will be a topic for a panel discussion in Bordeaux event.
The final version and agreement of the guidelines are expected soon.
The first step should start after 19.11 release,
and will forbid any change which is not ABI-compatible with 19.11.
This first period must be used to prepare a longer freeze.
Then it will be allowed to break ABI compatibility in release 20.11,
and a longer compatibility period (2 years) could start from 20.11.
Note that experimental functions can change at any time.
4/ Build systems
Meson support is almost complete.
Bruce will present the status in Bordeaux event.
Makefile system will be deprecated in the release 19.11,
and there will be a blocking warning in the default configuration
of the release 20.02. The date for Makefile removal has not been decided.
The configurability of current Meson integration needs to be discussed.
5/ Next techboard meeting will be face to face
in Bordeaux, on September 19 Thursday, at 9pm.
^ permalink raw reply [relevance 5%]
* [dpdk-dev] [PATCH v4 02/54] ethdev: change rte_eth_dev_info_get() return value to int
@ 2019-09-12 16:42 3% ` Andrew Rybchenko
2019-09-13 10:18 0% ` Iremonger, Bernard
0 siblings, 1 reply; 200+ results
From: Andrew Rybchenko @ 2019-09-12 16:42 UTC (permalink / raw)
To: Neil Horman, John McNamara, Marko Kovacevic, Thomas Monjalon,
Ferruh Yigit
Cc: dev, Ivan Ilchenko
From: Ivan Ilchenko <Ivan.Ilchenko@oktetlabs.com>
Change rte_eth_dev_info_get() return value from void to int and return
negative errno values in case of error conditions.
Modify rte_eth_dev_info_get() usage across the ethdev according
to new return type.
Signed-off-by: Ivan Ilchenko <Ivan.Ilchenko@oktetlabs.com>
Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
Reviewed-by: Ferruh Yigit <ferruh.yigit@intel.com>
---
doc/guides/rel_notes/deprecation.rst | 1 -
doc/guides/rel_notes/release_19_11.rst | 5 +-
lib/librte_ethdev/rte_ethdev.c | 69 ++++++++++++++++++--------
lib/librte_ethdev/rte_ethdev.h | 6 ++-
4 files changed, 57 insertions(+), 24 deletions(-)
diff --git a/doc/guides/rel_notes/deprecation.rst b/doc/guides/rel_notes/deprecation.rst
index 0ee8533b1..cbb4c34ef 100644
--- a/doc/guides/rel_notes/deprecation.rst
+++ b/doc/guides/rel_notes/deprecation.rst
@@ -88,7 +88,6 @@ Deprecation Notices
negative errno values to indicate various error conditions (e.g.
invalid port ID, unsupported operation, failed operation):
- - ``rte_eth_dev_info_get``
- ``rte_eth_promiscuous_enable`` and ``rte_eth_promiscuous_disable``
- ``rte_eth_allmulticast_enable`` and ``rte_eth_allmulticast_disable``
- ``rte_eth_link_get`` and ``rte_eth_link_get_nowait``
diff --git a/doc/guides/rel_notes/release_19_11.rst b/doc/guides/rel_notes/release_19_11.rst
index 66297d8f3..c8d97f16e 100644
--- a/doc/guides/rel_notes/release_19_11.rst
+++ b/doc/guides/rel_notes/release_19_11.rst
@@ -94,6 +94,9 @@ API Changes
Also, make sure to start the actual text at the margin.
=========================================================
+* ethdev: changed ``rte_eth_dev_infos_get`` return value from ``void`` to
+ ``int`` to provide a way to report various error conditions.
+
ABI Changes
-----------
@@ -145,7 +148,7 @@ The libraries prepended with a plus sign were incremented in this version.
librte_distributor.so.1
librte_eal.so.11
librte_efd.so.1
- librte_ethdev.so.12
+ + librte_ethdev.so.13
librte_eventdev.so.7
librte_flow_classify.so.1
librte_gro.so.1
diff --git a/lib/librte_ethdev/rte_ethdev.c b/lib/librte_ethdev/rte_ethdev.c
index 17d183e1f..42b1d6e30 100644
--- a/lib/librte_ethdev/rte_ethdev.c
+++ b/lib/librte_ethdev/rte_ethdev.c
@@ -1125,7 +1125,6 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
dev = &rte_eth_devices[port_id];
- RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP);
if (dev->data->dev_started) {
@@ -1144,7 +1143,9 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
*/
memcpy(&dev->data->dev_conf, dev_conf, sizeof(dev->data->dev_conf));
- rte_eth_dev_info_get(port_id, &dev_info);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ goto rollback;
/* If number of queues specified by application for both Rx and Tx is
* zero, use driver preferred values. This cannot be done individually
@@ -1406,6 +1407,7 @@ rte_eth_dev_start(uint16_t port_id)
struct rte_eth_dev *dev;
struct rte_eth_dev_info dev_info;
int diag;
+ int ret;
RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
@@ -1420,7 +1422,9 @@ rte_eth_dev_start(uint16_t port_id)
return 0;
}
- rte_eth_dev_info_get(port_id, &dev_info);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
/* Lets restore MAC now if device does not support live change */
if (*dev_info.dev_flags & RTE_ETH_DEV_NOLIVE_MAC_ADDR)
@@ -1584,7 +1588,6 @@ rte_eth_rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
return -EINVAL;
}
- RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_setup, -ENOTSUP);
/*
@@ -1592,7 +1595,10 @@ rte_eth_rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
* This value must be provided in the private data of the memory pool.
* First check that the memory pool has a valid private data.
*/
- rte_eth_dev_info_get(port_id, &dev_info);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
+
if (mp->private_data_size < sizeof(struct rte_pktmbuf_pool_private)) {
RTE_ETHDEV_LOG(ERR, "%s private_data_size %d < %d\n",
mp->name, (int)mp->private_data_size,
@@ -1703,6 +1709,7 @@ rte_eth_tx_queue_setup(uint16_t port_id, uint16_t tx_queue_id,
struct rte_eth_dev_info dev_info;
struct rte_eth_txconf local_conf;
void **txq;
+ int ret;
RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
@@ -1712,10 +1719,11 @@ rte_eth_tx_queue_setup(uint16_t port_id, uint16_t tx_queue_id,
return -EINVAL;
}
- RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_setup, -ENOTSUP);
- rte_eth_dev_info_get(port_id, &dev_info);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
/* Use default specified by driver, if nb_tx_desc is zero */
if (nb_tx_desc == 0) {
@@ -2540,7 +2548,7 @@ rte_eth_dev_fw_version_get(uint16_t port_id, char *fw_version, size_t fw_size)
fw_version, fw_size));
}
-void
+int
rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info)
{
struct rte_eth_dev *dev;
@@ -2558,7 +2566,7 @@ rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info)
*/
memset(dev_info, 0, sizeof(struct rte_eth_dev_info));
- RTE_ETH_VALID_PORTID_OR_RET(port_id);
+ RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
dev = &rte_eth_devices[port_id];
dev_info->rx_desc_lim = lim;
@@ -2567,13 +2575,15 @@ rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info)
dev_info->min_mtu = RTE_ETHER_MIN_MTU;
dev_info->max_mtu = UINT16_MAX;
- RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_infos_get);
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
(*dev->dev_ops->dev_infos_get)(dev, dev_info);
dev_info->driver_name = dev->device->driver->name;
dev_info->nb_rx_queues = dev->data->nb_rx_queues;
dev_info->nb_tx_queues = dev->data->nb_tx_queues;
dev_info->dev_flags = &dev->data->dev_flags;
+
+ return 0;
}
int
@@ -2643,7 +2653,10 @@ rte_eth_dev_set_mtu(uint16_t port_id, uint16_t mtu)
* which relies on dev->dev_ops->dev_infos_get.
*/
if (*dev->dev_ops->dev_infos_get != NULL) {
- rte_eth_dev_info_get(port_id, &dev_info);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
+
if (mtu < dev_info.min_mtu || mtu > dev_info.max_mtu)
return -EINVAL;
}
@@ -2991,10 +3004,15 @@ rte_eth_dev_rss_hash_update(uint16_t port_id,
{
struct rte_eth_dev *dev;
struct rte_eth_dev_info dev_info = { .flow_type_rss_offloads = 0, };
+ int ret;
RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
+
dev = &rte_eth_devices[port_id];
- rte_eth_dev_info_get(port_id, &dev_info);
if ((dev_info.flow_type_rss_offloads | rss_conf->rss_hf) !=
dev_info.flow_type_rss_offloads) {
RTE_ETHDEV_LOG(ERR,
@@ -3100,9 +3118,11 @@ get_mac_addr_index(uint16_t port_id, const struct rte_ether_addr *addr)
struct rte_eth_dev_info dev_info;
struct rte_eth_dev *dev = &rte_eth_devices[port_id];
unsigned i;
+ int ret;
- RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
- rte_eth_dev_info_get(port_id, &dev_info);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return -1;
for (i = 0; i < dev_info.max_mac_addrs; i++)
if (memcmp(addr, &dev->data->mac_addrs[i],
@@ -3233,8 +3253,12 @@ get_hash_mac_addr_index(uint16_t port_id, const struct rte_ether_addr *addr)
struct rte_eth_dev_info dev_info;
struct rte_eth_dev *dev = &rte_eth_devices[port_id];
unsigned i;
+ int ret;
+
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return -1;
- rte_eth_dev_info_get(port_id, &dev_info);
if (!dev->data->hash_mac_addrs)
return -1;
@@ -3319,11 +3343,15 @@ int rte_eth_set_queue_rate_limit(uint16_t port_id, uint16_t queue_idx,
struct rte_eth_dev *dev;
struct rte_eth_dev_info dev_info;
struct rte_eth_link link;
+ int ret;
RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
+
dev = &rte_eth_devices[port_id];
- rte_eth_dev_info_get(port_id, &dev_info);
link = dev->data->dev_link;
if (queue_idx > dev_info.max_tx_queues) {
@@ -4363,15 +4391,14 @@ rte_eth_dev_adjust_nb_rx_tx_desc(uint16_t port_id,
uint16_t *nb_rx_desc,
uint16_t *nb_tx_desc)
{
- struct rte_eth_dev *dev;
struct rte_eth_dev_info dev_info;
+ int ret;
RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
- dev = &rte_eth_devices[port_id];
- RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
-
- rte_eth_dev_info_get(port_id, &dev_info);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
if (nb_rx_desc != NULL)
rte_eth_dev_adjust_nb_desc(nb_rx_desc, &dev_info.rx_desc_lim);
diff --git a/lib/librte_ethdev/rte_ethdev.h b/lib/librte_ethdev/rte_ethdev.h
index d9871782e..475dbdae1 100644
--- a/lib/librte_ethdev/rte_ethdev.h
+++ b/lib/librte_ethdev/rte_ethdev.h
@@ -2366,8 +2366,12 @@ void rte_eth_macaddr_get(uint16_t port_id, struct rte_ether_addr *mac_addr);
* @param dev_info
* A pointer to a structure of type *rte_eth_dev_info* to be filled with
* the contextual information of the Ethernet device.
+ * @return
+ * - (0) if successful.
+ * - (-ENOTSUP) if support for dev_infos_get() does not exist for the device.
+ * - (-ENODEV) if *port_id* invalid.
*/
-void rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info);
+int rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info);
/**
* Retrieve the firmware version of a device.
--
2.17.1
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] [RFC PATCH 1/9] security: introduce CPU Crypto action type and API
2019-09-11 12:29 4% ` Ananyev, Konstantin
@ 2019-09-12 14:12 5% ` Akhil Goyal
2019-09-16 14:53 3% ` Ananyev, Konstantin
0 siblings, 1 reply; 200+ results
From: Akhil Goyal @ 2019-09-12 14:12 UTC (permalink / raw)
To: Ananyev, Konstantin, dev, De Lara Guarch, Pablo, Thomas Monjalon
Cc: Zhang, Roy Fan, Doherty, Declan, Anoob Joseph
Hi Konstantin,
> Hi Akhil,
> > >
> > > > > This action type allows the burst of symmetric crypto workload using the
> > > same
> > > > > algorithm, key, and direction being processed by CPU cycles
> synchronously.
> > > > > This flexible action type does not require external hardware involvement,
> > > > > having the crypto workload processed synchronously, and is more
> > > performant
> > > > > than Cryptodev SW PMD due to the saved cycles on removed "async
> mode
> > > > > simulation" as well as 3 cacheline access of the crypto ops.
> > > >
> > > > Does that mean application will not call the cryptodev_enqueue_burst and
> > > corresponding dequeue burst.
> > >
> > > Yes, instead it just call rte_security_process_cpu_crypto_bulk(...)
> > >
> > > > It would be a new API something like process_packets and it will have the
> > > crypto processed packets while returning from the API?
> > >
> > > Yes, though the plan is that API will operate on raw data buffers, not mbufs.
> > >
> > > >
> > > > I still do not understand why we cannot do with the conventional crypto lib
> > > only.
> > > > As far as I can understand, you are not doing any protocol processing or
> any
> > > value add
> > > > To the crypto processing. IMO, you just need a synchronous crypto
> processing
> > > API which
> > > > Can be defined in cryptodev, you don't need to re-create a crypto session
> in
> > > the name of
> > > > Security session in the driver just to do a synchronous processing.
> > >
> > > I suppose your question is why not to have
> > > rte_crypot_process_cpu_crypto_bulk(...) instead?
> > > The main reason is that would require disruptive changes in existing
> cryptodev
> > > API
> > > (would cause ABI/API breakage).
> > > Session for RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO need some extra
> > > information
> > > that normal crypto_sym_xform doesn't contain
> > > (cipher offset from the start of the buffer, might be something extra in
> future).
> >
> > Cipher offset will be part of rte_crypto_op.
>
> fill/read (+ alloc/free) is one of the main things that slowdown current crypto-op
> approach.
> That's why the general idea - have all data that wouldn't change from packet to
> packet
> included into the session and setup it once at session_init().
I agree that you cannot use crypto-op.
You can have the new API in crypto.
As per the current patch, you only need cipher_offset which you can have it as a parameter until
You get it approved in the crypto xform. I believe it will be beneficial in case of other crypto cases as well.
We can have cipher offset at both places(crypto-op and cipher_xform). It will give flexibility to the user to
override it.
>
> > If you intend not to use rte_crypto_op
> > You can pass this as an argument in the new cryptodev API.
>
> You mean extra parameter in rte_security_process_cpu_crypto_bulk()?
> It can be in theory, but that solution looks a bit ugly:
> why to pass for each call something that would be constant per session?
> Again having that value constant per session might allow some extra
> optimisations
> That would be hard to achieve for dynamic case.
> and not extendable:
> Suppose tomorrow will need to add something extra (some new algorithm
> support or so).
> With what you proposing will need to new parameter to the function,
> which means API breakage.
>
> > Something extra will also cause ABI breakage in security as well.
> > So it will be same.
>
> I don't think it would.
> AFAIK, right now this patch doesn't introduce any API/ABI breakage.
> Iinside struct rte_security_session_conf we have a union of xforms
> depending on session type.
> So as long as cpu_crypto_xform wouldn't exceed sizes of other xform -
> I believe no ABI breakage will appear.
Agreed, it will not break ABI in case of security till we do not exceed current size.
Saving an ABI/API breakage is more important or placing the code at the correct place.
We need to find a tradeoff. Others can comment on this.
@Thomas Monjalon, @De Lara Guarch, Pablo Any comments?
>
>
> >
> > > Also right now there is no way to add new type of crypto_sym_session
> without
> > > either breaking existing crypto-dev ABI/API or introducing new structure
> > > (rte_crypto_sym_cpu_session or so) for that.
> >
> > What extra info is required in rte_cryptodev_sym_session to get the
> rte_crypto_sym_cpu_session.
>
> Right now - just cipher_offset (see above).
> What else in future (if any) - don't know.
>
> > I don't think there is any.
> > I believe the same crypto session will be able to work synchronously as well.
>
> Exactly the same - problematically, see above.
>
> > We would only need a new API to perform synchronous actions.
> > That will reduce the duplication code significantly
> > in the driver to support 2 different kind of APIs with similar code inside.
> > Please correct me in case I am missing something.
>
> To add new API into crypto-dev would also require changes in the PMD,
> it wouldn't come totally free and I believe would require roughly the same
> amount of changes.
It will be required only in the PMDs which support it and would be minimal.
You would need a feature flag, support for that synchronous API. Session information will
already be there in the session. The changes wrt cipher_offset need to be added
but with some default value to identify override will be done or not.
>
> >
> >
> > > While rte_security is designed in a way that we can add new session types
> and
> > > related parameters without causing API/ABI breakage.
> >
> > Yes the intent is to add new sessions based on various protocols that can be
> supported by the driver.
>
> Various protocols and different types of sessions (and devices they belong to).
> Let say right now we have INLINE_CRYPTO, INLINE_PROTO, LOOKASIDE_PROTO,
> etc.
> Here we introduce new type of session.
What is the new value add to the existing sessions. The changes that we are doing
here is just to avoid an API/ABI breakage. The synchronous processing can happen on both
crypto and security session. This would mean, only the processing API should be defined,
rest all should be already there in the sessions.
In All other cases, INLINE - eth device was not having any format to perform crypto op
LOOKASIDE - PROTO - add protocol specific sessions which is not available in crypto.
>
> > It is not that we should find it as an alternative to cryptodev and using it just
> because it will not cause
> > ABI/API breakage.
>
> I am considering this new API as an alternative to existing ones, but as an
> extension.
> Existing crypto-op API has its own advantages (generic), and I think we should
> keep it supported by all crypto-devs.
> From other side rte_security is an extendable framework that suits the purpose:
> allows easily (and yes without ABI breakage) introduce new API for special type
> of crypto-dev (SW based).
>
>
Adding a synchronous processing API is understandable and can be added in both
Crypto as well as Security, but a new action type for it is not required.
Now whether to support that, we have ABI/API breakage, that is a different issue.
And we may have to deal with it if no other option is there.
>
>
>
> > IMO the code should be placed where its intent is.
> >
> > >
> > > BTW, what is your concern with proposed approach (via rte_security)?
> > > From my perspective it is a lightweight change and it is totally optional
> > > for the crypto PMDs to support it or not.
> > > Konstantin
> > >
> > > > >
> > > > > AESNI-GCM and AESNI-MB PMDs are updated with this support. There is
> a
> > > small
> > > > > performance test app under app/test/security_aesni_gcm(mb)_perftest
> to
> > > > > prove.
> > > > >
> > > > > For the new API
> > > > > The packet is sent to the crypto device for symmetric crypto
> > > > > processing. The device will encrypt or decrypt the buffer based on the
> > > session
> > > > > data specified and preprocessed in the security session. Different
> > > > > than the inline or lookaside modes, when the function exits, the user will
> > > > > expect the buffers are either processed successfully, or having the error
> > > number
> > > > > assigned to the appropriate index of the status array.
> > > > >
> > > > > Will update the program's guide in the v1 patch.
> > > > >
> > > > > Regards,
> > > > > Fan
> > > > >
> > > > > > -----Original Message-----
> > > > > > From: Akhil Goyal [mailto:akhil.goyal@nxp.com]
> > > > > > Sent: Wednesday, September 4, 2019 11:33 AM
> > > > > > To: Zhang, Roy Fan <roy.fan.zhang@intel.com>; dev@dpdk.org
> > > > > > Cc: Ananyev, Konstantin <konstantin.ananyev@intel.com>; Doherty,
> > > Declan
> > > > > > <declan.doherty@intel.com>; De Lara Guarch, Pablo
> > > > > > <pablo.de.lara.guarch@intel.com>
> > > > > > Subject: RE: [RFC PATCH 1/9] security: introduce CPU Crypto action
> type
> > > and
> > > > > > API
> > > > > >
> > > > > > Hi Fan,
> > > > > >
> > > > > > >
> > > > > > > This patch introduce new
> RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO
> > > > > > action
> > > > > > > type to security library. The type represents performing crypto
> > > > > > > operation with CPU cycles. The patch also includes a new API to
> > > > > > > process crypto operations in bulk and the function pointers for PMDs.
> > > > > > >
> > > > > > I am not able to get the flow of execution for this action type. Could
> you
> > > > > > please elaborate the flow in the documentation. If not in
> documentation
> > > > > > right now, then please elaborate the flow in cover letter.
> > > > > > Also I see that there are new APIs for processing crypto operations in
> bulk.
> > > > > > What does that mean. How are they different from the existing APIs
> which
> > > > > > are also handling bulk crypto ops depending on the budget.
> > > > > >
> > > > > >
> > > > > > -Akhil
^ permalink raw reply [relevance 5%]
* Re: [dpdk-dev] [PATCH v5 00/12] lib: add RIB and FIB liraries
2019-09-12 7:37 0% ` Morten Brørup
@ 2019-09-12 9:47 0% ` Medvedkin, Vladimir
0 siblings, 0 replies; 200+ results
From: Medvedkin, Vladimir @ 2019-09-12 9:47 UTC (permalink / raw)
To: Morten Brørup; +Cc: bruce.richardson, dev
Hi Brørup,
On 12/09/2019 08:37, Morten Brørup wrote:
>> -----Original Message-----
>> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Vladimir Medvedkin
>>
>> This is heavily reworked version of previous RIB library series:
>> https://mails.dpdk.org/archives/dev/2018-April/099492.html
>>
>> Current lpm implementation while provides really good lookup
>> performance has number of problems.
>> One of them is very low speed for control plane operations
>> such as add or delete a route.
>> Another disadvantage is fixed number of bits for userdata
>> (24 for v4 and 21 for v6)
>> Also it is hard to introduce changes in existing LPM code or add new
>> algorithms without breaking ABI.
>>
>> This patch series tries to solve this problems by:
>> Introduce two new libraries - RIB and FIB.
>> RIB that is Routing Information Base.
>> It implements a control plane struct containing routes in a tree and
>> provides fast add/del operations for routes. Also it allows to perform
>> fast subtree traversals (i.e. retrieve existing subroutes for a given
>> prefix). This structure will be used as a control plane helper
>> structure
>> for FIB implementation.
>> Also it might be used standalone in other different places such as
>> bitmaps for example.
>>
> Great!
>
>
>> Second library is FIB that is Forwarding Information Base. It
>> represents
>> dataplane related struct and algorithms for longest prefix match.
>> Internally it consists of two parts - RIB (control plane ops) and
>> implementation for the dataplane tasks.
>> Initial version provides two implementations for both ipv4 and ipv6:
>> dummy (uses RIB as a dataplane) and DIR24_8 (same as current LPM)
>> Due to proposed design it allows to extend FIB with new algorithms in
>> future
>> (for example DXR, poptrie, etc).
> The feedback following here is meant as a comment, not an objection. Feel free to ignore!
>
> This FIB library is designed for IP based forwarding only.
>
> How about forwarding based on other criteria?
> E.g. the FIB in a standard Ethernet switch is based on VLAN+MAC.
>
> Such a FIB would probably require a different library, based on a hash structure, and would also require a compare-and-set function callable from the data plane in order to provide wire speed learning.
>
> So I suggest that the documentation highlights that this FIB library is for IP based forwarding. Optionally also reconsider the name of the library and its functions, structures etc..
Thanks for the feedback.
Yes, at the moment FIB has only longest prefix match algorithms.
However, it is possible to add different exact match algorithms for
VLAN+MAC/MPLS/etc processing.
It is always hard to find proper name for library/function/variable, so
if you think that fib name is not relevant feel free to suggest better :)
>
>> From our measurements we saw 10x speedup for control plane operations
>> comparing with current LPM library (depending on prefix length
>> distribution)
>>
>> ToDo:
>> - introduce new performance measurement app.
>> - add documentation.
>> - add support into existing examples (l3fwd)
>>
>
> Med venlig hilsen / kind regards
> - Morten Brørup
--
Regards,
Vladimir
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [PATCH v5 00/12] lib: add RIB and FIB liraries
2019-09-11 17:09 2% ` [dpdk-dev] [PATCH v5 00/12] lib: add RIB and FIB liraries Vladimir Medvedkin
@ 2019-09-12 7:37 0% ` Morten Brørup
2019-09-12 9:47 0% ` Medvedkin, Vladimir
0 siblings, 1 reply; 200+ results
From: Morten Brørup @ 2019-09-12 7:37 UTC (permalink / raw)
To: Vladimir Medvedkin; +Cc: bruce.richardson, dev
> -----Original Message-----
> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Vladimir Medvedkin
>
> This is heavily reworked version of previous RIB library series:
> https://mails.dpdk.org/archives/dev/2018-April/099492.html
>
> Current lpm implementation while provides really good lookup
> performance has number of problems.
> One of them is very low speed for control plane operations
> such as add or delete a route.
> Another disadvantage is fixed number of bits for userdata
> (24 for v4 and 21 for v6)
> Also it is hard to introduce changes in existing LPM code or add new
> algorithms without breaking ABI.
>
> This patch series tries to solve this problems by:
> Introduce two new libraries - RIB and FIB.
> RIB that is Routing Information Base.
> It implements a control plane struct containing routes in a tree and
> provides fast add/del operations for routes. Also it allows to perform
> fast subtree traversals (i.e. retrieve existing subroutes for a given
> prefix). This structure will be used as a control plane helper
> structure
> for FIB implementation.
> Also it might be used standalone in other different places such as
> bitmaps for example.
>
Great!
> Second library is FIB that is Forwarding Information Base. It
> represents
> dataplane related struct and algorithms for longest prefix match.
> Internally it consists of two parts - RIB (control plane ops) and
> implementation for the dataplane tasks.
> Initial version provides two implementations for both ipv4 and ipv6:
> dummy (uses RIB as a dataplane) and DIR24_8 (same as current LPM)
> Due to proposed design it allows to extend FIB with new algorithms in
> future
> (for example DXR, poptrie, etc).
The feedback following here is meant as a comment, not an objection. Feel free to ignore!
This FIB library is designed for IP based forwarding only.
How about forwarding based on other criteria?
E.g. the FIB in a standard Ethernet switch is based on VLAN+MAC.
Such a FIB would probably require a different library, based on a hash structure, and would also require a compare-and-set function callable from the data plane in order to provide wire speed learning.
So I suggest that the documentation highlights that this FIB library is for IP based forwarding. Optionally also reconsider the name of the library and its functions, structures etc..
>
> From our measurements we saw 10x speedup for control plane operations
> comparing with current LPM library (depending on prefix length
> distribution)
>
> ToDo:
> - introduce new performance measurement app.
> - add documentation.
> - add support into existing examples (l3fwd)
>
Med venlig hilsen / kind regards
- Morten Brørup
^ permalink raw reply [relevance 0%]
* [dpdk-dev] [PATCH v5 00/12] lib: add RIB and FIB liraries
@ 2019-09-11 17:09 2% ` Vladimir Medvedkin
2019-09-12 7:37 0% ` Morten Brørup
0 siblings, 1 reply; 200+ results
From: Vladimir Medvedkin @ 2019-09-11 17:09 UTC (permalink / raw)
To: dev; +Cc: bruce.richardson
This is heavily reworked version of previous RIB library series:
https://mails.dpdk.org/archives/dev/2018-April/099492.html
Current lpm implementation while provides really good lookup
performance has number of problems.
One of them is very low speed for control plane operations
such as add or delete a route.
Another disadvantage is fixed number of bits for userdata
(24 for v4 and 21 for v6)
Also it is hard to introduce changes in existing LPM code or add new
algorithms without breaking ABI.
This patch series tries to solve this problems by:
Introduce two new libraries - RIB and FIB.
RIB that is Routing Information Base.
It implements a control plane struct containing routes in a tree and
provides fast add/del operations for routes. Also it allows to perform
fast subtree traversals (i.e. retrieve existing subroutes for a given
prefix). This structure will be used as a control plane helper structure
for FIB implementation.
Also it might be used standalone in other different places such as
bitmaps for example.
Second library is FIB that is Forwarding Information Base. It represents
dataplane related struct and algorithms for longest prefix match.
Internally it consists of two parts - RIB (control plane ops) and
implementation for the dataplane tasks.
Initial version provides two implementations for both ipv4 and ipv6:
dummy (uses RIB as a dataplane) and DIR24_8 (same as current LPM)
Due to proposed design it allows to extend FIB with new algorithms in future
(for example DXR, poptrie, etc).
From our measurements we saw 10x speedup for control plane operations
comparing with current LPM library (depending on prefix length distribution)
ToDo:
- introduce new performance measurement app.
- add documentation.
- add support into existing examples (l3fwd)
Vladimir Medvedkin (12):
rib: add RIB library
test/rib: add RIB library autotests
rib: add ipv6 support for RIB
test/rib: add ipv6 support for RIB autotests
fib: add FIB library
fib: add FIB ipv6 support
fib: add DIR24-8 dataplane algorithm
fib: add dataplane algorithm for ipv6
test/fib: add FIB library autotests
test/fib: add ipv6 support for FIB autotests
test/fib: add FIB library performance autotests
test/fib: add FIB library ipv6 performance autotests
app/test/Makefile | 7 +
app/test/autotest_data.py | 36 ++
app/test/meson.build | 14 +
app/test/test_fib.c | 397 +++++++++++++++++++
app/test/test_fib6.c | 405 ++++++++++++++++++++
app/test/test_fib6_perf.c | 157 ++++++++
app/test/test_fib_perf.c | 411 ++++++++++++++++++++
app/test/test_rib.c | 351 +++++++++++++++++
app/test/test_rib6.c | 357 +++++++++++++++++
config/common_base | 11 +
doc/api/doxy-api.conf.in | 2 +
lib/Makefile | 4 +
lib/librte_fib/Makefile | 25 ++
lib/librte_fib/dir24_8.c | 737 +++++++++++++++++++++++++++++++++++
lib/librte_fib/dir24_8.h | 36 ++
lib/librte_fib/meson.build | 8 +
lib/librte_fib/rte_fib.c | 319 ++++++++++++++++
lib/librte_fib/rte_fib.h | 188 +++++++++
lib/librte_fib/rte_fib6.c | 322 ++++++++++++++++
lib/librte_fib/rte_fib6.h | 193 ++++++++++
lib/librte_fib/rte_fib_version.map | 23 ++
lib/librte_fib/trie.c | 760 +++++++++++++++++++++++++++++++++++++
lib/librte_fib/trie.h | 37 ++
lib/librte_rib/Makefile | 25 ++
lib/librte_rib/meson.build | 8 +
lib/librte_rib/rte_rib.c | 532 ++++++++++++++++++++++++++
lib/librte_rib/rte_rib.h | 277 ++++++++++++++
lib/librte_rib/rte_rib6.c | 598 +++++++++++++++++++++++++++++
lib/librte_rib/rte_rib6.h | 334 ++++++++++++++++
lib/librte_rib/rte_rib_version.map | 35 ++
lib/meson.build | 4 +-
mk/rte.app.mk | 2 +
32 files changed, 6614 insertions(+), 1 deletion(-)
create mode 100644 app/test/test_fib.c
create mode 100644 app/test/test_fib6.c
create mode 100644 app/test/test_fib6_perf.c
create mode 100644 app/test/test_fib_perf.c
create mode 100644 app/test/test_rib.c
create mode 100644 app/test/test_rib6.c
create mode 100644 lib/librte_fib/Makefile
create mode 100644 lib/librte_fib/dir24_8.c
create mode 100644 lib/librte_fib/dir24_8.h
create mode 100644 lib/librte_fib/meson.build
create mode 100644 lib/librte_fib/rte_fib.c
create mode 100644 lib/librte_fib/rte_fib.h
create mode 100644 lib/librte_fib/rte_fib6.c
create mode 100644 lib/librte_fib/rte_fib6.h
create mode 100644 lib/librte_fib/rte_fib_version.map
create mode 100644 lib/librte_fib/trie.c
create mode 100644 lib/librte_fib/trie.h
create mode 100644 lib/librte_rib/Makefile
create mode 100644 lib/librte_rib/meson.build
create mode 100644 lib/librte_rib/rte_rib.c
create mode 100644 lib/librte_rib/rte_rib.h
create mode 100644 lib/librte_rib/rte_rib6.c
create mode 100644 lib/librte_rib/rte_rib6.h
create mode 100644 lib/librte_rib/rte_rib_version.map
--
2.7.4
^ permalink raw reply [relevance 2%]
* [dpdk-dev] DPDK techboard minutes (2019-07-17)
@ 2019-09-11 15:05 3% Olivier Matz
0 siblings, 0 replies; 200+ results
From: Olivier Matz @ 2019-09-11 15:05 UTC (permalink / raw)
To: dev
Hi,
Here are the meeting notes for the DPDK technical board meeting held on
2019-07-17.
Attendees:
- Bruce Richardson
- Ferruh Yigit
- Hemant Agrawal
- Jerin Jacob
- Olivier Matz
- Thomas Monjalon
1) New next-net tree co-maintainer
==================================
Andrew Rybchenko <arybchenko@solarflare.com> is accepted as new
co-maintainer for next-net sub-tree.
2) DPDK API/ABI Stability
=========================
Good progresses and discussions.
Some points should be clarified before the userspace summit:
- Splitting DPDK into core, non-core and experimental libraries
- OS Packaging of DPDK
- UNH Testing of DPDK
3) Some examples are suggested to be removed in 19.11
=====================================================
Bruce to kick off the discussion on the ML.
(done: https://mails.dpdk.org/archives/dev/2019-July/138676.html )
4) Security process update
==========================
Request has been done to Trishan at last governing board that LF becomes
a CNA (CVE Numbering Authority).
Stephen (techboard representative at govboard) to pass the request via
email.
5) Proposal for regexdev subsystem
===================================
RFC proposal from Jerin Jacob <jerinj@marvell.com>:
https://patchwork.dpdk.org/patch/55505/
Several vendors seems interested by this feature. Concerned people
are encouraged to reply to this thread.
6) SPDX licenses
================
Hemant + Bruce to check if we can achieve SPDX compliance for 19.11.
7) Next meeting
===============
On 2019-07-31, Maxime will chair it.
^ permalink raw reply [relevance 3%]
* [dpdk-dev] DPDK techboard minutes of July 31
@ 2019-09-11 15:04 5% Maxime Coquelin
0 siblings, 0 replies; 200+ results
From: Maxime Coquelin @ 2019-09-11 15:04 UTC (permalink / raw)
To: DPDK Techboard, dev
Meeting notes for the DPDK technical board meeting held on 2018-07-31
Attendees:
- Bruce Richardson
- Ferruh Yigit
- Hemant Agrawal
- Jerin Jacob
- Maxime Coquelin
- Stephen Hemminger
- Thomas Monjalon
1) Security process updates
- The governing board acknowledges the security process
- Last governing board meeting didn't covered the technical board
request for LF to become a CNA
- Stephen to do the request at the next governing board meeting
2) SPDX licences
- Still some subsystems not compliant with SPDK, target is still v19.11
- Hemant sent a reminder on DPDK dev mailing-list
- Hemant will give a status update at next meeting
3) DPDK community survey follow-up
- 39% of the respondents said they do not get enough reviews
* Situation has improved since we have more sub-trees
* Could further be improved if maintainers asked trusted contributors
to help reviewing patches
* In case the contributor feels the review process does not work, he
should e-mail the techboard
* Thomas and Honnappa to update the contribution documentation and
send a pointer to that doc on the dev mailing-list
4) DPDK API/ABI Stability discussions
- Ray prepared a v2 of the policy patch, main changes:
* Change to 1 year initial ABI stability period, with a review
afterward to elongate the stability period
* Changes to the handling of experimental and depreciation process
- Open question is from where to start the major ABI version numbering.
* Should it be the year of the declaration of stable ABI version?
* Or start at v3.0 and continue from there?
* Bruce mentioned that we already have libraries at version > 10, so
using 3 would not work
* Techboard voted to use the year of stabilization as starting point.
5) Next meeting
- On 2019-08-14, Stephen will chair it.
^ permalink raw reply [relevance 5%]
* Re: [dpdk-dev] [RFC PATCH 1/9] security: introduce CPU Crypto action type and API
2019-09-10 10:44 4% ` Akhil Goyal
@ 2019-09-11 12:29 4% ` Ananyev, Konstantin
2019-09-12 14:12 5% ` Akhil Goyal
0 siblings, 1 reply; 200+ results
From: Ananyev, Konstantin @ 2019-09-11 12:29 UTC (permalink / raw)
To: Akhil Goyal, dev; +Cc: Zhang, Roy Fan, Doherty, Declan, De Lara Guarch, Pablo
Hi Akhil,
> >
> > > > This action type allows the burst of symmetric crypto workload using the
> > same
> > > > algorithm, key, and direction being processed by CPU cycles synchronously.
> > > > This flexible action type does not require external hardware involvement,
> > > > having the crypto workload processed synchronously, and is more
> > performant
> > > > than Cryptodev SW PMD due to the saved cycles on removed "async mode
> > > > simulation" as well as 3 cacheline access of the crypto ops.
> > >
> > > Does that mean application will not call the cryptodev_enqueue_burst and
> > corresponding dequeue burst.
> >
> > Yes, instead it just call rte_security_process_cpu_crypto_bulk(...)
> >
> > > It would be a new API something like process_packets and it will have the
> > crypto processed packets while returning from the API?
> >
> > Yes, though the plan is that API will operate on raw data buffers, not mbufs.
> >
> > >
> > > I still do not understand why we cannot do with the conventional crypto lib
> > only.
> > > As far as I can understand, you are not doing any protocol processing or any
> > value add
> > > To the crypto processing. IMO, you just need a synchronous crypto processing
> > API which
> > > Can be defined in cryptodev, you don't need to re-create a crypto session in
> > the name of
> > > Security session in the driver just to do a synchronous processing.
> >
> > I suppose your question is why not to have
> > rte_crypot_process_cpu_crypto_bulk(...) instead?
> > The main reason is that would require disruptive changes in existing cryptodev
> > API
> > (would cause ABI/API breakage).
> > Session for RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO need some extra
> > information
> > that normal crypto_sym_xform doesn't contain
> > (cipher offset from the start of the buffer, might be something extra in future).
>
> Cipher offset will be part of rte_crypto_op.
fill/read (+ alloc/free) is one of the main things that slowdown current crypto-op approach.
That's why the general idea - have all data that wouldn't change from packet to packet
included into the session and setup it once at session_init().
> If you intend not to use rte_crypto_op
> You can pass this as an argument in the new cryptodev API.
You mean extra parameter in rte_security_process_cpu_crypto_bulk()?
It can be in theory, but that solution looks a bit ugly:
why to pass for each call something that would be constant per session?
Again having that value constant per session might allow some extra optimisations
That would be hard to achieve for dynamic case.
and not extendable:
Suppose tomorrow will need to add something extra (some new algorithm support or so).
With what you proposing will need to new parameter to the function,
which means API breakage.
> Something extra will also cause ABI breakage in security as well.
> So it will be same.
I don't think it would.
AFAIK, right now this patch doesn't introduce any API/ABI breakage.
Iinside struct rte_security_session_conf we have a union of xforms
depending on session type.
So as long as cpu_crypto_xform wouldn't exceed sizes of other xform -
I believe no ABI breakage will appear.
>
> > Also right now there is no way to add new type of crypto_sym_session without
> > either breaking existing crypto-dev ABI/API or introducing new structure
> > (rte_crypto_sym_cpu_session or so) for that.
>
> What extra info is required in rte_cryptodev_sym_session to get the rte_crypto_sym_cpu_session.
Right now - just cipher_offset (see above).
What else in future (if any) - don't know.
> I don't think there is any.
> I believe the same crypto session will be able to work synchronously as well.
Exactly the same - problematically, see above.
> We would only need a new API to perform synchronous actions.
> That will reduce the duplication code significantly
> in the driver to support 2 different kind of APIs with similar code inside.
> Please correct me in case I am missing something.
To add new API into crypto-dev would also require changes in the PMD,
it wouldn't come totally free and I believe would require roughly the same amount of changes.
>
>
> > While rte_security is designed in a way that we can add new session types and
> > related parameters without causing API/ABI breakage.
>
> Yes the intent is to add new sessions based on various protocols that can be supported by the driver.
Various protocols and different types of sessions (and devices they belong to).
Let say right now we have INLINE_CRYPTO, INLINE_PROTO, LOOKASIDE_PROTO, etc.
Here we introduce new type of session.
> It is not that we should find it as an alternative to cryptodev and using it just because it will not cause
> ABI/API breakage.
I am considering this new API as an alternative to existing ones, but as an extension.
Existing crypto-op API has its own advantages (generic), and I think we should keep it supported by all crypto-devs.
From other side rte_security is an extendable framework that suits the purpose:
allows easily (and yes without ABI breakage) introduce new API for special type of crypto-dev (SW based).
> IMO the code should be placed where its intent is.
>
> >
> > BTW, what is your concern with proposed approach (via rte_security)?
> > From my perspective it is a lightweight change and it is totally optional
> > for the crypto PMDs to support it or not.
> > Konstantin
> >
> > > >
> > > > AESNI-GCM and AESNI-MB PMDs are updated with this support. There is a
> > small
> > > > performance test app under app/test/security_aesni_gcm(mb)_perftest to
> > > > prove.
> > > >
> > > > For the new API
> > > > The packet is sent to the crypto device for symmetric crypto
> > > > processing. The device will encrypt or decrypt the buffer based on the
> > session
> > > > data specified and preprocessed in the security session. Different
> > > > than the inline or lookaside modes, when the function exits, the user will
> > > > expect the buffers are either processed successfully, or having the error
> > number
> > > > assigned to the appropriate index of the status array.
> > > >
> > > > Will update the program's guide in the v1 patch.
> > > >
> > > > Regards,
> > > > Fan
> > > >
> > > > > -----Original Message-----
> > > > > From: Akhil Goyal [mailto:akhil.goyal@nxp.com]
> > > > > Sent: Wednesday, September 4, 2019 11:33 AM
> > > > > To: Zhang, Roy Fan <roy.fan.zhang@intel.com>; dev@dpdk.org
> > > > > Cc: Ananyev, Konstantin <konstantin.ananyev@intel.com>; Doherty,
> > Declan
> > > > > <declan.doherty@intel.com>; De Lara Guarch, Pablo
> > > > > <pablo.de.lara.guarch@intel.com>
> > > > > Subject: RE: [RFC PATCH 1/9] security: introduce CPU Crypto action type
> > and
> > > > > API
> > > > >
> > > > > Hi Fan,
> > > > >
> > > > > >
> > > > > > This patch introduce new RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO
> > > > > action
> > > > > > type to security library. The type represents performing crypto
> > > > > > operation with CPU cycles. The patch also includes a new API to
> > > > > > process crypto operations in bulk and the function pointers for PMDs.
> > > > > >
> > > > > I am not able to get the flow of execution for this action type. Could you
> > > > > please elaborate the flow in the documentation. If not in documentation
> > > > > right now, then please elaborate the flow in cover letter.
> > > > > Also I see that there are new APIs for processing crypto operations in bulk.
> > > > > What does that mean. How are they different from the existing APIs which
> > > > > are also handling bulk crypto ops depending on the budget.
> > > > >
> > > > >
> > > > > -Akhil
^ permalink raw reply [relevance 4%]
* Re: [dpdk-dev] [PATCH] mbuf: add bulk free function
@ 2019-09-11 11:39 3% ` Stephen Hemminger
0 siblings, 0 replies; 200+ results
From: Stephen Hemminger @ 2019-09-11 11:39 UTC (permalink / raw)
To: Olivier Matz; +Cc: Morten Brørup, dev
On Wed, 11 Sep 2019 13:33:13 +0200
Olivier Matz <olivier.matz@6wind.com> wrote:
> Hi,
>
> On Wed, Sep 11, 2019 at 12:18:34PM +0100, Stephen Hemminger wrote:
> > On Wed, 11 Sep 2019 09:19:08 +0000
> > Morten Brørup <mb@smartsharesystems.com> wrote:
> >
> > > Add function for freeing a bulk of mbufs.
> > >
> > > Signed-off-by: Morten Brørup <mb@smartsharesystems.com>
> > > ---
> > > lib/librte_mbuf/rte_mbuf.h | 17 +++++++++++++++++
> > > 1 file changed, 17 insertions(+)
> > >
> > > diff --git a/lib/librte_mbuf/rte_mbuf.h b/lib/librte_mbuf/rte_mbuf.h
> > > index 98225ec80..f2e174da1 100644
> > > --- a/lib/librte_mbuf/rte_mbuf.h
> > > +++ b/lib/librte_mbuf/rte_mbuf.h
> > > @@ -1907,6 +1907,23 @@ static inline void rte_pktmbuf_free(struct rte_mbuf *m)
> > > }
> > > }
> > >
> > > +/**
> > > + * Free a bulk of mbufs back into their original mempool.
> > > + *
> > > + * @param mbufs
> > > + * Array of pointers to mbufs
> > > + * @param count
> > > + * Array size
> > > + */
> > > +static inline void
> > > +rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned count)
> > > +{
> > > + unsigned idx = 0;
> > > +
> > > + for (idx = 0; idx < count; idx++)
> > > + rte_pktmbuf_free(mbufs[idx]);
> > > +}
> > > +
> >
> > You can optimize this to use mempool bulk put operation.
>
> A bulk free for mbuf is not as simple as a bulk mempool put, because
> of indirect mbufs, and because mbufs may return in different mempools.
>
> Morten, do you have more details about why do you need such a function?
>
> Thanks,
> Olivier
I was thinking of a function that looked at the list and if they were all
the same pool and safe to bulk put, then use that as a fast path. This would
be the most common case.
Also, less inline functions please. When it is an inline it adds more API/ABI
dependencies.
^ permalink raw reply [relevance 3%]
* [dpdk-dev] [RFC v3 0/4] get Rx/Tx packet burst mode information
@ 2019-09-10 16:33 3% Haiyue Wang
0 siblings, 0 replies; 200+ results
From: Haiyue Wang @ 2019-09-10 16:33 UTC (permalink / raw)
To: dev, ferruh.yigit, mdr, chenmin.sun; +Cc: Haiyue Wang
v1 -> v2:
Avoid ABI breaking, add a simple trace API to export string type
information.
v2 -> v3:
Use bit fields to present the burst mode setting, and rename the
APIs.
Haiyue Wang (4):
ethdev: add the API for getting burst mode information
net/i40e: support to get the Rx/Tx burst mode
net/ice: support to get the Rx/Tx burst mode
app/testpmd: show the Rx/Tx burst mode description
app/test-pmd/config.c | 25 ++++++++
doc/guides/rel_notes/release_19_11.rst | 8 +++
drivers/net/i40e/i40e_ethdev.c | 2 +
drivers/net/i40e/i40e_ethdev.h | 4 ++
drivers/net/i40e/i40e_rxtx.c | 72 +++++++++++++++++++++
drivers/net/ice/ice_ethdev.c | 2 +
drivers/net/ice/ice_rxtx.c | 54 ++++++++++++++++
drivers/net/ice/ice_rxtx.h | 4 ++
lib/librte_ethdev/rte_ethdev.c | 75 ++++++++++++++++++++++
lib/librte_ethdev/rte_ethdev.h | 82 ++++++++++++++++++++++++
lib/librte_ethdev/rte_ethdev_core.h | 5 ++
lib/librte_ethdev/rte_ethdev_version.map | 5 ++
12 files changed, 338 insertions(+)
--
2.17.1
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] [RFC PATCH 1/9] security: introduce CPU Crypto action type and API
2019-09-06 13:27 4% ` Ananyev, Konstantin
@ 2019-09-10 10:44 4% ` Akhil Goyal
2019-09-11 12:29 4% ` Ananyev, Konstantin
0 siblings, 1 reply; 200+ results
From: Akhil Goyal @ 2019-09-10 10:44 UTC (permalink / raw)
To: Ananyev, Konstantin, dev
Cc: Zhang, Roy Fan, Doherty, Declan, De Lara Guarch, Pablo
Hi Konstantin,
>
> Hi Akhil,
>
> > > This action type allows the burst of symmetric crypto workload using the
> same
> > > algorithm, key, and direction being processed by CPU cycles synchronously.
> > > This flexible action type does not require external hardware involvement,
> > > having the crypto workload processed synchronously, and is more
> performant
> > > than Cryptodev SW PMD due to the saved cycles on removed "async mode
> > > simulation" as well as 3 cacheline access of the crypto ops.
> >
> > Does that mean application will not call the cryptodev_enqueue_burst and
> corresponding dequeue burst.
>
> Yes, instead it just call rte_security_process_cpu_crypto_bulk(...)
>
> > It would be a new API something like process_packets and it will have the
> crypto processed packets while returning from the API?
>
> Yes, though the plan is that API will operate on raw data buffers, not mbufs.
>
> >
> > I still do not understand why we cannot do with the conventional crypto lib
> only.
> > As far as I can understand, you are not doing any protocol processing or any
> value add
> > To the crypto processing. IMO, you just need a synchronous crypto processing
> API which
> > Can be defined in cryptodev, you don't need to re-create a crypto session in
> the name of
> > Security session in the driver just to do a synchronous processing.
>
> I suppose your question is why not to have
> rte_crypot_process_cpu_crypto_bulk(...) instead?
> The main reason is that would require disruptive changes in existing cryptodev
> API
> (would cause ABI/API breakage).
> Session for RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO need some extra
> information
> that normal crypto_sym_xform doesn't contain
> (cipher offset from the start of the buffer, might be something extra in future).
Cipher offset will be part of rte_crypto_op. If you intend not to use rte_crypto_op
You can pass this as an argument in the new cryptodev API.
Something extra will also cause ABI breakage in security as well.
So it will be same.
> Also right now there is no way to add new type of crypto_sym_session without
> either breaking existing crypto-dev ABI/API or introducing new structure
> (rte_crypto_sym_cpu_session or so) for that.
What extra info is required in rte_cryptodev_sym_session to get the rte_crypto_sym_cpu_session.
I don't think there is any.
I believe the same crypto session will be able to work synchronously as well. We would only need
a new API to perform synchronous actions. That will reduce the duplication code significantly
in the driver to support 2 different kind of APIs with similar code inside.
Please correct me in case I am missing something.
> While rte_security is designed in a way that we can add new session types and
> related parameters without causing API/ABI breakage.
Yes the intent is to add new sessions based on various protocols that can be supported by the driver.
It is not that we should find it as an alternative to cryptodev and using it just because it will not cause
ABI/API breakage. IMO the code should be placed where its intent is.
>
> BTW, what is your concern with proposed approach (via rte_security)?
> From my perspective it is a lightweight change and it is totally optional
> for the crypto PMDs to support it or not.
> Konstantin
>
> > >
> > > AESNI-GCM and AESNI-MB PMDs are updated with this support. There is a
> small
> > > performance test app under app/test/security_aesni_gcm(mb)_perftest to
> > > prove.
> > >
> > > For the new API
> > > The packet is sent to the crypto device for symmetric crypto
> > > processing. The device will encrypt or decrypt the buffer based on the
> session
> > > data specified and preprocessed in the security session. Different
> > > than the inline or lookaside modes, when the function exits, the user will
> > > expect the buffers are either processed successfully, or having the error
> number
> > > assigned to the appropriate index of the status array.
> > >
> > > Will update the program's guide in the v1 patch.
> > >
> > > Regards,
> > > Fan
> > >
> > > > -----Original Message-----
> > > > From: Akhil Goyal [mailto:akhil.goyal@nxp.com]
> > > > Sent: Wednesday, September 4, 2019 11:33 AM
> > > > To: Zhang, Roy Fan <roy.fan.zhang@intel.com>; dev@dpdk.org
> > > > Cc: Ananyev, Konstantin <konstantin.ananyev@intel.com>; Doherty,
> Declan
> > > > <declan.doherty@intel.com>; De Lara Guarch, Pablo
> > > > <pablo.de.lara.guarch@intel.com>
> > > > Subject: RE: [RFC PATCH 1/9] security: introduce CPU Crypto action type
> and
> > > > API
> > > >
> > > > Hi Fan,
> > > >
> > > > >
> > > > > This patch introduce new RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO
> > > > action
> > > > > type to security library. The type represents performing crypto
> > > > > operation with CPU cycles. The patch also includes a new API to
> > > > > process crypto operations in bulk and the function pointers for PMDs.
> > > > >
> > > > I am not able to get the flow of execution for this action type. Could you
> > > > please elaborate the flow in the documentation. If not in documentation
> > > > right now, then please elaborate the flow in cover letter.
> > > > Also I see that there are new APIs for processing crypto operations in bulk.
> > > > What does that mean. How are they different from the existing APIs which
> > > > are also handling bulk crypto ops depending on the budget.
> > > >
> > > >
> > > > -Akhil
^ permalink raw reply [relevance 4%]
* [dpdk-dev] [PATCH 1/1] ethdev: change owner delete function return value to int
@ 2019-09-10 9:02 4% ` Andrew Rybchenko
0 siblings, 0 replies; 200+ results
From: Andrew Rybchenko @ 2019-09-10 9:02 UTC (permalink / raw)
To: Neil Horman, John McNamara, Marko Kovacevic, Stephen Hemminger,
K. Y. Srinivasan, Haiyang Zhang, Thomas Monjalon, Ferruh Yigit
Cc: dev, Igor Romanov
From: Igor Romanov <igor.romanov@oktetlabs.ru>
Change rte_eth_dev_owner_delete() return value from void to int
and return negative errno values in case of error conditions.
Right now there is only one error case for rte_eth_dev_owner_delete() -
invalid owner, but it still makes sense to return error to catch bugs
in the code which uses the function.
Also update the usage of the function in drivers/netvsc
according to the new return type.
Signed-off-by: Igor Romanov <igor.romanov@oktetlabs.ru>
Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
---
doc/guides/rel_notes/deprecation.rst | 1 -
doc/guides/rel_notes/release_19_11.rst | 3 +++
drivers/net/netvsc/hn_ethdev.c | 5 ++++-
lib/librte_ethdev/rte_ethdev.c | 6 +++++-
lib/librte_ethdev/rte_ethdev.h | 4 +++-
5 files changed, 15 insertions(+), 4 deletions(-)
diff --git a/doc/guides/rel_notes/deprecation.rst b/doc/guides/rel_notes/deprecation.rst
index 61ba2e0bc..237813b64 100644
--- a/doc/guides/rel_notes/deprecation.rst
+++ b/doc/guides/rel_notes/deprecation.rst
@@ -90,7 +90,6 @@ Deprecation Notices
- ``rte_eth_dev_stop``
- ``rte_eth_dev_close``
- - ``rte_eth_dev_owner_delete``
* ethdev: New offload flags ``DEV_RX_OFFLOAD_RSS_HASH`` and
``DEV_RX_OFFLOAD_FLOW_MARK`` will be added in 19.11.
diff --git a/doc/guides/rel_notes/release_19_11.rst b/doc/guides/rel_notes/release_19_11.rst
index 033ed54f4..54950277b 100644
--- a/doc/guides/rel_notes/release_19_11.rst
+++ b/doc/guides/rel_notes/release_19_11.rst
@@ -115,6 +115,9 @@ API Changes
* ethdev: changed ``rte_eth_macaddr_get`` return value from ``void`` to
``int`` to provide a way to report various error conditions.
+* ethdev: changed ``rte_eth_dev_owner_delete`` return value from ``void`` to
+ ``int`` to provide a way to report various error conditions.
+
ABI Changes
-----------
diff --git a/drivers/net/netvsc/hn_ethdev.c b/drivers/net/netvsc/hn_ethdev.c
index ca96e80a4..eed8dece9 100644
--- a/drivers/net/netvsc/hn_ethdev.c
+++ b/drivers/net/netvsc/hn_ethdev.c
@@ -1004,6 +1004,7 @@ static int
eth_hn_dev_uninit(struct rte_eth_dev *eth_dev)
{
struct hn_data *hv = eth_dev->data->dev_private;
+ int ret;
PMD_INIT_FUNC_TRACE();
@@ -1021,7 +1022,9 @@ eth_hn_dev_uninit(struct rte_eth_dev *eth_dev)
hn_tx_pool_uninit(eth_dev);
rte_vmbus_chan_close(hv->primary->chan);
rte_free(hv->primary);
- rte_eth_dev_owner_delete(hv->owner.id);
+ ret = rte_eth_dev_owner_delete(hv->owner.id);
+ if (ret != 0)
+ return ret;
return 0;
}
diff --git a/lib/librte_ethdev/rte_ethdev.c b/lib/librte_ethdev/rte_ethdev.c
index 29ecb9274..e50bac847 100644
--- a/lib/librte_ethdev/rte_ethdev.c
+++ b/lib/librte_ethdev/rte_ethdev.c
@@ -687,10 +687,11 @@ rte_eth_dev_owner_unset(const uint16_t port_id, const uint64_t owner_id)
return ret;
}
-void
+int
rte_eth_dev_owner_delete(const uint64_t owner_id)
{
uint16_t port_id;
+ int ret = 0;
rte_eth_dev_shared_data_prepare();
@@ -708,9 +709,12 @@ rte_eth_dev_owner_delete(const uint64_t owner_id)
RTE_ETHDEV_LOG(ERR,
"Invalid owner id=%016"PRIx64"\n",
owner_id);
+ ret = -EINVAL;
}
rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
+
+ return ret;
}
int
diff --git a/lib/librte_ethdev/rte_ethdev.h b/lib/librte_ethdev/rte_ethdev.h
index 9c213e072..d937fb429 100644
--- a/lib/librte_ethdev/rte_ethdev.h
+++ b/lib/librte_ethdev/rte_ethdev.h
@@ -1566,9 +1566,11 @@ int rte_eth_dev_owner_unset(const uint16_t port_id,
*
* @param owner_id
* The owner identifier.
+ * @return
+ * 0 on success, negative errno value on error.
*/
__rte_experimental
-void rte_eth_dev_owner_delete(const uint64_t owner_id);
+int rte_eth_dev_owner_delete(const uint64_t owner_id);
/**
* @warning
--
2.17.1
^ permalink raw reply [relevance 4%]
* [dpdk-dev] [PATCH 1/7] ethdev: change MAC addr get function return value to int
@ 2019-09-10 8:52 4% ` Andrew Rybchenko
0 siblings, 0 replies; 200+ results
From: Andrew Rybchenko @ 2019-09-10 8:52 UTC (permalink / raw)
To: Neil Horman, John McNamara, Marko Kovacevic, Bernard Iremonger,
Thomas Monjalon, Ferruh Yigit
Cc: dev, Igor Romanov
From: Igor Romanov <igor.romanov@oktetlabs.ru>
Change rte_eth_macaddr_get() return value from void to int
and return negative errno values in case of error conditions.
Signed-off-by: Igor Romanov <igor.romanov@oktetlabs.ru>
Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
---
doc/guides/rel_notes/deprecation.rst | 1 -
doc/guides/rel_notes/release_19_11.rst | 3 +++
doc/guides/sample_app_ug/flow_classify.rst | 4 +++-
lib/librte_ethdev/rte_ethdev.c | 6 ++++--
lib/librte_ethdev/rte_ethdev.h | 5 ++++-
5 files changed, 14 insertions(+), 5 deletions(-)
diff --git a/doc/guides/rel_notes/deprecation.rst b/doc/guides/rel_notes/deprecation.rst
index 43b15ec2f..61ba2e0bc 100644
--- a/doc/guides/rel_notes/deprecation.rst
+++ b/doc/guides/rel_notes/deprecation.rst
@@ -90,7 +90,6 @@ Deprecation Notices
- ``rte_eth_dev_stop``
- ``rte_eth_dev_close``
- - ``rte_eth_macaddr_get``
- ``rte_eth_dev_owner_delete``
* ethdev: New offload flags ``DEV_RX_OFFLOAD_RSS_HASH`` and
diff --git a/doc/guides/rel_notes/release_19_11.rst b/doc/guides/rel_notes/release_19_11.rst
index 3ff1296a2..033ed54f4 100644
--- a/doc/guides/rel_notes/release_19_11.rst
+++ b/doc/guides/rel_notes/release_19_11.rst
@@ -112,6 +112,9 @@ API Changes
return value from ``void`` to ``int`` to provide a way to report various
error conditions.
+* ethdev: changed ``rte_eth_macaddr_get`` return value from ``void`` to
+ ``int`` to provide a way to report various error conditions.
+
ABI Changes
-----------
diff --git a/doc/guides/sample_app_ug/flow_classify.rst b/doc/guides/sample_app_ug/flow_classify.rst
index 7c2b6dcf8..bc234b50a 100644
--- a/doc/guides/sample_app_ug/flow_classify.rst
+++ b/doc/guides/sample_app_ug/flow_classify.rst
@@ -306,7 +306,9 @@ Forwarding application is shown below:
return retval;
/* Display the port MAC address. */
- rte_eth_macaddr_get(port, &addr);
+ retval = rte_eth_macaddr_get(port, &addr);
+ if (retval < 0)
+ return retval;
printf("Port %u MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8
" %02" PRIx8 " %02" PRIx8 " %02" PRIx8 "\n",
port,
diff --git a/lib/librte_ethdev/rte_ethdev.c b/lib/librte_ethdev/rte_ethdev.c
index b9fa5f562..29ecb9274 100644
--- a/lib/librte_ethdev/rte_ethdev.c
+++ b/lib/librte_ethdev/rte_ethdev.c
@@ -2702,14 +2702,16 @@ rte_eth_dev_get_supported_ptypes(uint16_t port_id, uint32_t ptype_mask,
return j;
}
-void
+int
rte_eth_macaddr_get(uint16_t port_id, struct rte_ether_addr *mac_addr)
{
struct rte_eth_dev *dev;
- RTE_ETH_VALID_PORTID_OR_RET(port_id);
+ RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
dev = &rte_eth_devices[port_id];
rte_ether_addr_copy(&dev->data->mac_addrs[0], mac_addr);
+
+ return 0;
}
diff --git a/lib/librte_ethdev/rte_ethdev.h b/lib/librte_ethdev/rte_ethdev.h
index aba5b4c86..9c213e072 100644
--- a/lib/librte_ethdev/rte_ethdev.h
+++ b/lib/librte_ethdev/rte_ethdev.h
@@ -2361,8 +2361,11 @@ int rte_eth_dev_set_rx_queue_stats_mapping(uint16_t port_id,
* @param mac_addr
* A pointer to a structure of type *ether_addr* to be filled with
* the Ethernet address of the Ethernet device.
+ * @return
+ * - (0) if successful
+ * - (-ENODEV) if *port_id* invalid.
*/
-void rte_eth_macaddr_get(uint16_t port_id, struct rte_ether_addr *mac_addr);
+int rte_eth_macaddr_get(uint16_t port_id, struct rte_ether_addr *mac_addr);
/**
* Retrieve the contextual information of an Ethernet device.
--
2.17.1
^ permalink raw reply [relevance 4%]
* [dpdk-dev] [PATCH 02/18] ethdev: change link status get functions return value to int
@ 2019-09-10 8:25 3% ` Andrew Rybchenko
0 siblings, 0 replies; 200+ results
From: Andrew Rybchenko @ 2019-09-10 8:25 UTC (permalink / raw)
To: Neil Horman, John McNamara, Marko Kovacevic, Ori Kam,
Bruce Richardson, Pablo de Lara, Radu Nicolau, Akhil Goyal,
Tomasz Kantecki, Chas Williams, Thomas Monjalon, Ferruh Yigit
Cc: dev, Igor Romanov
From: Igor Romanov <igor.romanov@oktetlabs.ru>
Change rte_eth_link_get() and rte_eth_link_get_nowait() return value
from void to int and return negative errno values in case of error
conditions.
Return value of link_update callback is ignored since the callback
returns not errors but whether link up status has changed or not.
Signed-off-by: Igor Romanov <igor.romanov@oktetlabs.ru>
Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
---
doc/guides/rel_notes/deprecation.rst | 1 -
doc/guides/rel_notes/release_19_11.rst | 4 ++++
doc/guides/sample_app_ug/link_status_intr.rst | 9 ++++++---
drivers/net/bonding/rte_eth_bond_pmd.c | 2 +-
lib/librte_ethdev/rte_ethdev.c | 16 ++++++++++------
lib/librte_ethdev/rte_ethdev.h | 12 ++++++++++--
6 files changed, 31 insertions(+), 13 deletions(-)
diff --git a/doc/guides/rel_notes/deprecation.rst b/doc/guides/rel_notes/deprecation.rst
index 165d13726..43b15ec2f 100644
--- a/doc/guides/rel_notes/deprecation.rst
+++ b/doc/guides/rel_notes/deprecation.rst
@@ -88,7 +88,6 @@ Deprecation Notices
negative errno values to indicate various error conditions (e.g.
invalid port ID, unsupported operation, failed operation):
- - ``rte_eth_link_get`` and ``rte_eth_link_get_nowait``
- ``rte_eth_dev_stop``
- ``rte_eth_dev_close``
- ``rte_eth_macaddr_get``
diff --git a/doc/guides/rel_notes/release_19_11.rst b/doc/guides/rel_notes/release_19_11.rst
index d728592c8..3ff1296a2 100644
--- a/doc/guides/rel_notes/release_19_11.rst
+++ b/doc/guides/rel_notes/release_19_11.rst
@@ -108,6 +108,10 @@ API Changes
* ethdev: changed ``rte_eth_dev_xstats_reset`` return value from ``void`` to
``int`` to provide a way to report various error conditions.
+* ethdev: changed ``rte_eth_link_get`` and ``rte_eth_link_get_nowait``
+ return value from ``void`` to ``int`` to provide a way to report various
+ error conditions.
+
ABI Changes
-----------
diff --git a/doc/guides/sample_app_ug/link_status_intr.rst b/doc/guides/sample_app_ug/link_status_intr.rst
index cfb1bcd58..5283be8b7 100644
--- a/doc/guides/sample_app_ug/link_status_intr.rst
+++ b/doc/guides/sample_app_ug/link_status_intr.rst
@@ -164,6 +164,7 @@ An example callback function that has been written as indicated below.
lsi_event_callback(uint16_t port_id, enum rte_eth_event_type type, void *param)
{
struct rte_eth_link link;
+ int ret;
RTE_SET_USED(param);
@@ -171,9 +172,11 @@ An example callback function that has been written as indicated below.
printf("Event type: %s\n", type == RTE_ETH_EVENT_INTR_LSC ? "LSC interrupt" : "unknown event");
- rte_eth_link_get_nowait(port_id, &link);
-
- if (link.link_status) {
+ ret = rte_eth_link_get_nowait(port_id, &link);
+ if (ret < 0) {
+ printf("Failed to get port %d link status: %s\n\n",
+ port_id, rte_strerror(-ret));
+ } else if (link.link_status) {
printf("Port %d Link Up - speed %u Mbps - %s\n\n", port_id, (unsigned)link.link_speed,
(link.link_duplex == ETH_LINK_FULL_DUPLEX) ? ("full-duplex") : ("half-duplex"));
} else
diff --git a/drivers/net/bonding/rte_eth_bond_pmd.c b/drivers/net/bonding/rte_eth_bond_pmd.c
index fed71bd95..9316f93f7 100644
--- a/drivers/net/bonding/rte_eth_bond_pmd.c
+++ b/drivers/net/bonding/rte_eth_bond_pmd.c
@@ -2358,7 +2358,7 @@ bond_ethdev_slave_link_status_change_monitor(void *cb_arg)
static int
bond_ethdev_link_update(struct rte_eth_dev *ethdev, int wait_to_complete)
{
- void (*link_update)(uint16_t port_id, struct rte_eth_link *eth_link);
+ int (*link_update)(uint16_t port_id, struct rte_eth_link *eth_link);
struct bond_dev_private *bond_ctx;
struct rte_eth_link slave_link;
diff --git a/lib/librte_ethdev/rte_ethdev.c b/lib/librte_ethdev/rte_ethdev.c
index 1bd1e32b0..b9fa5f562 100644
--- a/lib/librte_ethdev/rte_ethdev.c
+++ b/lib/librte_ethdev/rte_ethdev.c
@@ -2015,40 +2015,44 @@ rte_eth_allmulticast_get(uint16_t port_id)
return dev->data->all_multicast;
}
-void
+int
rte_eth_link_get(uint16_t port_id, struct rte_eth_link *eth_link)
{
struct rte_eth_dev *dev;
- RTE_ETH_VALID_PORTID_OR_RET(port_id);
+ RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
dev = &rte_eth_devices[port_id];
if (dev->data->dev_conf.intr_conf.lsc &&
dev->data->dev_started)
rte_eth_linkstatus_get(dev, eth_link);
else {
- RTE_FUNC_PTR_OR_RET(*dev->dev_ops->link_update);
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->link_update, -ENOTSUP);
(*dev->dev_ops->link_update)(dev, 1);
*eth_link = dev->data->dev_link;
}
+
+ return 0;
}
-void
+int
rte_eth_link_get_nowait(uint16_t port_id, struct rte_eth_link *eth_link)
{
struct rte_eth_dev *dev;
- RTE_ETH_VALID_PORTID_OR_RET(port_id);
+ RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
dev = &rte_eth_devices[port_id];
if (dev->data->dev_conf.intr_conf.lsc &&
dev->data->dev_started)
rte_eth_linkstatus_get(dev, eth_link);
else {
- RTE_FUNC_PTR_OR_RET(*dev->dev_ops->link_update);
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->link_update, -ENOTSUP);
(*dev->dev_ops->link_update)(dev, 0);
*eth_link = dev->data->dev_link;
}
+
+ return 0;
}
int
diff --git a/lib/librte_ethdev/rte_ethdev.h b/lib/librte_ethdev/rte_ethdev.h
index 14420dbbe..aba5b4c86 100644
--- a/lib/librte_ethdev/rte_ethdev.h
+++ b/lib/librte_ethdev/rte_ethdev.h
@@ -2103,8 +2103,12 @@ int rte_eth_allmulticast_get(uint16_t port_id);
* @param link
* A pointer to an *rte_eth_link* structure to be filled with
* the status, the speed and the mode of the Ethernet device link.
+ * @return
+ * - (0) if successful.
+ * - (-ENOTSUP) if the function is not supported in PMD driver.
+ * - (-ENODEV) if *port_id* invalid.
*/
-void rte_eth_link_get(uint16_t port_id, struct rte_eth_link *link);
+int rte_eth_link_get(uint16_t port_id, struct rte_eth_link *link);
/**
* Retrieve the status (ON/OFF), the speed (in Mbps) and the mode (HALF-DUPLEX
@@ -2116,8 +2120,12 @@ void rte_eth_link_get(uint16_t port_id, struct rte_eth_link *link);
* @param link
* A pointer to an *rte_eth_link* structure to be filled with
* the status, the speed and the mode of the Ethernet device link.
+ * @return
+ * - (0) if successful.
+ * - (-ENOTSUP) if the function is not supported in PMD driver.
+ * - (-ENODEV) if *port_id* invalid.
*/
-void rte_eth_link_get_nowait(uint16_t port_id, struct rte_eth_link *link);
+int rte_eth_link_get_nowait(uint16_t port_id, struct rte_eth_link *link);
/**
* Retrieve the general I/O statistics of an Ethernet device.
--
2.17.1
^ permalink raw reply [relevance 3%]
* [dpdk-dev] [PATCH 1/7] ethdev: change allmulticast mode controllers to return errors
@ 2019-09-09 12:13 4% ` Andrew Rybchenko
0 siblings, 0 replies; 200+ results
From: Andrew Rybchenko @ 2019-09-09 12:13 UTC (permalink / raw)
To: Neil Horman, John McNamara, Marko Kovacevic, Thomas Monjalon,
Ferruh Yigit
Cc: dev, Ivan Ilchenko
From: Ivan Ilchenko <Ivan.Ilchenko@oktetlabs.ru>
Change rte_eth_allmulticast_enable()/rte_eth_allmulticast_disable()
return value from void to int and return negative errno values
in case of error conditions.
Modify usage of these functions across the ethdev according
to new return type.
Signed-off-by: Ivan Ilchenko <Ivan.Ilchenko@oktetlabs.ru>
Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
---
doc/guides/rel_notes/deprecation.rst | 1 -
doc/guides/rel_notes/release_19_11.rst | 4 +++
lib/librte_ethdev/rte_ethdev.c | 37 +++++++++++++++++++-------
lib/librte_ethdev/rte_ethdev.h | 14 ++++++++--
4 files changed, 43 insertions(+), 13 deletions(-)
diff --git a/doc/guides/rel_notes/deprecation.rst b/doc/guides/rel_notes/deprecation.rst
index b2e0a1fc7c..0d8e231ef5 100644
--- a/doc/guides/rel_notes/deprecation.rst
+++ b/doc/guides/rel_notes/deprecation.rst
@@ -88,7 +88,6 @@ Deprecation Notices
negative errno values to indicate various error conditions (e.g.
invalid port ID, unsupported operation, failed operation):
- - ``rte_eth_allmulticast_enable`` and ``rte_eth_allmulticast_disable``
- ``rte_eth_link_get`` and ``rte_eth_link_get_nowait``
- ``rte_eth_dev_stop``
- ``rte_eth_dev_close``
diff --git a/doc/guides/rel_notes/release_19_11.rst b/doc/guides/rel_notes/release_19_11.rst
index 5299157df7..a79e6a08e6 100644
--- a/doc/guides/rel_notes/release_19_11.rst
+++ b/doc/guides/rel_notes/release_19_11.rst
@@ -101,6 +101,10 @@ API Changes
``rte_eth_promiscuous_disable`` return value from ``void`` to ``int`` to
provide a way to report various error conditions.
+* ethdev: changed ``rte_eth_allmulticast_enable`` and
+ ``rte_eth_allmulticast_disable`` return value from ``void`` to ``int`` to
+ provide a way to report various error conditions.
+
ABI Changes
-----------
diff --git a/lib/librte_ethdev/rte_ethdev.c b/lib/librte_ethdev/rte_ethdev.c
index 98fe533c5a..577640fdbe 100644
--- a/lib/librte_ethdev/rte_ethdev.c
+++ b/lib/librte_ethdev/rte_ethdev.c
@@ -1416,10 +1416,23 @@ rte_eth_dev_config_restore(struct rte_eth_dev *dev,
}
/* replay all multicast configuration */
- if (rte_eth_allmulticast_get(port_id) == 1)
- rte_eth_allmulticast_enable(port_id);
- else if (rte_eth_allmulticast_get(port_id) == 0)
- rte_eth_allmulticast_disable(port_id);
+ if (rte_eth_allmulticast_get(port_id) == 1) {
+ ret = rte_eth_allmulticast_enable(port_id);
+ if (ret != 0 && ret != -ENOTSUP) {
+ RTE_ETHDEV_LOG(ERR,
+ "Failed to enable allmulticast mode for device (port %u): %s\n",
+ port_id, rte_strerror(-ret));
+ return ret;
+ }
+ } else if (rte_eth_allmulticast_get(port_id) == 0) {
+ ret = rte_eth_allmulticast_disable(port_id);
+ if (ret != 0 && ret != -ENOTSUP) {
+ RTE_ETHDEV_LOG(ERR,
+ "Failed to disable allmulticast mode for device (port %u): %s\n",
+ port_id, rte_strerror(-ret));
+ return ret;
+ }
+ }
return 0;
}
@@ -1945,30 +1958,34 @@ rte_eth_promiscuous_get(uint16_t port_id)
return dev->data->promiscuous;
}
-void
+int
rte_eth_allmulticast_enable(uint16_t port_id)
{
struct rte_eth_dev *dev;
- RTE_ETH_VALID_PORTID_OR_RET(port_id);
+ RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
dev = &rte_eth_devices[port_id];
- RTE_FUNC_PTR_OR_RET(*dev->dev_ops->allmulticast_enable);
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->allmulticast_enable, -ENOTSUP);
(*dev->dev_ops->allmulticast_enable)(dev);
dev->data->all_multicast = 1;
+
+ return 0;
}
-void
+int
rte_eth_allmulticast_disable(uint16_t port_id)
{
struct rte_eth_dev *dev;
- RTE_ETH_VALID_PORTID_OR_RET(port_id);
+ RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
dev = &rte_eth_devices[port_id];
- RTE_FUNC_PTR_OR_RET(*dev->dev_ops->allmulticast_disable);
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->allmulticast_disable, -ENOTSUP);
dev->data->all_multicast = 0;
(*dev->dev_ops->allmulticast_disable)(dev);
+
+ return 0;
}
int
diff --git a/lib/librte_ethdev/rte_ethdev.h b/lib/librte_ethdev/rte_ethdev.h
index f07a829b29..d24e3f9a24 100644
--- a/lib/librte_ethdev/rte_ethdev.h
+++ b/lib/librte_ethdev/rte_ethdev.h
@@ -2060,16 +2060,26 @@ int rte_eth_promiscuous_get(uint16_t port_id);
*
* @param port_id
* The port identifier of the Ethernet device.
+ * @return
+ * - (0) if successful.
+ * - (-ENOTSUP) if support for allmulticast_enable() does not exist
+ * for the device.
+ * - (-ENODEV) if *port_id* invalid.
*/
-void rte_eth_allmulticast_enable(uint16_t port_id);
+int rte_eth_allmulticast_enable(uint16_t port_id);
/**
* Disable the receipt of all multicast frames by an Ethernet device.
*
* @param port_id
* The port identifier of the Ethernet device.
+ * @return
+ * - (0) if successful.
+ * - (-ENOTSUP) if support for allmulticast_disable() does not exist
+ * for the device.
+ * - (-ENODEV) if *port_id* invalid.
*/
-void rte_eth_allmulticast_disable(uint16_t port_id);
+int rte_eth_allmulticast_disable(uint16_t port_id);
/**
* Return the value of allmulticast mode for an Ethernet device.
--
2.17.1
^ permalink raw reply [relevance 4%]
* [dpdk-dev] [PATCH v2 01/13] ethdev: change promiscuous mode controllers to return errors
@ 2019-09-09 11:58 3% ` Andrew Rybchenko
0 siblings, 0 replies; 200+ results
From: Andrew Rybchenko @ 2019-09-09 11:58 UTC (permalink / raw)
To: Neil Horman, John McNamara, Marko Kovacevic, Bernard Iremonger,
Ori Kam, Bruce Richardson, Pablo de Lara, Radu Nicolau,
Akhil Goyal, Tomasz Kantecki, Harry van Haaren, Xiaoyun Li,
Thomas Monjalon, Ferruh Yigit
Cc: dev, Ivan Ilchenko
From: Ivan Ilchenko <Ivan.Ilchenko@oktetlabs.ru>
Change rte_eth_promiscuous_enable()/rte_eth_promiscuous_disable()
return value from void to int and return negative errno values
in case of error conditions.
Modify usage of these functions across the ethdev according
to new return type.
Signed-off-by: Ivan Ilchenko <Ivan.Ilchenko@oktetlabs.ru>
Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
---
doc/guides/rel_notes/deprecation.rst | 1 -
doc/guides/rel_notes/release_19_11.rst | 4 ++
doc/guides/sample_app_ug/flow_classify.rst | 6 ++-
doc/guides/sample_app_ug/flow_filtering.rst | 15 +++++-
doc/guides/sample_app_ug/rxtx_callbacks.rst | 5 +-
doc/guides/sample_app_ug/skeleton.rst | 6 ++-
lib/librte_ethdev/rte_ethdev.c | 52 ++++++++++++++++-----
lib/librte_ethdev/rte_ethdev.h | 14 +++++-
8 files changed, 80 insertions(+), 23 deletions(-)
diff --git a/doc/guides/rel_notes/deprecation.rst b/doc/guides/rel_notes/deprecation.rst
index cbb4c34efd..b2e0a1fc7c 100644
--- a/doc/guides/rel_notes/deprecation.rst
+++ b/doc/guides/rel_notes/deprecation.rst
@@ -88,7 +88,6 @@ Deprecation Notices
negative errno values to indicate various error conditions (e.g.
invalid port ID, unsupported operation, failed operation):
- - ``rte_eth_promiscuous_enable`` and ``rte_eth_promiscuous_disable``
- ``rte_eth_allmulticast_enable`` and ``rte_eth_allmulticast_disable``
- ``rte_eth_link_get`` and ``rte_eth_link_get_nowait``
- ``rte_eth_dev_stop``
diff --git a/doc/guides/rel_notes/release_19_11.rst b/doc/guides/rel_notes/release_19_11.rst
index 152f120197..5299157df7 100644
--- a/doc/guides/rel_notes/release_19_11.rst
+++ b/doc/guides/rel_notes/release_19_11.rst
@@ -97,6 +97,10 @@ API Changes
* ethdev: changed ``rte_eth_dev_infos_get`` return value from ``void`` to
``int`` to provide a way to report various error conditions.
+* ethdev: changed ``rte_eth_promiscuous_enable`` and
+ ``rte_eth_promiscuous_disable`` return value from ``void`` to ``int`` to
+ provide a way to report various error conditions.
+
ABI Changes
-----------
diff --git a/doc/guides/sample_app_ug/flow_classify.rst b/doc/guides/sample_app_ug/flow_classify.rst
index 96a5c66d0f..7c2b6dcf83 100644
--- a/doc/guides/sample_app_ug/flow_classify.rst
+++ b/doc/guides/sample_app_ug/flow_classify.rst
@@ -315,7 +315,9 @@ Forwarding application is shown below:
addr.addr_bytes[4], addr.addr_bytes[5]);
/* Enable RX in promiscuous mode for the Ethernet device. */
- rte_eth_promiscuous_enable(port);
+ retval = rte_eth_promiscuous_enable(port);
+ if (retval != 0)
+ return retval;
return 0;
}
@@ -343,7 +345,7 @@ Finally the RX port is set in promiscuous mode:
.. code-block:: c
- rte_eth_promiscuous_enable(port);
+ retval = rte_eth_promiscuous_enable(port);
The Add Rules function
~~~~~~~~~~~~~~~~~~~~~~
diff --git a/doc/guides/sample_app_ug/flow_filtering.rst b/doc/guides/sample_app_ug/flow_filtering.rst
index 02fc675506..de3e4ab0b6 100644
--- a/doc/guides/sample_app_ug/flow_filtering.rst
+++ b/doc/guides/sample_app_ug/flow_filtering.rst
@@ -193,7 +193,13 @@ application is shown below:
}
}
- rte_eth_promiscuous_enable(port_id);
+ ret = rte_eth_promiscuous_enable(port_id);
+ if (ret != 0) {
+ rte_exit(EXIT_FAILURE,
+ ":: cannot enable promiscuous mode: err=%d, port=%u\n",
+ ret, port_id);
+ }
+
ret = rte_eth_dev_start(port_id);
if (ret < 0) {
rte_exit(EXIT_FAILURE,
@@ -278,7 +284,12 @@ We are setting the RX port to promiscuous mode:
.. code-block:: c
- rte_eth_promiscuous_enable(port_id);
+ ret = rte_eth_promiscuous_enable(port_id);
+ if (ret != 0) {
+ rte_exit(EXIT_FAILURE,
+ ":: cannot enable promiscuous mode: err=%d, port=%u\n",
+ ret, port_id);
+ }
The last step is to start the port.
diff --git a/doc/guides/sample_app_ug/rxtx_callbacks.rst b/doc/guides/sample_app_ug/rxtx_callbacks.rst
index 32c120992f..0a69ec71ab 100644
--- a/doc/guides/sample_app_ug/rxtx_callbacks.rst
+++ b/doc/guides/sample_app_ug/rxtx_callbacks.rst
@@ -117,8 +117,9 @@ comments:
return retval;
/* Enable RX in promiscuous mode for the Ethernet device. */
- rte_eth_promiscuous_enable(port);
-
+ retval = rte_eth_promiscuous_enable(port);
+ if (retval != 0)
+ return retval;
/* Add the callbacks for RX and TX.*/
rte_eth_add_rx_callback(port, 0, add_timestamps, NULL);
diff --git a/doc/guides/sample_app_ug/skeleton.rst b/doc/guides/sample_app_ug/skeleton.rst
index 59ca511d33..1d0a2760d4 100644
--- a/doc/guides/sample_app_ug/skeleton.rst
+++ b/doc/guides/sample_app_ug/skeleton.rst
@@ -149,7 +149,9 @@ Forwarding application is shown below:
return retval;
/* Enable RX in promiscuous mode for the Ethernet device. */
- rte_eth_promiscuous_enable(port);
+ retval = rte_eth_promiscuous_enable(port);
+ if (retval != 0)
+ return retval;
return 0;
}
@@ -177,7 +179,7 @@ Finally the RX port is set in promiscuous mode:
.. code-block:: c
- rte_eth_promiscuous_enable(port);
+ retval = rte_eth_promiscuous_enable(port);
The Lcores Main
diff --git a/lib/librte_ethdev/rte_ethdev.c b/lib/librte_ethdev/rte_ethdev.c
index 30b0c7803f..b97dd8aa85 100644
--- a/lib/librte_ethdev/rte_ethdev.c
+++ b/lib/librte_ethdev/rte_ethdev.c
@@ -1381,24 +1381,41 @@ rte_eth_dev_mac_restore(struct rte_eth_dev *dev,
}
}
-static void
+static int
rte_eth_dev_config_restore(struct rte_eth_dev *dev,
struct rte_eth_dev_info *dev_info, uint16_t port_id)
{
+ int ret;
+
if (!(*dev_info->dev_flags & RTE_ETH_DEV_NOLIVE_MAC_ADDR))
rte_eth_dev_mac_restore(dev, dev_info);
/* replay promiscuous configuration */
- if (rte_eth_promiscuous_get(port_id) == 1)
- rte_eth_promiscuous_enable(port_id);
- else if (rte_eth_promiscuous_get(port_id) == 0)
- rte_eth_promiscuous_disable(port_id);
+ if (rte_eth_promiscuous_get(port_id) == 1) {
+ ret = rte_eth_promiscuous_enable(port_id);
+ if (ret != 0 && ret != -ENOTSUP) {
+ RTE_ETHDEV_LOG(ERR,
+ "Failed to enable promiscuous mode for device (port %u): %s\n",
+ port_id, rte_strerror(-ret));
+ return ret;
+ }
+ } else if (rte_eth_promiscuous_get(port_id) == 0) {
+ ret = rte_eth_promiscuous_disable(port_id);
+ if (ret != 0 && ret != -ENOTSUP) {
+ RTE_ETHDEV_LOG(ERR,
+ "Failed to disable promiscuous mode for device (port %u): %s\n",
+ port_id, rte_strerror(-ret));
+ return ret;
+ }
+ }
/* replay all multicast configuration */
if (rte_eth_allmulticast_get(port_id) == 1)
rte_eth_allmulticast_enable(port_id);
else if (rte_eth_allmulticast_get(port_id) == 0)
rte_eth_allmulticast_disable(port_id);
+
+ return 0;
}
int
@@ -1436,7 +1453,14 @@ rte_eth_dev_start(uint16_t port_id)
else
return eth_err(port_id, diag);
- rte_eth_dev_config_restore(dev, &dev_info, port_id);
+ ret = rte_eth_dev_config_restore(dev, &dev_info, port_id);
+ if (ret != 0) {
+ RTE_ETHDEV_LOG(ERR,
+ "Error during restoring configuration for device (port %u): %s\n",
+ port_id, rte_strerror(-ret));
+ rte_eth_dev_stop(port_id);
+ return ret;
+ }
if (dev->data->dev_conf.intr_conf.lsc == 0) {
RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->link_update, -ENOTSUP);
@@ -1864,30 +1888,34 @@ rte_eth_tx_done_cleanup(uint16_t port_id, uint16_t queue_id, uint32_t free_cnt)
return eth_err(port_id, ret);
}
-void
+int
rte_eth_promiscuous_enable(uint16_t port_id)
{
struct rte_eth_dev *dev;
- RTE_ETH_VALID_PORTID_OR_RET(port_id);
+ RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
dev = &rte_eth_devices[port_id];
- RTE_FUNC_PTR_OR_RET(*dev->dev_ops->promiscuous_enable);
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->promiscuous_enable, -ENOTSUP);
(*dev->dev_ops->promiscuous_enable)(dev);
dev->data->promiscuous = 1;
+
+ return 0;
}
-void
+int
rte_eth_promiscuous_disable(uint16_t port_id)
{
struct rte_eth_dev *dev;
- RTE_ETH_VALID_PORTID_OR_RET(port_id);
+ RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
dev = &rte_eth_devices[port_id];
- RTE_FUNC_PTR_OR_RET(*dev->dev_ops->promiscuous_disable);
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->promiscuous_disable, -ENOTSUP);
dev->data->promiscuous = 0;
(*dev->dev_ops->promiscuous_disable)(dev);
+
+ return 0;
}
int
diff --git a/lib/librte_ethdev/rte_ethdev.h b/lib/librte_ethdev/rte_ethdev.h
index 475dbdae17..f07a829b29 100644
--- a/lib/librte_ethdev/rte_ethdev.h
+++ b/lib/librte_ethdev/rte_ethdev.h
@@ -2022,16 +2022,26 @@ int rte_eth_dev_reset(uint16_t port_id);
*
* @param port_id
* The port identifier of the Ethernet device.
+ * @return
+ * - (0) if successful.
+ * - (-ENOTSUP) if support for promiscuous_enable() does not exist
+ * for the device.
+ * - (-ENODEV) if *port_id* invalid.
*/
-void rte_eth_promiscuous_enable(uint16_t port_id);
+int rte_eth_promiscuous_enable(uint16_t port_id);
/**
* Disable receipt in promiscuous mode for an Ethernet device.
*
* @param port_id
* The port identifier of the Ethernet device.
+ * @return
+ * - (0) if successful.
+ * - (-ENOTSUP) if support for promiscuous_disable() does not exist
+ * for the device.
+ * - (-ENODEV) if *port_id* invalid.
*/
-void rte_eth_promiscuous_disable(uint16_t port_id);
+int rte_eth_promiscuous_disable(uint16_t port_id);
/**
* Return the value of promiscuous mode for an Ethernet device.
--
2.17.1
^ permalink raw reply [relevance 3%]
* [dpdk-dev] [PATCH v2 15/15] sched: remove redundant code
@ 2019-09-09 10:05 4% ` Jasvinder Singh
1 sibling, 0 replies; 200+ results
From: Jasvinder Singh @ 2019-09-09 10:05 UTC (permalink / raw)
To: dev; +Cc: cristian.dumitrescu, Lukasz Krakowiak
Remove redundant data structure fields from port level data
structures and update release notes.
Signed-off-by: Jasvinder Singh <jasvinder.singh@intel.com>
Signed-off-by: Lukasz Krakowiak <lukaszx.krakowiak@intel.com>
---
doc/guides/rel_notes/release_19_11.rst | 6 +++-
lib/librte_sched/rte_sched.c | 43 +-------------------------
lib/librte_sched/rte_sched.h | 22 -------------
3 files changed, 6 insertions(+), 65 deletions(-)
diff --git a/doc/guides/rel_notes/release_19_11.rst b/doc/guides/rel_notes/release_19_11.rst
index 8490d897c..746aeb0ea 100644
--- a/doc/guides/rel_notes/release_19_11.rst
+++ b/doc/guides/rel_notes/release_19_11.rst
@@ -85,6 +85,10 @@ API Changes
Also, make sure to start the actual text at the margin.
=========================================================
+* sched: The pipe nodes configuration parameters such as number of pipes,
+ pipe queue sizes, pipe profiles, etc., are moved from port level structure
+ to subport level. This allows different subports of the same port to
+ have different configuration for the pipe nodes.
ABI Changes
-----------
@@ -172,7 +176,7 @@ The libraries prepended with a plus sign were incremented in this version.
librte_rcu.so.1
librte_reorder.so.1
librte_ring.so.2
- librte_sched.so.3
+ + librte_sched.so.4
librte_security.so.2
librte_stack.so.1
librte_table.so.3
diff --git a/lib/librte_sched/rte_sched.c b/lib/librte_sched/rte_sched.c
index 8a7727286..ba504da1e 100644
--- a/lib/librte_sched/rte_sched.c
+++ b/lib/librte_sched/rte_sched.c
@@ -207,10 +207,8 @@ struct rte_sched_subport {
struct rte_sched_port {
/* User parameters */
uint32_t n_subports_per_port;
- uint32_t n_pipes_per_subport;
uint32_t n_max_pipes_per_subport;
uint32_t n_max_pipes_per_subport_log2;
- uint32_t n_pipes_per_subport_log2;
uint16_t pipe_queue[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
uint8_t pipe_tc[RTE_SCHED_QUEUES_PER_PIPE];
uint8_t tc_queue[RTE_SCHED_QUEUES_PER_PIPE];
@@ -218,13 +216,7 @@ struct rte_sched_port {
uint32_t mtu;
uint32_t frame_overhead;
int socket;
- uint16_t qsize[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
- uint32_t n_pipe_profiles;
uint32_t n_max_pipe_profiles;
- uint32_t pipe_tc_be_rate_max;
-#ifdef RTE_SCHED_RED
- struct rte_red_config red_config[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE][RTE_COLORS];
-#endif
/* Timing */
uint64_t time_cpu_cycles; /* Current CPU time measured in CPU cyles */
@@ -232,48 +224,15 @@ struct rte_sched_port {
uint64_t time; /* Current NIC TX time measured in bytes */
struct rte_reciprocal inv_cycles_per_byte; /* CPU cycles per byte */
- /* Scheduling loop detection */
- uint32_t pipe_loop;
- uint32_t pipe_exhaustion;
-
- /* Bitmap */
- struct rte_bitmap *bmp;
- uint32_t grinder_base_bmp_pos[RTE_SCHED_PORT_N_GRINDERS] __rte_aligned_16;
-
/* Grinders */
- struct rte_sched_grinder grinder[RTE_SCHED_PORT_N_GRINDERS];
- uint32_t busy_grinders;
struct rte_mbuf **pkts_out;
uint32_t n_pkts_out;
uint32_t subport_id;
- /* Queue base calculation */
- uint32_t qsize_add[RTE_SCHED_QUEUES_PER_PIPE];
- uint32_t qsize_sum;
-
/* Large data structures */
- struct rte_sched_subport *subports[0];
- struct rte_sched_subport *subport;
- struct rte_sched_pipe *pipe;
- struct rte_sched_queue *queue;
- struct rte_sched_queue_extra *queue_extra;
- struct rte_sched_pipe_profile *pipe_profiles;
- uint8_t *bmp_array;
- struct rte_mbuf **queue_array;
- uint8_t memory[0] __rte_cache_aligned;
+ struct rte_sched_subport *subports[0] __rte_cache_aligned;
} __rte_cache_aligned;
-enum rte_sched_port_array {
- e_RTE_SCHED_PORT_ARRAY_SUBPORT = 0,
- e_RTE_SCHED_PORT_ARRAY_PIPE,
- e_RTE_SCHED_PORT_ARRAY_QUEUE,
- e_RTE_SCHED_PORT_ARRAY_QUEUE_EXTRA,
- e_RTE_SCHED_PORT_ARRAY_PIPE_PROFILES,
- e_RTE_SCHED_PORT_ARRAY_BMP_ARRAY,
- e_RTE_SCHED_PORT_ARRAY_QUEUE_ARRAY,
- e_RTE_SCHED_PORT_ARRAY_TOTAL,
-};
-
enum rte_sched_subport_array {
e_RTE_SCHED_SUBPORT_ARRAY_PIPE = 0,
e_RTE_SCHED_SUBPORT_ARRAY_QUEUE,
diff --git a/lib/librte_sched/rte_sched.h b/lib/librte_sched/rte_sched.h
index ea2b07448..db1b0232f 100644
--- a/lib/librte_sched/rte_sched.h
+++ b/lib/librte_sched/rte_sched.h
@@ -246,33 +246,11 @@ struct rte_sched_port_params {
/** Number of subports */
uint32_t n_subports_per_port;
- /** Number of subport_pipes */
- uint32_t n_pipes_per_subport;
-
/** Maximum number of subport_pipes */
uint32_t n_max_pipes_per_subport;
- /** Packet queue size for each traffic class.
- * All the pipes within the same subport share the similar
- * configuration for the queues.
- */
- uint16_t qsize[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
-
- /** Pipe profile table.
- * Every pipe is configured using one of the profiles from this table.
- */
- struct rte_sched_pipe_params *pipe_profiles;
-
- /** Profiles in the pipe profile table */
- uint32_t n_pipe_profiles;
-
/** Max profiles allowed in the pipe profile table */
uint32_t n_max_pipe_profiles;
-
-#ifdef RTE_SCHED_RED
- /** RED parameters */
- struct rte_red_params red_params[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE][RTE_COLORS];
-#endif
};
/*
--
2.21.0
^ permalink raw reply [relevance 4%]
* Re: [dpdk-dev] [RFC] ethdev: support hairpin queue
2019-09-06 3:08 0% ` Wu, Jingjing
@ 2019-09-08 6:44 0% ` Ori Kam
0 siblings, 0 replies; 200+ results
From: Ori Kam @ 2019-09-08 6:44 UTC (permalink / raw)
To: Wu, Jingjing, Thomas Monjalon, Yigit, Ferruh, arybchenko,
Shahaf Shuler, Slava Ovsiienko, Alex Rosenbaum
Cc: dev
Hi Jingjing,
PSB
> -----Original Message-----
> From: Wu, Jingjing <jingjing.wu@intel.com>
> Sent: Friday, September 6, 2019 6:08 AM
> To: Ori Kam <orika@mellanox.com>; Thomas Monjalon
> <thomas@monjalon.net>; Yigit, Ferruh <ferruh.yigit@intel.com>;
> arybchenko@solarflare.com; Shahaf Shuler <shahafs@mellanox.com>; Slava
> Ovsiienko <viacheslavo@mellanox.com>; Alex Rosenbaum
> <alexr@mellanox.com>
> Cc: dev@dpdk.org
> Subject: RE: [dpdk-dev] [RFC] ethdev: support hairpin queue
>
> Hi, Ori
>
> Thanks for the explanation. I have more question below.
>
> Thanks
> Jingjing
>
> > -----Original Message-----
> > From: Ori Kam [mailto:orika@mellanox.com]
> > Sent: Thursday, September 5, 2019 1:45 PM
> > To: Wu, Jingjing <jingjing.wu@intel.com>; Thomas Monjalon
> <thomas@monjalon.net>;
> > Yigit, Ferruh <ferruh.yigit@intel.com>; arybchenko@solarflare.com; Shahaf
> Shuler
> > <shahafs@mellanox.com>; Slava Ovsiienko <viacheslavo@mellanox.com>;
> Alex
> > Rosenbaum <alexr@mellanox.com>
> > Cc: dev@dpdk.org
> > Subject: RE: [dpdk-dev] [RFC] ethdev: support hairpin queue
> >
> > Hi Wu,
> > Thanks for your comments PSB,
> >
> > Ori
> >
> > > -----Original Message-----
> > > From: Wu, Jingjing <jingjing.wu@intel.com>
> > > Sent: Thursday, September 5, 2019 7:01 AM
> > > To: Ori Kam <orika@mellanox.com>; Thomas Monjalon
> > > <thomas@monjalon.net>; Yigit, Ferruh <ferruh.yigit@intel.com>;
> > > arybchenko@solarflare.com; Shahaf Shuler <shahafs@mellanox.com>;
> Slava
> > > Ovsiienko <viacheslavo@mellanox.com>; Alex Rosenbaum
> > > <alexr@mellanox.com>
> > > Cc: dev@dpdk.org
> > > Subject: RE: [dpdk-dev] [RFC] ethdev: support hairpin queue
> > >
> > >
> > > > -----Original Message-----
> > > > From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Ori Kam
> > > > Sent: Tuesday, August 13, 2019 9:38 PM
> > > > To: thomas@monjalon.net; Yigit, Ferruh <ferruh.yigit@intel.com>;
> > > > arybchenko@solarflare.com; shahafs@mellanox.com;
> > > viacheslavo@mellanox.com;
> > > > alexr@mellanox.com
> > > > Cc: dev@dpdk.org; orika@mellanox.com
> > > > Subject: [dpdk-dev] [RFC] ethdev: support hairpin queue
> > > >
> > > > This RFC replaces RFC[1].
> > > >
> > > > The hairpin feature (different name can be forward) acts as "bump on the
> > > wire",
> > > > meaning that a packet that is received from the wire can be modified
> using
> > > > offloaded action and then sent back to the wire without application
> > > intervention
> > > > which save CPU cycles.
> > > >
> > > > The hairpin is the inverse function of loopback in which application
> > > > sends a packet then it is received again by the
> > > > application without being sent to the wire.
> > > >
> > > > The hairpin can be used by a number of different NVF, for example load
> > > > balancer, gateway and so on.
> > > >
> > > > As can be seen from the hairpin description, hairpin is basically RX queue
> > > > connected to TX queue.
> > > >
> > > > During the design phase I was thinking of two ways to implement this
> > > > feature the first one is adding a new rte flow action. and the second
> > > > one is create a special kind of queue.
> > > >
> > > > The advantages of using the queue approch:
> > > > 1. More control for the application. queue depth (the memory size that
> > > > should be used).
> > > > 2. Enable QoS. QoS is normaly a parametr of queue, so in this approch it
> > > > will be easy to integrate with such system.
> > >
> > >
> > > Which kind of QoS?
> >
> > For example latency , packet rate those kinds of makes sense in the queue
> level.
> > I know we don't have any current support but I think we will have during the
> next year.
> >
> Where would be the QoS API loading? TM API? Or propose other new?
I think it will be a new API, The TM is more like limiting the bandwidth of a target flow, while in
QoS should influence more the priority between the queues.
> > >
> > > > 3. Native integression with the rte flow API. Just setting the target
> > > > queue/rss to hairpin queue, will result that the traffic will be routed
> > > > to the hairpin queue.
> > > > 4. Enable queue offloading.
> > > >
> > > Looks like the hairpin queue is just hardware queue, it has no relationship
> with
> > > host memory. It makes the queue concept a little bit confusing. And why do
> we
> > > need to setup queues, maybe some info in eth_conf is enough?
> >
> > Like stated above it makes sense to have queue related parameters.
> > For example I can think of application that most packets are going threw that
> hairpin
> > queue, but some control packets are
> > from the application. So the application can configure the QoS between those
> two
> > queues. In addtion this will enable the application
> > to use the queue like normal queue from rte_flow (see comment below) and
> every other
> > aspect.
> >
> Yes, it is typical use case. And rte_flow is used to classify to different queue?
> If I understand correct, your hairpin queue is using host memory/or on-card
> memory for buffering, but CPU cannot touch it, all the packet processing is
> done by NIC.
> Queue is created, where the queue ID is used? Tx queue ID may be used as
> action of rte_flow? I still don't understand where the hairpin Rx queue ID be
> used.
> In my opinion, if no rx/tx function, it should not be a true queue from host view.
>
Yes rte_flow is used to classify the traffic between the queues, in order to use the hairpin feature in
the basic usage, the application just insert any ingress flow that the target queue/RSS is hairpin queue.
For example assuming that queue index 4 is hairpin queue, hairpin will look something like this:
Flow create 0 ingress group 0 pattern eth / ipv4 .... / end actions decap / encap / queue index 4 / end
I understand but don't agree about your point that if there is no rx/tx function it is not a queue.
In hairpin queue we are offloading the data path. Unrelated to this RFC we are working on VDPA driver.
This is not ethdev driver but what it does is offloading the vhost and offloads the enqueue and dequeue functions[1].
> > >
> > > Not sure how your hardware make the hairpin work? Use rte_flow for
> packet
> > > modification offload? Then how does HW distribute packets to those
> hardware
> > > queue, classification? If So, why not just extend rte_flow with the hairpin
> > > action?
> > >
> >
> > You are correct, the application uses rte_flow and just points the traffic to the
> requested
> > hairpin queue/rss.
> > We could have added a new rte_flow command. The reasons we didn't:
> > 1. Like stated above some of the hairpin makes sense in queue level.
> > 2. In the near future, we will also want to support hairpin between different
> ports. This
> > makes much more
> > sense using queues.
> >
> > > > Each hairpin Rxq can be connected Txq / number of Txqs which can
> belong to
> > > a
> > > > different ports assuming the PMD supports it. The same goes the other
> > > > way each hairpin Txq can be connected to one or more Rxqs.
> > > > This is the reason that both the Txq setup and Rxq setup are getting the
> > > > hairpin configuration structure.
> > > >
> > > > From PMD prespctive the number of Rxq/Txq is the total of standard
> > > > queues + hairpin queues.
> > > >
> > > > To configure hairpin queue the user should call
> > > > rte_eth_rx_hairpin_queue_setup / rte_eth_tx_hairpin_queue_setup
> insteed
> > > > of the normal queue setup functions.
> > >
> > > If the new API introduced to avoid ABI change, would one API
> > > rte_eth_rx_hairpin_setup be enough?
> >
> > I'm not sure I understand your comment.
> > The rx_hairpin_setup was created for two main reasons:
> > 1. Avoid API change.
> > 2. I think it is more correct to use different API since the parameters are
> different.
> >
> I mean not use queue setup concept, set hairpin feature through one hairpin
> configuration API.
>
I'm not sure I understand.
API that will look something like this will be better?
Int hairpin_bind(uint16_t rx_port, uint16_t rx queue, struct hairpin_conf *rx_hairpin_conf,
uint16_t tx_port, uint16_t tx_queue, struct hairpin_conf *tx_hairpin_conf)
The problem with such API, is that it will cause issue for nics that supports one to many connections.
For example assuming that some nic can support one rx queue to 4 tx queues.
Also we still need to configure the hairpin queue. So if I understand you correctly is that the hairpin queues
will not be setup and this API will set them.
> > The reason we have both rx and tx setup functions is that we want the user to
> have
> > control binding the two queues.
> > It is most important when we will advance to hairpin between ports.
>
> Hairpin between ports? It looks like switch but not hairpin, right?
Switch from my understanding is between VM meaning traffic sent from one VM will be routed
directly to the target VM. This is not the case of hairpin. In hairpin traffic comes from the wire and goes
back to the wire. There are no VM in the system. Example application for hairpin is load balancers or gateways,
were we get for example one port is connected to one system and the second port connected to a second system.
It is the job of the application to check if the packet should pass and if so modify it, to match the second system.
For example moving VXLAN tunnel packet to MPLS tunnel in the other system.
> >
> > >
> > > Thanks
> > > Jingjing
> >
> > Thanks,
> > Ori
Thanks,
Ori
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [PATCH v6 02/10] vhost: add packed ring
@ 2019-09-06 16:42 3% ` Maxime Coquelin
0 siblings, 0 replies; 200+ results
From: Maxime Coquelin @ 2019-09-06 16:42 UTC (permalink / raw)
To: JinYu, dev
Cc: changpeng.liu, tiwei.bie, zhihong.wang, Lin Li, Xun Ni, Yu Zhang
On 8/29/19 4:12 PM, JinYu wrote:
> This patch add the packed ring in the rte_vhost_vring.
>
> Signed-off-by: Lin Li <lilin24@baidu.com>
> Signed-off-by: Xun Ni <nixun@baidu.com>
> Signed-off-by: Yu Zhang <zhangyu31@baidu.com>
> Signed-off-by: Jin Yu <jin.yu@intel.com>
> ---
> lib/librte_vhost/rte_vhost.h | 15 ++++++++++++---
> 1 file changed, 12 insertions(+), 3 deletions(-)
>
> diff --git a/lib/librte_vhost/rte_vhost.h b/lib/librte_vhost/rte_vhost.h
> index 9943575ce..7257f0965 100644
> --- a/lib/librte_vhost/rte_vhost.h
> +++ b/lib/librte_vhost/rte_vhost.h
> @@ -103,9 +103,18 @@ struct rte_vhost_memory {
> };
>
> struct rte_vhost_vring {
> - struct vring_desc *desc;
> - struct vring_avail *avail;
> - struct vring_used *used;
> + union {
> + struct vring_desc *desc;
> + struct vring_packed_desc *desc_packed;
> + };
> + union {
> + struct vring_avail *avail;
> + struct vring_packed_desc_event *driver_event;
> + };
> + union {
> + struct vring_used *used;
> + struct vring_packed_desc_event *device_event;
> + };
> uint64_t log_guest_addr;
>
> /** Deprecated, use rte_vhost_vring_call() instead. */
>
My understanding is that it does neither break the API nor ABI.
Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
Thanks,
Maxime
^ permalink raw reply [relevance 3%]
* [dpdk-dev] [PATCH 1/2] ethdev: change xstats reset function return value to int
@ 2019-09-06 14:34 4% ` Andrew Rybchenko
0 siblings, 0 replies; 200+ results
From: Andrew Rybchenko @ 2019-09-06 14:34 UTC (permalink / raw)
To: Maryam Tahhan, Reshma Pattan, Wenzhuo Lu, Jingjing Wu,
Bernard Iremonger, Neil Horman, John McNamara, Marko Kovacevic,
Thomas Monjalon, Ferruh Yigit
Cc: dev, Igor Romanov
From: Igor Romanov <igor.romanov@oktetlabs.ru>
Change rte_eth_xstats_reset() return value from void to int and
return negative errno values in case of error conditions.
Signed-off-by: Igor Romanov <igor.romanov@oktetlabs.ru>
Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
---
app/proc-info/main.c | 10 +++++++++-
app/test-pmd/config.c | 8 +++++++-
doc/guides/rel_notes/deprecation.rst | 1 -
doc/guides/rel_notes/release_19_11.rst | 3 +++
lib/librte_ethdev/rte_ethdev.c | 8 ++++----
lib/librte_ethdev/rte_ethdev.h | 7 ++++++-
6 files changed, 29 insertions(+), 8 deletions(-)
diff --git a/app/proc-info/main.c b/app/proc-info/main.c
index 34eb7a7cc4..94e808dc6e 100644
--- a/app/proc-info/main.c
+++ b/app/proc-info/main.c
@@ -580,8 +580,16 @@ nic_xstats_display(uint16_t port_id)
static void
nic_xstats_clear(uint16_t port_id)
{
+ int ret;
+
printf("\n Clearing NIC xstats for port %d\n", port_id);
- rte_eth_xstats_reset(port_id);
+ ret = rte_eth_xstats_reset(port_id);
+ if (ret != 0) {
+ printf("\n Error clearing xstats for port %d: %s\n", port_id,
+ strerror(-ret));
+ return;
+ }
+
printf("\n NIC extended statistics for port %d cleared\n", port_id);
}
diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
index 24158e5f7d..857b6dabc9 100644
--- a/app/test-pmd/config.c
+++ b/app/test-pmd/config.c
@@ -273,7 +273,13 @@ nic_xstats_display(portid_t port_id)
void
nic_xstats_clear(portid_t port_id)
{
- rte_eth_xstats_reset(port_id);
+ int ret;
+
+ ret = rte_eth_xstats_reset(port_id);
+ if (ret != 0) {
+ printf("%s: Error: failed to reset xstats (port %u): %s",
+ __func__, port_id, strerror(ret));
+ }
}
void
diff --git a/doc/guides/rel_notes/deprecation.rst b/doc/guides/rel_notes/deprecation.rst
index b2e0a1fc7c..beb9cc3b65 100644
--- a/doc/guides/rel_notes/deprecation.rst
+++ b/doc/guides/rel_notes/deprecation.rst
@@ -92,7 +92,6 @@ Deprecation Notices
- ``rte_eth_link_get`` and ``rte_eth_link_get_nowait``
- ``rte_eth_dev_stop``
- ``rte_eth_dev_close``
- - ``rte_eth_xstats_reset``
- ``rte_eth_macaddr_get``
- ``rte_eth_dev_owner_delete``
diff --git a/doc/guides/rel_notes/release_19_11.rst b/doc/guides/rel_notes/release_19_11.rst
index 5299157df7..3f492ac20d 100644
--- a/doc/guides/rel_notes/release_19_11.rst
+++ b/doc/guides/rel_notes/release_19_11.rst
@@ -101,6 +101,9 @@ API Changes
``rte_eth_promiscuous_disable`` return value from ``void`` to ``int`` to
provide a way to report various error conditions.
+* ethdev: changed ``rte_eth_dev_xstats_reset`` return value from ``void`` to
+ ``int`` to provide a way to report various error conditions.
+
ABI Changes
-----------
diff --git a/lib/librte_ethdev/rte_ethdev.c b/lib/librte_ethdev/rte_ethdev.c
index 98fe533c5a..b843bbc208 100644
--- a/lib/librte_ethdev/rte_ethdev.c
+++ b/lib/librte_ethdev/rte_ethdev.c
@@ -2518,22 +2518,22 @@ rte_eth_xstats_get(uint16_t port_id, struct rte_eth_xstat *xstats,
}
/* reset ethdev extended statistics */
-void
+int
rte_eth_xstats_reset(uint16_t port_id)
{
struct rte_eth_dev *dev;
- RTE_ETH_VALID_PORTID_OR_RET(port_id);
+ RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
dev = &rte_eth_devices[port_id];
/* implemented by the driver */
if (dev->dev_ops->xstats_reset != NULL) {
(*dev->dev_ops->xstats_reset)(dev);
- return;
+ return 0;
}
/* fallback to default */
- rte_eth_stats_reset(port_id);
+ return rte_eth_stats_reset(port_id);
}
static int
diff --git a/lib/librte_ethdev/rte_ethdev.h b/lib/librte_ethdev/rte_ethdev.h
index f07a829b29..328503d1be 100644
--- a/lib/librte_ethdev/rte_ethdev.h
+++ b/lib/librte_ethdev/rte_ethdev.h
@@ -2284,8 +2284,13 @@ int rte_eth_xstats_get_id_by_name(uint16_t port_id, const char *xstat_name,
*
* @param port_id
* The port identifier of the Ethernet device.
+ * @return
+ * - (0) if device notified to reset extended stats.
+ * - (-ENOTSUP) if pmd doesn't support both
+ * extended stats and basic stats reset.
+ * - (-ENODEV) if *port_id* invalid.
*/
-void rte_eth_xstats_reset(uint16_t port_id);
+int rte_eth_xstats_reset(uint16_t port_id);
/**
* Set a mapping for the specified transmit queue to the specified per-queue
--
2.17.1
^ permalink raw reply [relevance 4%]
* Re: [dpdk-dev] [RFC 19.11 v2 0/3] Hide DPDK internal struct from public API
@ 2019-09-06 14:00 3% ` Bruce Richardson
0 siblings, 0 replies; 200+ results
From: Bruce Richardson @ 2019-09-06 14:00 UTC (permalink / raw)
To: Marcin Zapolski; +Cc: dev, jerinj
On Fri, Sep 06, 2019 at 03:18:10PM +0200, Marcin Zapolski wrote:
> Several DPDK internal structures are exposed to direct access by user
> applications. This patch removes them from public API, and makes core DPDK
> functions that use them non-inline.
>
> v2:
> This patch set no longer makes internal DPDK functions non-inline. Instead
> it splits the rte_eth_dev structure to private and public part and modifies
> function arguments of rx and tx functions. This should bring less performance
> impact, but at the cost of needing to modify every PMD to use new rx and tx
> functions.
> For testing purposes, the ixgbe and i40e drivers are modified to acommodate for
> the changes.
>
> Marcin Zapolski (3):
> ethdev: hide key ethdev structures from public API
> i40e: make driver compatible with changes in ethdev
> ixgbe: make driver compatible with changes in ethdev
>
Thanks for testing this out Marcin. The performance impact seems lower
alright. The amount of changes needed I still am not particularly happy
about, so I'd like to propose a third option for consideration.
How about leaving the existing inline functions as they are, but also
providing the uninline functions for backward compatibility, with a
build-time switch to select between the two? Standard builds could use the
uninline versions for API/ABI compatibility, while any builds which
absolutely need the most performance can switch to using the inline
versions at the cost of compatibility. We could even make the build-switch
generic to indicate across all components a preference for absolute
performance over compatibility.
Regards,
/Bruce
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] [RFC PATCH 1/9] security: introduce CPU Crypto action type and API
@ 2019-09-06 13:27 4% ` Ananyev, Konstantin
2019-09-10 10:44 4% ` Akhil Goyal
0 siblings, 1 reply; 200+ results
From: Ananyev, Konstantin @ 2019-09-06 13:27 UTC (permalink / raw)
To: Akhil Goyal, dev; +Cc: Zhang, Roy Fan, Doherty, Declan, De Lara Guarch, Pablo
Hi Akhil,
> > This action type allows the burst of symmetric crypto workload using the same
> > algorithm, key, and direction being processed by CPU cycles synchronously.
> > This flexible action type does not require external hardware involvement,
> > having the crypto workload processed synchronously, and is more performant
> > than Cryptodev SW PMD due to the saved cycles on removed "async mode
> > simulation" as well as 3 cacheline access of the crypto ops.
>
> Does that mean application will not call the cryptodev_enqueue_burst and corresponding dequeue burst.
Yes, instead it just call rte_security_process_cpu_crypto_bulk(...)
> It would be a new API something like process_packets and it will have the crypto processed packets while returning from the API?
Yes, though the plan is that API will operate on raw data buffers, not mbufs.
>
> I still do not understand why we cannot do with the conventional crypto lib only.
> As far as I can understand, you are not doing any protocol processing or any value add
> To the crypto processing. IMO, you just need a synchronous crypto processing API which
> Can be defined in cryptodev, you don't need to re-create a crypto session in the name of
> Security session in the driver just to do a synchronous processing.
I suppose your question is why not to have rte_crypot_process_cpu_crypto_bulk(...) instead?
The main reason is that would require disruptive changes in existing cryptodev API
(would cause ABI/API breakage).
Session for RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO need some extra information
that normal crypto_sym_xform doesn't contain
(cipher offset from the start of the buffer, might be something extra in future).
Also right now there is no way to add new type of crypto_sym_session without
either breaking existing crypto-dev ABI/API or introducing new structure
(rte_crypto_sym_cpu_session or so) for that.
While rte_security is designed in a way that we can add new session types and
related parameters without causing API/ABI breakage.
BTW, what is your concern with proposed approach (via rte_security)?
From my perspective it is a lightweight change and it is totally optional
for the crypto PMDs to support it or not.
Konstantin
> >
> > AESNI-GCM and AESNI-MB PMDs are updated with this support. There is a small
> > performance test app under app/test/security_aesni_gcm(mb)_perftest to
> > prove.
> >
> > For the new API
> > The packet is sent to the crypto device for symmetric crypto
> > processing. The device will encrypt or decrypt the buffer based on the session
> > data specified and preprocessed in the security session. Different
> > than the inline or lookaside modes, when the function exits, the user will
> > expect the buffers are either processed successfully, or having the error number
> > assigned to the appropriate index of the status array.
> >
> > Will update the program's guide in the v1 patch.
> >
> > Regards,
> > Fan
> >
> > > -----Original Message-----
> > > From: Akhil Goyal [mailto:akhil.goyal@nxp.com]
> > > Sent: Wednesday, September 4, 2019 11:33 AM
> > > To: Zhang, Roy Fan <roy.fan.zhang@intel.com>; dev@dpdk.org
> > > Cc: Ananyev, Konstantin <konstantin.ananyev@intel.com>; Doherty, Declan
> > > <declan.doherty@intel.com>; De Lara Guarch, Pablo
> > > <pablo.de.lara.guarch@intel.com>
> > > Subject: RE: [RFC PATCH 1/9] security: introduce CPU Crypto action type and
> > > API
> > >
> > > Hi Fan,
> > >
> > > >
> > > > This patch introduce new RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO
> > > action
> > > > type to security library. The type represents performing crypto
> > > > operation with CPU cycles. The patch also includes a new API to
> > > > process crypto operations in bulk and the function pointers for PMDs.
> > > >
> > > I am not able to get the flow of execution for this action type. Could you
> > > please elaborate the flow in the documentation. If not in documentation
> > > right now, then please elaborate the flow in cover letter.
> > > Also I see that there are new APIs for processing crypto operations in bulk.
> > > What does that mean. How are they different from the existing APIs which
> > > are also handling bulk crypto ops depending on the budget.
> > >
> > >
> > > -Akhil
^ permalink raw reply [relevance 4%]
* [dpdk-dev] [PATCH v3 02/54] ethdev: change rte_eth_dev_info_get() return value to int
@ 2019-09-06 7:30 3% ` Andrew Rybchenko
0 siblings, 0 replies; 200+ results
From: Andrew Rybchenko @ 2019-09-06 7:30 UTC (permalink / raw)
To: Neil Horman, John McNamara, Marko Kovacevic, Thomas Monjalon,
Ferruh Yigit
Cc: dev, Ivan Ilchenko
From: Ivan Ilchenko <Ivan.Ilchenko@oktetlabs.com>
Change rte_eth_dev_info_get() return value from void to int and return
negative errno values in case of error conditions.
Modify rte_eth_dev_info_get() usage across the ethdev according
to new return type.
Signed-off-by: Ivan Ilchenko <Ivan.Ilchenko@oktetlabs.com>
Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
---
v3:
- return -1 from get_mac_addr_index() and get_hash_mac_addr_index()
- rollback dev_conf in the case of rte_eth_dev_info_get() failure
doc/guides/rel_notes/deprecation.rst | 1 -
doc/guides/rel_notes/release_19_11.rst | 5 +-
lib/librte_ethdev/rte_ethdev.c | 69 ++++++++++++++++++--------
lib/librte_ethdev/rte_ethdev.h | 6 ++-
4 files changed, 57 insertions(+), 24 deletions(-)
diff --git a/doc/guides/rel_notes/deprecation.rst b/doc/guides/rel_notes/deprecation.rst
index 0ee8533b13..cbb4c34efd 100644
--- a/doc/guides/rel_notes/deprecation.rst
+++ b/doc/guides/rel_notes/deprecation.rst
@@ -88,7 +88,6 @@ Deprecation Notices
negative errno values to indicate various error conditions (e.g.
invalid port ID, unsupported operation, failed operation):
- - ``rte_eth_dev_info_get``
- ``rte_eth_promiscuous_enable`` and ``rte_eth_promiscuous_disable``
- ``rte_eth_allmulticast_enable`` and ``rte_eth_allmulticast_disable``
- ``rte_eth_link_get`` and ``rte_eth_link_get_nowait``
diff --git a/doc/guides/rel_notes/release_19_11.rst b/doc/guides/rel_notes/release_19_11.rst
index 27cfbd9e38..152f120197 100644
--- a/doc/guides/rel_notes/release_19_11.rst
+++ b/doc/guides/rel_notes/release_19_11.rst
@@ -94,6 +94,9 @@ API Changes
Also, make sure to start the actual text at the margin.
=========================================================
+* ethdev: changed ``rte_eth_dev_infos_get`` return value from ``void`` to
+ ``int`` to provide a way to report various error conditions.
+
ABI Changes
-----------
@@ -145,7 +148,7 @@ The libraries prepended with a plus sign were incremented in this version.
librte_distributor.so.1
librte_eal.so.11
librte_efd.so.1
- librte_ethdev.so.12
+ + librte_ethdev.so.13
librte_eventdev.so.7
librte_flow_classify.so.1
librte_gro.so.1
diff --git a/lib/librte_ethdev/rte_ethdev.c b/lib/librte_ethdev/rte_ethdev.c
index 17d183e1f0..42b1d6e30a 100644
--- a/lib/librte_ethdev/rte_ethdev.c
+++ b/lib/librte_ethdev/rte_ethdev.c
@@ -1125,7 +1125,6 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
dev = &rte_eth_devices[port_id];
- RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP);
if (dev->data->dev_started) {
@@ -1144,7 +1143,9 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
*/
memcpy(&dev->data->dev_conf, dev_conf, sizeof(dev->data->dev_conf));
- rte_eth_dev_info_get(port_id, &dev_info);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ goto rollback;
/* If number of queues specified by application for both Rx and Tx is
* zero, use driver preferred values. This cannot be done individually
@@ -1406,6 +1407,7 @@ rte_eth_dev_start(uint16_t port_id)
struct rte_eth_dev *dev;
struct rte_eth_dev_info dev_info;
int diag;
+ int ret;
RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
@@ -1420,7 +1422,9 @@ rte_eth_dev_start(uint16_t port_id)
return 0;
}
- rte_eth_dev_info_get(port_id, &dev_info);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
/* Lets restore MAC now if device does not support live change */
if (*dev_info.dev_flags & RTE_ETH_DEV_NOLIVE_MAC_ADDR)
@@ -1584,7 +1588,6 @@ rte_eth_rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
return -EINVAL;
}
- RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_setup, -ENOTSUP);
/*
@@ -1592,7 +1595,10 @@ rte_eth_rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
* This value must be provided in the private data of the memory pool.
* First check that the memory pool has a valid private data.
*/
- rte_eth_dev_info_get(port_id, &dev_info);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
+
if (mp->private_data_size < sizeof(struct rte_pktmbuf_pool_private)) {
RTE_ETHDEV_LOG(ERR, "%s private_data_size %d < %d\n",
mp->name, (int)mp->private_data_size,
@@ -1703,6 +1709,7 @@ rte_eth_tx_queue_setup(uint16_t port_id, uint16_t tx_queue_id,
struct rte_eth_dev_info dev_info;
struct rte_eth_txconf local_conf;
void **txq;
+ int ret;
RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
@@ -1712,10 +1719,11 @@ rte_eth_tx_queue_setup(uint16_t port_id, uint16_t tx_queue_id,
return -EINVAL;
}
- RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_setup, -ENOTSUP);
- rte_eth_dev_info_get(port_id, &dev_info);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
/* Use default specified by driver, if nb_tx_desc is zero */
if (nb_tx_desc == 0) {
@@ -2540,7 +2548,7 @@ rte_eth_dev_fw_version_get(uint16_t port_id, char *fw_version, size_t fw_size)
fw_version, fw_size));
}
-void
+int
rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info)
{
struct rte_eth_dev *dev;
@@ -2558,7 +2566,7 @@ rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info)
*/
memset(dev_info, 0, sizeof(struct rte_eth_dev_info));
- RTE_ETH_VALID_PORTID_OR_RET(port_id);
+ RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
dev = &rte_eth_devices[port_id];
dev_info->rx_desc_lim = lim;
@@ -2567,13 +2575,15 @@ rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info)
dev_info->min_mtu = RTE_ETHER_MIN_MTU;
dev_info->max_mtu = UINT16_MAX;
- RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_infos_get);
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
(*dev->dev_ops->dev_infos_get)(dev, dev_info);
dev_info->driver_name = dev->device->driver->name;
dev_info->nb_rx_queues = dev->data->nb_rx_queues;
dev_info->nb_tx_queues = dev->data->nb_tx_queues;
dev_info->dev_flags = &dev->data->dev_flags;
+
+ return 0;
}
int
@@ -2643,7 +2653,10 @@ rte_eth_dev_set_mtu(uint16_t port_id, uint16_t mtu)
* which relies on dev->dev_ops->dev_infos_get.
*/
if (*dev->dev_ops->dev_infos_get != NULL) {
- rte_eth_dev_info_get(port_id, &dev_info);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
+
if (mtu < dev_info.min_mtu || mtu > dev_info.max_mtu)
return -EINVAL;
}
@@ -2991,10 +3004,15 @@ rte_eth_dev_rss_hash_update(uint16_t port_id,
{
struct rte_eth_dev *dev;
struct rte_eth_dev_info dev_info = { .flow_type_rss_offloads = 0, };
+ int ret;
RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
+
dev = &rte_eth_devices[port_id];
- rte_eth_dev_info_get(port_id, &dev_info);
if ((dev_info.flow_type_rss_offloads | rss_conf->rss_hf) !=
dev_info.flow_type_rss_offloads) {
RTE_ETHDEV_LOG(ERR,
@@ -3100,9 +3118,11 @@ get_mac_addr_index(uint16_t port_id, const struct rte_ether_addr *addr)
struct rte_eth_dev_info dev_info;
struct rte_eth_dev *dev = &rte_eth_devices[port_id];
unsigned i;
+ int ret;
- RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
- rte_eth_dev_info_get(port_id, &dev_info);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return -1;
for (i = 0; i < dev_info.max_mac_addrs; i++)
if (memcmp(addr, &dev->data->mac_addrs[i],
@@ -3233,8 +3253,12 @@ get_hash_mac_addr_index(uint16_t port_id, const struct rte_ether_addr *addr)
struct rte_eth_dev_info dev_info;
struct rte_eth_dev *dev = &rte_eth_devices[port_id];
unsigned i;
+ int ret;
+
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return -1;
- rte_eth_dev_info_get(port_id, &dev_info);
if (!dev->data->hash_mac_addrs)
return -1;
@@ -3319,11 +3343,15 @@ int rte_eth_set_queue_rate_limit(uint16_t port_id, uint16_t queue_idx,
struct rte_eth_dev *dev;
struct rte_eth_dev_info dev_info;
struct rte_eth_link link;
+ int ret;
RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
+
dev = &rte_eth_devices[port_id];
- rte_eth_dev_info_get(port_id, &dev_info);
link = dev->data->dev_link;
if (queue_idx > dev_info.max_tx_queues) {
@@ -4363,15 +4391,14 @@ rte_eth_dev_adjust_nb_rx_tx_desc(uint16_t port_id,
uint16_t *nb_rx_desc,
uint16_t *nb_tx_desc)
{
- struct rte_eth_dev *dev;
struct rte_eth_dev_info dev_info;
+ int ret;
RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
- dev = &rte_eth_devices[port_id];
- RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
-
- rte_eth_dev_info_get(port_id, &dev_info);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
if (nb_rx_desc != NULL)
rte_eth_dev_adjust_nb_desc(nb_rx_desc, &dev_info.rx_desc_lim);
diff --git a/lib/librte_ethdev/rte_ethdev.h b/lib/librte_ethdev/rte_ethdev.h
index d9871782e3..475dbdae17 100644
--- a/lib/librte_ethdev/rte_ethdev.h
+++ b/lib/librte_ethdev/rte_ethdev.h
@@ -2366,8 +2366,12 @@ void rte_eth_macaddr_get(uint16_t port_id, struct rte_ether_addr *mac_addr);
* @param dev_info
* A pointer to a structure of type *rte_eth_dev_info* to be filled with
* the contextual information of the Ethernet device.
+ * @return
+ * - (0) if successful.
+ * - (-ENOTSUP) if support for dev_infos_get() does not exist for the device.
+ * - (-ENODEV) if *port_id* invalid.
*/
-void rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info);
+int rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info);
/**
* Retrieve the firmware version of a device.
--
2.17.1
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] [RFC] ethdev: support hairpin queue
2019-09-05 5:44 0% ` Ori Kam
@ 2019-09-06 3:08 0% ` Wu, Jingjing
2019-09-08 6:44 0% ` Ori Kam
0 siblings, 1 reply; 200+ results
From: Wu, Jingjing @ 2019-09-06 3:08 UTC (permalink / raw)
To: Ori Kam, Thomas Monjalon, Yigit, Ferruh, arybchenko,
Shahaf Shuler, Slava Ovsiienko, Alex Rosenbaum
Cc: dev
Hi, Ori
Thanks for the explanation. I have more question below.
Thanks
Jingjing
> -----Original Message-----
> From: Ori Kam [mailto:orika@mellanox.com]
> Sent: Thursday, September 5, 2019 1:45 PM
> To: Wu, Jingjing <jingjing.wu@intel.com>; Thomas Monjalon <thomas@monjalon.net>;
> Yigit, Ferruh <ferruh.yigit@intel.com>; arybchenko@solarflare.com; Shahaf Shuler
> <shahafs@mellanox.com>; Slava Ovsiienko <viacheslavo@mellanox.com>; Alex
> Rosenbaum <alexr@mellanox.com>
> Cc: dev@dpdk.org
> Subject: RE: [dpdk-dev] [RFC] ethdev: support hairpin queue
>
> Hi Wu,
> Thanks for your comments PSB,
>
> Ori
>
> > -----Original Message-----
> > From: Wu, Jingjing <jingjing.wu@intel.com>
> > Sent: Thursday, September 5, 2019 7:01 AM
> > To: Ori Kam <orika@mellanox.com>; Thomas Monjalon
> > <thomas@monjalon.net>; Yigit, Ferruh <ferruh.yigit@intel.com>;
> > arybchenko@solarflare.com; Shahaf Shuler <shahafs@mellanox.com>; Slava
> > Ovsiienko <viacheslavo@mellanox.com>; Alex Rosenbaum
> > <alexr@mellanox.com>
> > Cc: dev@dpdk.org
> > Subject: RE: [dpdk-dev] [RFC] ethdev: support hairpin queue
> >
> >
> > > -----Original Message-----
> > > From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Ori Kam
> > > Sent: Tuesday, August 13, 2019 9:38 PM
> > > To: thomas@monjalon.net; Yigit, Ferruh <ferruh.yigit@intel.com>;
> > > arybchenko@solarflare.com; shahafs@mellanox.com;
> > viacheslavo@mellanox.com;
> > > alexr@mellanox.com
> > > Cc: dev@dpdk.org; orika@mellanox.com
> > > Subject: [dpdk-dev] [RFC] ethdev: support hairpin queue
> > >
> > > This RFC replaces RFC[1].
> > >
> > > The hairpin feature (different name can be forward) acts as "bump on the
> > wire",
> > > meaning that a packet that is received from the wire can be modified using
> > > offloaded action and then sent back to the wire without application
> > intervention
> > > which save CPU cycles.
> > >
> > > The hairpin is the inverse function of loopback in which application
> > > sends a packet then it is received again by the
> > > application without being sent to the wire.
> > >
> > > The hairpin can be used by a number of different NVF, for example load
> > > balancer, gateway and so on.
> > >
> > > As can be seen from the hairpin description, hairpin is basically RX queue
> > > connected to TX queue.
> > >
> > > During the design phase I was thinking of two ways to implement this
> > > feature the first one is adding a new rte flow action. and the second
> > > one is create a special kind of queue.
> > >
> > > The advantages of using the queue approch:
> > > 1. More control for the application. queue depth (the memory size that
> > > should be used).
> > > 2. Enable QoS. QoS is normaly a parametr of queue, so in this approch it
> > > will be easy to integrate with such system.
> >
> >
> > Which kind of QoS?
>
> For example latency , packet rate those kinds of makes sense in the queue level.
> I know we don't have any current support but I think we will have during the next year.
>
Where would be the QoS API loading? TM API? Or propose other new?
> >
> > > 3. Native integression with the rte flow API. Just setting the target
> > > queue/rss to hairpin queue, will result that the traffic will be routed
> > > to the hairpin queue.
> > > 4. Enable queue offloading.
> > >
> > Looks like the hairpin queue is just hardware queue, it has no relationship with
> > host memory. It makes the queue concept a little bit confusing. And why do we
> > need to setup queues, maybe some info in eth_conf is enough?
>
> Like stated above it makes sense to have queue related parameters.
> For example I can think of application that most packets are going threw that hairpin
> queue, but some control packets are
> from the application. So the application can configure the QoS between those two
> queues. In addtion this will enable the application
> to use the queue like normal queue from rte_flow (see comment below) and every other
> aspect.
>
Yes, it is typical use case. And rte_flow is used to classify to different queue?
If I understand correct, your hairpin queue is using host memory/or on-card memory for buffering, but CPU cannot touch it, all the packet processing is done by NIC.
Queue is created, where the queue ID is used? Tx queue ID may be used as action of rte_flow? I still don't understand where the hairpin Rx queue ID be used.
In my opinion, if no rx/tx function, it should not be a true queue from host view.
> >
> > Not sure how your hardware make the hairpin work? Use rte_flow for packet
> > modification offload? Then how does HW distribute packets to those hardware
> > queue, classification? If So, why not just extend rte_flow with the hairpin
> > action?
> >
>
> You are correct, the application uses rte_flow and just points the traffic to the requested
> hairpin queue/rss.
> We could have added a new rte_flow command. The reasons we didn't:
> 1. Like stated above some of the hairpin makes sense in queue level.
> 2. In the near future, we will also want to support hairpin between different ports. This
> makes much more
> sense using queues.
>
> > > Each hairpin Rxq can be connected Txq / number of Txqs which can belong to
> > a
> > > different ports assuming the PMD supports it. The same goes the other
> > > way each hairpin Txq can be connected to one or more Rxqs.
> > > This is the reason that both the Txq setup and Rxq setup are getting the
> > > hairpin configuration structure.
> > >
> > > From PMD prespctive the number of Rxq/Txq is the total of standard
> > > queues + hairpin queues.
> > >
> > > To configure hairpin queue the user should call
> > > rte_eth_rx_hairpin_queue_setup / rte_eth_tx_hairpin_queue_setup insteed
> > > of the normal queue setup functions.
> >
> > If the new API introduced to avoid ABI change, would one API
> > rte_eth_rx_hairpin_setup be enough?
>
> I'm not sure I understand your comment.
> The rx_hairpin_setup was created for two main reasons:
> 1. Avoid API change.
> 2. I think it is more correct to use different API since the parameters are different.
>
I mean not use queue setup concept, set hairpin feature through one hairpin configuration API.
> The reason we have both rx and tx setup functions is that we want the user to have
> control binding the two queues.
> It is most important when we will advance to hairpin between ports.
Hairpin between ports? It looks like switch but not hairpin, right?
>
> >
> > Thanks
> > Jingjing
>
> Thanks,
> Ori
^ permalink raw reply [relevance 0%]
* [dpdk-dev] [PATCH 01/13] ethdev: change promiscuous mode controllers to return errors
@ 2019-09-05 16:10 3% ` Andrew Rybchenko
2 siblings, 0 replies; 200+ results
From: Andrew Rybchenko @ 2019-09-05 16:10 UTC (permalink / raw)
To: Neil Horman, John McNamara, Marko Kovacevic, Bernard Iremonger,
Ori Kam, Bruce Richardson, Pablo de Lara, Radu Nicolau,
Akhil Goyal, Tomasz Kantecki, Harry van Haaren, Xiaoyun Li,
Thomas Monjalon, Ferruh Yigit
Cc: dev, Ivan Ilchenko
From: Ivan Ilchenko <Ivan.Ilchenko@oktetlabs.ru>
Change rte_eth_promiscuous_enable()/rte_eth_promiscuous_disable()
return value from void to int and return negative errno values
in case of error conditions.
Modify usage of these functions across the ethdev according
to new return type.
Signed-off-by: Ivan Ilchenko <Ivan.Ilchenko@oktetlabs.ru>
Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
---
doc/guides/rel_notes/deprecation.rst | 1 -
doc/guides/rel_notes/release_19_11.rst | 4 ++
doc/guides/sample_app_ug/flow_classify.rst | 6 ++-
doc/guides/sample_app_ug/flow_filtering.rst | 15 +++++-
doc/guides/sample_app_ug/rxtx_callbacks.rst | 5 +-
doc/guides/sample_app_ug/skeleton.rst | 6 ++-
lib/librte_ethdev/rte_ethdev.c | 52 ++++++++++++++++-----
lib/librte_ethdev/rte_ethdev.h | 14 +++++-
8 files changed, 80 insertions(+), 23 deletions(-)
diff --git a/doc/guides/rel_notes/deprecation.rst b/doc/guides/rel_notes/deprecation.rst
index cbb4c34efd..b2e0a1fc7c 100644
--- a/doc/guides/rel_notes/deprecation.rst
+++ b/doc/guides/rel_notes/deprecation.rst
@@ -88,7 +88,6 @@ Deprecation Notices
negative errno values to indicate various error conditions (e.g.
invalid port ID, unsupported operation, failed operation):
- - ``rte_eth_promiscuous_enable`` and ``rte_eth_promiscuous_disable``
- ``rte_eth_allmulticast_enable`` and ``rte_eth_allmulticast_disable``
- ``rte_eth_link_get`` and ``rte_eth_link_get_nowait``
- ``rte_eth_dev_stop``
diff --git a/doc/guides/rel_notes/release_19_11.rst b/doc/guides/rel_notes/release_19_11.rst
index 152f120197..5299157df7 100644
--- a/doc/guides/rel_notes/release_19_11.rst
+++ b/doc/guides/rel_notes/release_19_11.rst
@@ -97,6 +97,10 @@ API Changes
* ethdev: changed ``rte_eth_dev_infos_get`` return value from ``void`` to
``int`` to provide a way to report various error conditions.
+* ethdev: changed ``rte_eth_promiscuous_enable`` and
+ ``rte_eth_promiscuous_disable`` return value from ``void`` to ``int`` to
+ provide a way to report various error conditions.
+
ABI Changes
-----------
diff --git a/doc/guides/sample_app_ug/flow_classify.rst b/doc/guides/sample_app_ug/flow_classify.rst
index 96a5c66d0f..7c2b6dcf83 100644
--- a/doc/guides/sample_app_ug/flow_classify.rst
+++ b/doc/guides/sample_app_ug/flow_classify.rst
@@ -315,7 +315,9 @@ Forwarding application is shown below:
addr.addr_bytes[4], addr.addr_bytes[5]);
/* Enable RX in promiscuous mode for the Ethernet device. */
- rte_eth_promiscuous_enable(port);
+ retval = rte_eth_promiscuous_enable(port);
+ if (retval != 0)
+ return retval;
return 0;
}
@@ -343,7 +345,7 @@ Finally the RX port is set in promiscuous mode:
.. code-block:: c
- rte_eth_promiscuous_enable(port);
+ retval = rte_eth_promiscuous_enable(port);
The Add Rules function
~~~~~~~~~~~~~~~~~~~~~~
diff --git a/doc/guides/sample_app_ug/flow_filtering.rst b/doc/guides/sample_app_ug/flow_filtering.rst
index 02fc675506..de3e4ab0b6 100644
--- a/doc/guides/sample_app_ug/flow_filtering.rst
+++ b/doc/guides/sample_app_ug/flow_filtering.rst
@@ -193,7 +193,13 @@ application is shown below:
}
}
- rte_eth_promiscuous_enable(port_id);
+ ret = rte_eth_promiscuous_enable(port_id);
+ if (ret != 0) {
+ rte_exit(EXIT_FAILURE,
+ ":: cannot enable promiscuous mode: err=%d, port=%u\n",
+ ret, port_id);
+ }
+
ret = rte_eth_dev_start(port_id);
if (ret < 0) {
rte_exit(EXIT_FAILURE,
@@ -278,7 +284,12 @@ We are setting the RX port to promiscuous mode:
.. code-block:: c
- rte_eth_promiscuous_enable(port_id);
+ ret = rte_eth_promiscuous_enable(port_id);
+ if (ret != 0) {
+ rte_exit(EXIT_FAILURE,
+ ":: cannot enable promiscuous mode: err=%d, port=%u\n",
+ ret, port_id);
+ }
The last step is to start the port.
diff --git a/doc/guides/sample_app_ug/rxtx_callbacks.rst b/doc/guides/sample_app_ug/rxtx_callbacks.rst
index 32c120992f..0a69ec71ab 100644
--- a/doc/guides/sample_app_ug/rxtx_callbacks.rst
+++ b/doc/guides/sample_app_ug/rxtx_callbacks.rst
@@ -117,8 +117,9 @@ comments:
return retval;
/* Enable RX in promiscuous mode for the Ethernet device. */
- rte_eth_promiscuous_enable(port);
-
+ retval = rte_eth_promiscuous_enable(port);
+ if (retval != 0)
+ return retval;
/* Add the callbacks for RX and TX.*/
rte_eth_add_rx_callback(port, 0, add_timestamps, NULL);
diff --git a/doc/guides/sample_app_ug/skeleton.rst b/doc/guides/sample_app_ug/skeleton.rst
index 59ca511d33..1d0a2760d4 100644
--- a/doc/guides/sample_app_ug/skeleton.rst
+++ b/doc/guides/sample_app_ug/skeleton.rst
@@ -149,7 +149,9 @@ Forwarding application is shown below:
return retval;
/* Enable RX in promiscuous mode for the Ethernet device. */
- rte_eth_promiscuous_enable(port);
+ retval = rte_eth_promiscuous_enable(port);
+ if (retval != 0)
+ return retval;
return 0;
}
@@ -177,7 +179,7 @@ Finally the RX port is set in promiscuous mode:
.. code-block:: c
- rte_eth_promiscuous_enable(port);
+ retval = rte_eth_promiscuous_enable(port);
The Lcores Main
diff --git a/lib/librte_ethdev/rte_ethdev.c b/lib/librte_ethdev/rte_ethdev.c
index 4b6cbe2343..0f6dedbe23 100644
--- a/lib/librte_ethdev/rte_ethdev.c
+++ b/lib/librte_ethdev/rte_ethdev.c
@@ -1381,24 +1381,41 @@ rte_eth_dev_mac_restore(struct rte_eth_dev *dev,
}
}
-static void
+static int
rte_eth_dev_config_restore(struct rte_eth_dev *dev,
struct rte_eth_dev_info *dev_info, uint16_t port_id)
{
+ int ret;
+
if (!(*dev_info->dev_flags & RTE_ETH_DEV_NOLIVE_MAC_ADDR))
rte_eth_dev_mac_restore(dev, dev_info);
/* replay promiscuous configuration */
- if (rte_eth_promiscuous_get(port_id) == 1)
- rte_eth_promiscuous_enable(port_id);
- else if (rte_eth_promiscuous_get(port_id) == 0)
- rte_eth_promiscuous_disable(port_id);
+ if (rte_eth_promiscuous_get(port_id) == 1) {
+ ret = rte_eth_promiscuous_enable(port_id);
+ if (ret != 0 && ret != -ENOTSUP) {
+ RTE_ETHDEV_LOG(ERR,
+ "Failed to enable promiscuous mode for device (port %u): %s\n",
+ port_id, rte_strerror(-ret));
+ return ret;
+ }
+ } else if (rte_eth_promiscuous_get(port_id) == 0) {
+ ret = rte_eth_promiscuous_disable(port_id);
+ if (ret != 0 && ret != -ENOTSUP) {
+ RTE_ETHDEV_LOG(ERR,
+ "Failed to disable promiscuous mode for device (port %u): %s\n",
+ port_id, rte_strerror(-ret));
+ return ret;
+ }
+ }
/* replay all multicast configuration */
if (rte_eth_allmulticast_get(port_id) == 1)
rte_eth_allmulticast_enable(port_id);
else if (rte_eth_allmulticast_get(port_id) == 0)
rte_eth_allmulticast_disable(port_id);
+
+ return 0;
}
int
@@ -1436,7 +1453,14 @@ rte_eth_dev_start(uint16_t port_id)
else
return eth_err(port_id, diag);
- rte_eth_dev_config_restore(dev, &dev_info, port_id);
+ ret = rte_eth_dev_config_restore(dev, &dev_info, port_id);
+ if (ret != 0) {
+ RTE_ETHDEV_LOG(ERR,
+ "Error during restoring configuration for device (port %u): %s\n",
+ port_id, rte_strerror(-ret));
+ rte_eth_dev_stop(port_id);
+ return ret;
+ }
if (dev->data->dev_conf.intr_conf.lsc == 0) {
RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->link_update, -ENOTSUP);
@@ -1864,30 +1888,34 @@ rte_eth_tx_done_cleanup(uint16_t port_id, uint16_t queue_id, uint32_t free_cnt)
return eth_err(port_id, ret);
}
-void
+int
rte_eth_promiscuous_enable(uint16_t port_id)
{
struct rte_eth_dev *dev;
- RTE_ETH_VALID_PORTID_OR_RET(port_id);
+ RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
dev = &rte_eth_devices[port_id];
- RTE_FUNC_PTR_OR_RET(*dev->dev_ops->promiscuous_enable);
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->promiscuous_enable, -ENOTSUP);
(*dev->dev_ops->promiscuous_enable)(dev);
dev->data->promiscuous = 1;
+
+ return 0;
}
-void
+int
rte_eth_promiscuous_disable(uint16_t port_id)
{
struct rte_eth_dev *dev;
- RTE_ETH_VALID_PORTID_OR_RET(port_id);
+ RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
dev = &rte_eth_devices[port_id];
- RTE_FUNC_PTR_OR_RET(*dev->dev_ops->promiscuous_disable);
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->promiscuous_disable, -ENOTSUP);
dev->data->promiscuous = 0;
(*dev->dev_ops->promiscuous_disable)(dev);
+
+ return 0;
}
int
diff --git a/lib/librte_ethdev/rte_ethdev.h b/lib/librte_ethdev/rte_ethdev.h
index eda9e5c628..56e47b96be 100644
--- a/lib/librte_ethdev/rte_ethdev.h
+++ b/lib/librte_ethdev/rte_ethdev.h
@@ -2022,16 +2022,26 @@ int rte_eth_dev_reset(uint16_t port_id);
*
* @param port_id
* The port identifier of the Ethernet device.
+ * @return
+ * - (0) if successful.
+ * - (-ENOTSUP) if support for promiscuous_enable() does not exist
+ * for the device.
+ * - (-ENODEV) if *port_id* invalid.
*/
-void rte_eth_promiscuous_enable(uint16_t port_id);
+int rte_eth_promiscuous_enable(uint16_t port_id);
/**
* Disable receipt in promiscuous mode for an Ethernet device.
*
* @param port_id
* The port identifier of the Ethernet device.
+ * @return
+ * - (0) if successful.
+ * - (-ENOTSUP) if support for promiscuous_disable() does not exist
+ * for the device.
+ * - (-ENODEV) if *port_id* invalid.
*/
-void rte_eth_promiscuous_disable(uint16_t port_id);
+int rte_eth_promiscuous_disable(uint16_t port_id);
/**
* Return the value of promiscuous mode for an Ethernet device.
--
2.17.1
^ permalink raw reply [relevance 3%]
* [dpdk-dev] [PATCH 1/2] version: 19.11-rc0
@ 2019-09-05 15:47 6% agupta3
0 siblings, 0 replies; 200+ results
From: agupta3 @ 2019-09-05 15:47 UTC (permalink / raw)
To: John McNamara, Marko Kovacevic; +Cc: dev, David Marchand
From: David Marchand <david.marchand@redhat.com>
Start a new release cycle with empty release notes.
Signed-off-by: David Marchand <david.marchand@redhat.com>
Acked-by: Thomas Monjalon <thomas@monjalon.net>
---
VERSION | 2 +-
doc/guides/rel_notes/index.rst | 1 +
doc/guides/rel_notes/release_19_11.rst | 216 +++++++++++++++++++++++++++++++++
3 files changed, 218 insertions(+), 1 deletion(-)
create mode 100644 doc/guides/rel_notes/release_19_11.rst
diff --git a/VERSION b/VERSION
index 909cfd6..fff18fc 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-19.08.0
+19.11.0-rc0
diff --git a/doc/guides/rel_notes/index.rst b/doc/guides/rel_notes/index.rst
index adfaf12..26f4a97 100644
--- a/doc/guides/rel_notes/index.rst
+++ b/doc/guides/rel_notes/index.rst
@@ -8,6 +8,7 @@ Release Notes
:maxdepth: 1
:numbered:
+ release_19_11
release_19_08
release_19_05
release_19_02
diff --git a/doc/guides/rel_notes/release_19_11.rst b/doc/guides/rel_notes/release_19_11.rst
new file mode 100644
index 0000000..8490d89
--- /dev/null
+++ b/doc/guides/rel_notes/release_19_11.rst
@@ -0,0 +1,216 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+ Copyright 2019 The DPDK contributors
+
+.. include:: <isonum.txt>
+
+DPDK Release 19.11
+==================
+
+.. **Read this first.**
+
+ The text in the sections below explains how to update the release notes.
+
+ Use proper spelling, capitalization and punctuation in all sections.
+
+ Variable and config names should be quoted as fixed width text:
+ ``LIKE_THIS``.
+
+ Build the docs and view the output file to ensure the changes are correct::
+
+ make doc-guides-html
+
+ xdg-open build/doc/html/guides/rel_notes/release_19_11.html
+
+
+New Features
+------------
+
+.. This section should contain new features added in this release.
+ Sample format:
+
+ * **Add a title in the past tense with a full stop.**
+
+ Add a short 1-2 sentence description in the past tense.
+ The description should be enough to allow someone scanning
+ the release notes to understand the new feature.
+
+ If the feature adds a lot of sub-features you can use a bullet list
+ like this:
+
+ * Added feature foo to do something.
+ * Enhanced feature bar to do something else.
+
+ Refer to the previous release notes for examples.
+
+ Suggested order in release notes items:
+ * Core libs (EAL, mempool, ring, mbuf, buses)
+ * Device abstraction libs and PMDs
+ - ethdev (lib, PMDs)
+ - cryptodev (lib, PMDs)
+ - eventdev (lib, PMDs)
+ - etc
+ * Other libs
+ * Apps, Examples, Tools (if significant)
+
+ This section is a comment. Do not overwrite or remove it.
+ Also, make sure to start the actual text at the margin.
+ =========================================================
+
+
+Removed Items
+-------------
+
+.. This section should contain removed items in this release. Sample format:
+
+ * Add a short 1-2 sentence description of the removed item
+ in the past tense.
+
+ This section is a comment. Do not overwrite or remove it.
+ Also, make sure to start the actual text at the margin.
+ =========================================================
+
+
+API Changes
+-----------
+
+.. This section should contain API changes. Sample format:
+
+ * sample: Add a short 1-2 sentence description of the API change
+ which was announced in the previous releases and made in this release.
+ Start with a scope label like "ethdev:".
+ Use fixed width quotes for ``function_names`` or ``struct_names``.
+ Use the past tense.
+
+ This section is a comment. Do not overwrite or remove it.
+ Also, make sure to start the actual text at the margin.
+ =========================================================
+
+
+ABI Changes
+-----------
+
+.. This section should contain ABI changes. Sample format:
+
+ * sample: Add a short 1-2 sentence description of the ABI change
+ which was announced in the previous releases and made in this release.
+ Start with a scope label like "ethdev:".
+ Use fixed width quotes for ``function_names`` or ``struct_names``.
+ Use the past tense.
+
+ This section is a comment. Do not overwrite or remove it.
+ Also, make sure to start the actual text at the margin.
+ =========================================================
+
+
+Shared Library Versions
+-----------------------
+
+.. Update any library version updated in this release
+ and prepend with a ``+`` sign, like this:
+
+ libfoo.so.1
+ + libupdated.so.2
+ libbar.so.1
+
+ This section is a comment. Do not overwrite or remove it.
+ =========================================================
+
+The libraries prepended with a plus sign were incremented in this version.
+
+.. code-block:: diff
+
+ librte_acl.so.2
+ librte_bbdev.so.1
+ librte_bitratestats.so.2
+ librte_bpf.so.1
+ librte_bus_dpaa.so.2
+ librte_bus_fslmc.so.2
+ librte_bus_ifpga.so.2
+ librte_bus_pci.so.2
+ librte_bus_vdev.so.2
+ librte_bus_vmbus.so.2
+ librte_cfgfile.so.2
+ librte_cmdline.so.2
+ librte_compressdev.so.1
+ librte_cryptodev.so.8
+ librte_distributor.so.1
+ librte_eal.so.11
+ librte_efd.so.1
+ librte_ethdev.so.12
+ librte_eventdev.so.7
+ librte_flow_classify.so.1
+ librte_gro.so.1
+ librte_gso.so.1
+ librte_hash.so.2
+ librte_ip_frag.so.1
+ librte_ipsec.so.1
+ librte_jobstats.so.1
+ librte_kni.so.2
+ librte_kvargs.so.1
+ librte_latencystats.so.1
+ librte_lpm.so.2
+ librte_mbuf.so.5
+ librte_member.so.1
+ librte_mempool.so.5
+ librte_meter.so.3
+ librte_metrics.so.1
+ librte_net.so.1
+ librte_pci.so.1
+ librte_pdump.so.3
+ librte_pipeline.so.3
+ librte_pmd_bnxt.so.2
+ librte_pmd_bond.so.2
+ librte_pmd_i40e.so.2
+ librte_pmd_ixgbe.so.2
+ librte_pmd_dpaa2_qdma.so.1
+ librte_pmd_ring.so.2
+ librte_pmd_softnic.so.1
+ librte_pmd_vhost.so.2
+ librte_port.so.3
+ librte_power.so.1
+ librte_rawdev.so.1
+ librte_rcu.so.1
+ librte_reorder.so.1
+ librte_ring.so.2
+ librte_sched.so.3
+ librte_security.so.2
+ librte_stack.so.1
+ librte_table.so.3
+ librte_timer.so.1
+ librte_vhost.so.4
+
+
+Known Issues
+------------
+
+.. This section should contain new known issues in this release. Sample format:
+
+ * **Add title in present tense with full stop.**
+
+ Add a short 1-2 sentence description of the known issue
+ in the present tense. Add information on any known workarounds.
+
+ This section is a comment. Do not overwrite or remove it.
+ Also, make sure to start the actual text at the margin.
+ =========================================================
+
+
+Tested Platforms
+----------------
+
+.. This section should contain a list of platforms that were tested
+ with this release.
+
+ The format is:
+
+ * <vendor> platform with <vendor> <type of devices> combinations
+
+ * List of CPU
+ * List of OS
+ * List of devices
+ * Other relevant details...
+
+ This section is a comment. Do not overwrite or remove it.
+ Also, make sure to start the actual text at the margin.
+ =========================================================
+
--
1.8.3.1
^ permalink raw reply [relevance 6%]
* Re: [dpdk-dev] [RFC] ethdev: support hairpin queue
2019-09-05 4:00 3% ` Wu, Jingjing
@ 2019-09-05 5:44 0% ` Ori Kam
2019-09-06 3:08 0% ` Wu, Jingjing
0 siblings, 1 reply; 200+ results
From: Ori Kam @ 2019-09-05 5:44 UTC (permalink / raw)
To: Wu, Jingjing, Thomas Monjalon, Yigit, Ferruh, arybchenko,
Shahaf Shuler, Slava Ovsiienko, Alex Rosenbaum
Cc: dev
Hi Wu,
Thanks for your comments PSB,
Ori
> -----Original Message-----
> From: Wu, Jingjing <jingjing.wu@intel.com>
> Sent: Thursday, September 5, 2019 7:01 AM
> To: Ori Kam <orika@mellanox.com>; Thomas Monjalon
> <thomas@monjalon.net>; Yigit, Ferruh <ferruh.yigit@intel.com>;
> arybchenko@solarflare.com; Shahaf Shuler <shahafs@mellanox.com>; Slava
> Ovsiienko <viacheslavo@mellanox.com>; Alex Rosenbaum
> <alexr@mellanox.com>
> Cc: dev@dpdk.org
> Subject: RE: [dpdk-dev] [RFC] ethdev: support hairpin queue
>
>
> > -----Original Message-----
> > From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Ori Kam
> > Sent: Tuesday, August 13, 2019 9:38 PM
> > To: thomas@monjalon.net; Yigit, Ferruh <ferruh.yigit@intel.com>;
> > arybchenko@solarflare.com; shahafs@mellanox.com;
> viacheslavo@mellanox.com;
> > alexr@mellanox.com
> > Cc: dev@dpdk.org; orika@mellanox.com
> > Subject: [dpdk-dev] [RFC] ethdev: support hairpin queue
> >
> > This RFC replaces RFC[1].
> >
> > The hairpin feature (different name can be forward) acts as "bump on the
> wire",
> > meaning that a packet that is received from the wire can be modified using
> > offloaded action and then sent back to the wire without application
> intervention
> > which save CPU cycles.
> >
> > The hairpin is the inverse function of loopback in which application
> > sends a packet then it is received again by the
> > application without being sent to the wire.
> >
> > The hairpin can be used by a number of different NVF, for example load
> > balancer, gateway and so on.
> >
> > As can be seen from the hairpin description, hairpin is basically RX queue
> > connected to TX queue.
> >
> > During the design phase I was thinking of two ways to implement this
> > feature the first one is adding a new rte flow action. and the second
> > one is create a special kind of queue.
> >
> > The advantages of using the queue approch:
> > 1. More control for the application. queue depth (the memory size that
> > should be used).
> > 2. Enable QoS. QoS is normaly a parametr of queue, so in this approch it
> > will be easy to integrate with such system.
>
>
> Which kind of QoS?
For example latency , packet rate those kinds of makes sense in the queue level.
I know we don't have any current support but I think we will have during the next year.
>
> > 3. Native integression with the rte flow API. Just setting the target
> > queue/rss to hairpin queue, will result that the traffic will be routed
> > to the hairpin queue.
> > 4. Enable queue offloading.
> >
> Looks like the hairpin queue is just hardware queue, it has no relationship with
> host memory. It makes the queue concept a little bit confusing. And why do we
> need to setup queues, maybe some info in eth_conf is enough?
Like stated above it makes sense to have queue related parameters.
For example I can think of application that most packets are going threw that hairpin queue, but some control packets are
from the application. So the application can configure the QoS between those two queues. In addtion this will enable the application
to use the queue like normal queue from rte_flow (see comment below) and every other aspect.
>
> Not sure how your hardware make the hairpin work? Use rte_flow for packet
> modification offload? Then how does HW distribute packets to those hardware
> queue, classification? If So, why not just extend rte_flow with the hairpin
> action?
>
You are correct, the application uses rte_flow and just points the traffic to the requested hairpin queue/rss.
We could have added a new rte_flow command. The reasons we didn't:
1. Like stated above some of the hairpin makes sense in queue level.
2. In the near future, we will also want to support hairpin between different ports. This makes much more
sense using queues.
> > Each hairpin Rxq can be connected Txq / number of Txqs which can belong to
> a
> > different ports assuming the PMD supports it. The same goes the other
> > way each hairpin Txq can be connected to one or more Rxqs.
> > This is the reason that both the Txq setup and Rxq setup are getting the
> > hairpin configuration structure.
> >
> > From PMD prespctive the number of Rxq/Txq is the total of standard
> > queues + hairpin queues.
> >
> > To configure hairpin queue the user should call
> > rte_eth_rx_hairpin_queue_setup / rte_eth_tx_hairpin_queue_setup insteed
> > of the normal queue setup functions.
>
> If the new API introduced to avoid ABI change, would one API
> rte_eth_rx_hairpin_setup be enough?
I'm not sure I understand your comment.
The rx_hairpin_setup was created for two main reasons:
1. Avoid API change.
2. I think it is more correct to use different API since the parameters are different.
The reason we have both rx and tx setup functions is that we want the user to have control binding the two queues.
It is most important when we will advance to hairpin between ports.
>
> Thanks
> Jingjing
Thanks,
Ori
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [RFC] ethdev: support hairpin queue
@ 2019-09-05 4:00 3% ` Wu, Jingjing
2019-09-05 5:44 0% ` Ori Kam
0 siblings, 1 reply; 200+ results
From: Wu, Jingjing @ 2019-09-05 4:00 UTC (permalink / raw)
To: Ori Kam, thomas, Yigit, Ferruh, arybchenko, shahafs, viacheslavo, alexr
Cc: dev
> -----Original Message-----
> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Ori Kam
> Sent: Tuesday, August 13, 2019 9:38 PM
> To: thomas@monjalon.net; Yigit, Ferruh <ferruh.yigit@intel.com>;
> arybchenko@solarflare.com; shahafs@mellanox.com; viacheslavo@mellanox.com;
> alexr@mellanox.com
> Cc: dev@dpdk.org; orika@mellanox.com
> Subject: [dpdk-dev] [RFC] ethdev: support hairpin queue
>
> This RFC replaces RFC[1].
>
> The hairpin feature (different name can be forward) acts as "bump on the wire",
> meaning that a packet that is received from the wire can be modified using
> offloaded action and then sent back to the wire without application intervention
> which save CPU cycles.
>
> The hairpin is the inverse function of loopback in which application
> sends a packet then it is received again by the
> application without being sent to the wire.
>
> The hairpin can be used by a number of different NVF, for example load
> balancer, gateway and so on.
>
> As can be seen from the hairpin description, hairpin is basically RX queue
> connected to TX queue.
>
> During the design phase I was thinking of two ways to implement this
> feature the first one is adding a new rte flow action. and the second
> one is create a special kind of queue.
>
> The advantages of using the queue approch:
> 1. More control for the application. queue depth (the memory size that
> should be used).
> 2. Enable QoS. QoS is normaly a parametr of queue, so in this approch it
> will be easy to integrate with such system.
Which kind of QoS?
> 3. Native integression with the rte flow API. Just setting the target
> queue/rss to hairpin queue, will result that the traffic will be routed
> to the hairpin queue.
> 4. Enable queue offloading.
>
Looks like the hairpin queue is just hardware queue, it has no relationship with host memory. It makes the queue concept a little bit confusing. And why do we need to setup queues, maybe some info in eth_conf is enough?
Not sure how your hardware make the hairpin work? Use rte_flow for packet modification offload? Then how does HW distribute packets to those hardware queue, classification? If So, why not just extend rte_flow with the hairpin action?
> Each hairpin Rxq can be connected Txq / number of Txqs which can belong to a
> different ports assuming the PMD supports it. The same goes the other
> way each hairpin Txq can be connected to one or more Rxqs.
> This is the reason that both the Txq setup and Rxq setup are getting the
> hairpin configuration structure.
>
> From PMD prespctive the number of Rxq/Txq is the total of standard
> queues + hairpin queues.
>
> To configure hairpin queue the user should call
> rte_eth_rx_hairpin_queue_setup / rte_eth_tx_hairpin_queue_setup insteed
> of the normal queue setup functions.
If the new API introduced to avoid ABI change, would one API rte_eth_rx_hairpin_setup be enough?
Thanks
Jingjing
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] [PATCH 02/11] log: define logtype register wrapper for drivers
2019-09-03 8:47 3% ` Ferruh Yigit
@ 2019-09-04 17:45 0% ` Thomas Monjalon
0 siblings, 0 replies; 200+ results
From: Thomas Monjalon @ 2019-09-04 17:45 UTC (permalink / raw)
To: Ferruh Yigit, David Marchand; +Cc: dev
03/09/2019 10:47, Ferruh Yigit:
> On 9/3/2019 9:06 AM, David Marchand wrote:
> > On Mon, Sep 2, 2019 at 4:29 PM Ferruh Yigit <ferruh.yigit@intel.com> wrote:
> >> On 8/19/2019 12:41 PM, David Marchand wrote:
> >>> The function rte_log_register_type_and_pick_level() fills a gap for
> >>> dynamically loaded code (especially drivers) who would not pick up
> >>> the log level passed at startup.
> >>>
> >>> Let's promote it to stable and export it for use by drivers via
> >>> a wrapper.
> >>>
> >>> Signed-off-by: David Marchand <david.marchand@redhat.com>
> >>> ---
[...]
> >>> /**
> >>> - * @warning
> >>> - * @b EXPERIMENTAL: this API may change without prior notice
> >>> - *
> >>> * Register a dynamic log type and try to pick its level from EAL options
[...]
> >>> -__rte_experimental
> >>> int rte_log_register_type_and_pick_level(const char *name, uint32_t level_def);
> >>
> >> +1 to remove experimental from the API.
I am not sure about this function API.
Why we combined register and level setting in one function?
> >>> +#define RTE_LOG_REGISTER(token, name, level, fallback) \
You really need to document this macro with doxygen.
> >>> +{ \
> >>> + token = rte_log_register_type_and_pick_level(name, level); \
> >>> + if (token < 0) \
> >>
> >> The failure can be because component can try to register existing log name, or
> >> there is no enough memory, do you think does it worth to do log, or some
> >> additional work if component is trying to register existing log name?
[...]
> >>> + token = fallback; \
> >>
> >> Does the 'fallback' needs to be provided by user, it looks like everyone will
> >> just copy/paste 'RTE_LOGTYPE_PMD' for drivers, and does it really needs to be
> >> configurable since it is fallback.
> >
> > This series only touches drivers, but I expected other components
> > would use this macro later.
> > I can add a RTE_PMD_REGISTER_LOG macro that hides the RTE_LOGTYPE_PMD
> > fallback value.
I agree we don't need to configure the fallback log.
If there is an error during log setup,
we can log everything next (at debug level).
Let's make fallback hardcoded.
> >> Why not provide a hardcoded type for the failure case? And for that case perhaps
> >> create a more generic logtype, something like "RTE_LOGTYPE_FALLBACK" so that it
> >> can be as it is from all components?
> >
> > I prefer to map all drivers to a logtype that means something, like
> > RTE_LOGTYPE_PMD.
>
> In that manner it make sense agreed, but previously drivers were using
> 'RTE_LOGTYPE_PMD' instead of having their own log types, Stephen did some work
> to replace the 'RTE_LOGTYPE_PMD' so that it can be deprecated,
>
> starting to use it again as fallback may lead drivers using it again as log type
> in their drivers, may they wouldn't but this is what I concern. Something with
> name 'RTE_LOGTYPE_FALLBACK' clear to not use as default logtype in drivers.
>
> > Having a "fallback" could be used for all components, but this would
> > have to be a static logtype and adding one is not possible without
> > breaking the abi (static entries are < 32 and all values are used).
RTE_LOGTYPE_PMD can be renamed to RTE_LOGTYPE_FALLBACK.
> There is a gap between 'RTE_LOGTYPE_GSO' & 'RTE_LOGTYPE_USER1' ...
Yes, there is room here. But I prefer to rename and re-use
RTE_LOGTYPE_PMD which is not used anymore.
It is part of the EAL API but it is not supposed to be used externally.
For out-of-tree PMDs, we are not supposed to provide such compat.
So I would say don't care with deprecation here.
^ permalink raw reply [relevance 0%]
* [dpdk-dev] [PATCH v2 02/54] ethdev: change rte_eth_dev_info_get() return value to int
@ 2019-09-03 13:56 3% ` Andrew Rybchenko
0 siblings, 0 replies; 200+ results
From: Andrew Rybchenko @ 2019-09-03 13:56 UTC (permalink / raw)
To: Neil Horman, John McNamara, Marko Kovacevic, Thomas Monjalon,
Ferruh Yigit
Cc: dev, Ivan Ilchenko
From: Ivan Ilchenko <Ivan.Ilchenko@oktetlabs.com>
Change rte_eth_dev_info_get() return value from void to int and return
negative errno values in case of error conditions.
Modify rte_eth_dev_info_get() usage across the ethdev according
to new return type.
Signed-off-by: Ivan Ilchenko <Ivan.Ilchenko@oktetlabs.com>
Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
---
doc/guides/rel_notes/deprecation.rst | 1 -
doc/guides/rel_notes/release_19_11.rst | 5 ++-
lib/librte_ethdev/rte_ethdev.c | 71 ++++++++++++++++++++++++----------
lib/librte_ethdev/rte_ethdev.h | 6 ++-
4 files changed, 60 insertions(+), 23 deletions(-)
diff --git a/doc/guides/rel_notes/deprecation.rst b/doc/guides/rel_notes/deprecation.rst
index 0ee8533..cbb4c34 100644
--- a/doc/guides/rel_notes/deprecation.rst
+++ b/doc/guides/rel_notes/deprecation.rst
@@ -88,7 +88,6 @@ Deprecation Notices
negative errno values to indicate various error conditions (e.g.
invalid port ID, unsupported operation, failed operation):
- - ``rte_eth_dev_info_get``
- ``rte_eth_promiscuous_enable`` and ``rte_eth_promiscuous_disable``
- ``rte_eth_allmulticast_enable`` and ``rte_eth_allmulticast_disable``
- ``rte_eth_link_get`` and ``rte_eth_link_get_nowait``
diff --git a/doc/guides/rel_notes/release_19_11.rst b/doc/guides/rel_notes/release_19_11.rst
index 27cfbd9..152f120 100644
--- a/doc/guides/rel_notes/release_19_11.rst
+++ b/doc/guides/rel_notes/release_19_11.rst
@@ -94,6 +94,9 @@ API Changes
Also, make sure to start the actual text at the margin.
=========================================================
+* ethdev: changed ``rte_eth_dev_infos_get`` return value from ``void`` to
+ ``int`` to provide a way to report various error conditions.
+
ABI Changes
-----------
@@ -145,7 +148,7 @@ The libraries prepended with a plus sign were incremented in this version.
librte_distributor.so.1
librte_eal.so.11
librte_efd.so.1
- librte_ethdev.so.12
+ + librte_ethdev.so.13
librte_eventdev.so.7
librte_flow_classify.so.1
librte_gro.so.1
diff --git a/lib/librte_ethdev/rte_ethdev.c b/lib/librte_ethdev/rte_ethdev.c
index 17d183e..4b6cbe2 100644
--- a/lib/librte_ethdev/rte_ethdev.c
+++ b/lib/librte_ethdev/rte_ethdev.c
@@ -1125,7 +1125,6 @@ struct rte_eth_dev *
dev = &rte_eth_devices[port_id];
- RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP);
if (dev->data->dev_started) {
@@ -1144,7 +1143,9 @@ struct rte_eth_dev *
*/
memcpy(&dev->data->dev_conf, dev_conf, sizeof(dev->data->dev_conf));
- rte_eth_dev_info_get(port_id, &dev_info);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
/* If number of queues specified by application for both Rx and Tx is
* zero, use driver preferred values. This cannot be done individually
@@ -1406,6 +1407,7 @@ struct rte_eth_dev *
struct rte_eth_dev *dev;
struct rte_eth_dev_info dev_info;
int diag;
+ int ret;
RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
@@ -1420,7 +1422,9 @@ struct rte_eth_dev *
return 0;
}
- rte_eth_dev_info_get(port_id, &dev_info);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
/* Lets restore MAC now if device does not support live change */
if (*dev_info.dev_flags & RTE_ETH_DEV_NOLIVE_MAC_ADDR)
@@ -1584,7 +1588,6 @@ struct rte_eth_dev *
return -EINVAL;
}
- RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_setup, -ENOTSUP);
/*
@@ -1592,7 +1595,10 @@ struct rte_eth_dev *
* This value must be provided in the private data of the memory pool.
* First check that the memory pool has a valid private data.
*/
- rte_eth_dev_info_get(port_id, &dev_info);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
+
if (mp->private_data_size < sizeof(struct rte_pktmbuf_pool_private)) {
RTE_ETHDEV_LOG(ERR, "%s private_data_size %d < %d\n",
mp->name, (int)mp->private_data_size,
@@ -1703,6 +1709,7 @@ struct rte_eth_dev *
struct rte_eth_dev_info dev_info;
struct rte_eth_txconf local_conf;
void **txq;
+ int ret;
RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
@@ -1712,10 +1719,11 @@ struct rte_eth_dev *
return -EINVAL;
}
- RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_setup, -ENOTSUP);
- rte_eth_dev_info_get(port_id, &dev_info);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
/* Use default specified by driver, if nb_tx_desc is zero */
if (nb_tx_desc == 0) {
@@ -2540,7 +2548,7 @@ struct rte_eth_dev *
fw_version, fw_size));
}
-void
+int
rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info)
{
struct rte_eth_dev *dev;
@@ -2558,7 +2566,7 @@ struct rte_eth_dev *
*/
memset(dev_info, 0, sizeof(struct rte_eth_dev_info));
- RTE_ETH_VALID_PORTID_OR_RET(port_id);
+ RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
dev = &rte_eth_devices[port_id];
dev_info->rx_desc_lim = lim;
@@ -2567,13 +2575,15 @@ struct rte_eth_dev *
dev_info->min_mtu = RTE_ETHER_MIN_MTU;
dev_info->max_mtu = UINT16_MAX;
- RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_infos_get);
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
(*dev->dev_ops->dev_infos_get)(dev, dev_info);
dev_info->driver_name = dev->device->driver->name;
dev_info->nb_rx_queues = dev->data->nb_rx_queues;
dev_info->nb_tx_queues = dev->data->nb_tx_queues;
dev_info->dev_flags = &dev->data->dev_flags;
+
+ return 0;
}
int
@@ -2643,7 +2653,10 @@ struct rte_eth_dev *
* which relies on dev->dev_ops->dev_infos_get.
*/
if (*dev->dev_ops->dev_infos_get != NULL) {
- rte_eth_dev_info_get(port_id, &dev_info);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
+
if (mtu < dev_info.min_mtu || mtu > dev_info.max_mtu)
return -EINVAL;
}
@@ -2991,10 +3004,15 @@ struct rte_eth_dev *
{
struct rte_eth_dev *dev;
struct rte_eth_dev_info dev_info = { .flow_type_rss_offloads = 0, };
+ int ret;
RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
+
dev = &rte_eth_devices[port_id];
- rte_eth_dev_info_get(port_id, &dev_info);
if ((dev_info.flow_type_rss_offloads | rss_conf->rss_hf) !=
dev_info.flow_type_rss_offloads) {
RTE_ETHDEV_LOG(ERR,
@@ -3100,9 +3118,13 @@ struct rte_eth_dev *
struct rte_eth_dev_info dev_info;
struct rte_eth_dev *dev = &rte_eth_devices[port_id];
unsigned i;
+ int ret;
RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
- rte_eth_dev_info_get(port_id, &dev_info);
+
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
for (i = 0; i < dev_info.max_mac_addrs; i++)
if (memcmp(addr, &dev->data->mac_addrs[i],
@@ -3233,8 +3255,14 @@ struct rte_eth_dev *
struct rte_eth_dev_info dev_info;
struct rte_eth_dev *dev = &rte_eth_devices[port_id];
unsigned i;
+ int ret;
+
+ RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
- rte_eth_dev_info_get(port_id, &dev_info);
if (!dev->data->hash_mac_addrs)
return -1;
@@ -3319,11 +3347,15 @@ int rte_eth_set_queue_rate_limit(uint16_t port_id, uint16_t queue_idx,
struct rte_eth_dev *dev;
struct rte_eth_dev_info dev_info;
struct rte_eth_link link;
+ int ret;
RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
+
dev = &rte_eth_devices[port_id];
- rte_eth_dev_info_get(port_id, &dev_info);
link = dev->data->dev_link;
if (queue_idx > dev_info.max_tx_queues) {
@@ -4363,15 +4395,14 @@ int rte_eth_set_queue_rate_limit(uint16_t port_id, uint16_t queue_idx,
uint16_t *nb_rx_desc,
uint16_t *nb_tx_desc)
{
- struct rte_eth_dev *dev;
struct rte_eth_dev_info dev_info;
+ int ret;
RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
- dev = &rte_eth_devices[port_id];
- RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
-
- rte_eth_dev_info_get(port_id, &dev_info);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
if (nb_rx_desc != NULL)
rte_eth_dev_adjust_nb_desc(nb_rx_desc, &dev_info.rx_desc_lim);
diff --git a/lib/librte_ethdev/rte_ethdev.h b/lib/librte_ethdev/rte_ethdev.h
index 8fa89bf..eda9e5c 100644
--- a/lib/librte_ethdev/rte_ethdev.h
+++ b/lib/librte_ethdev/rte_ethdev.h
@@ -2366,8 +2366,12 @@ int rte_eth_dev_set_rx_queue_stats_mapping(uint16_t port_id,
* @param dev_info
* A pointer to a structure of type *rte_eth_dev_info* to be filled with
* the contextual information of the Ethernet device.
+ * @return
+ * - (0) if successful.
+ * - (-ENOTSUP) if support for dev_infos_get() does not exist for the device.
+ * - (-ENODEV) if *port_id* invalid.
*/
-void rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info);
+int rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info);
/**
* Retrieve the firmware version of a device.
--
1.8.3.1
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] [PATCH 02/11] log: define logtype register wrapper for drivers
2019-09-03 8:06 4% ` David Marchand
@ 2019-09-03 8:47 3% ` Ferruh Yigit
2019-09-04 17:45 0% ` Thomas Monjalon
0 siblings, 1 reply; 200+ results
From: Ferruh Yigit @ 2019-09-03 8:47 UTC (permalink / raw)
To: David Marchand; +Cc: dev
On 9/3/2019 9:06 AM, David Marchand wrote:
> On Mon, Sep 2, 2019 at 4:29 PM Ferruh Yigit <ferruh.yigit@intel.com> wrote:
>>
>> On 8/19/2019 12:41 PM, David Marchand wrote:
>>> The function rte_log_register_type_and_pick_level() fills a gap for
>>> dynamically loaded code (especially drivers) who would not pick up
>>> the log level passed at startup.
>>>
>>> Let's promote it to stable and export it for use by drivers via
>>> a wrapper.
>>>
>>> Signed-off-by: David Marchand <david.marchand@redhat.com>
>>> ---
>>> lib/librte_eal/common/include/rte_log.h | 12 ++++++++----
>>> lib/librte_eal/rte_eal_version.map | 8 +++++++-
>>> 2 files changed, 15 insertions(+), 5 deletions(-)
>>>
>>> diff --git a/lib/librte_eal/common/include/rte_log.h b/lib/librte_eal/common/include/rte_log.h
>>> index cbb4184..c3aff00 100644
>>> --- a/lib/librte_eal/common/include/rte_log.h
>>> +++ b/lib/librte_eal/common/include/rte_log.h
>>> @@ -209,9 +209,6 @@ int rte_log_cur_msg_logtype(void);
>>> int rte_log_register(const char *name);
>>>
>>> /**
>>> - * @warning
>>> - * @b EXPERIMENTAL: this API may change without prior notice
>>> - *
>>> * Register a dynamic log type and try to pick its level from EAL options
>>> *
>>> * rte_log_register() is called inside. If successful, the function tries
>>> @@ -227,9 +224,16 @@ int rte_log_register(const char *name);
>>> * - >=0: the newly registered log type
>>> * - <0: rte_log_register() error value
>>> */
>>> -__rte_experimental
>>> int rte_log_register_type_and_pick_level(const char *name, uint32_t level_def);
>>
>> +1 to remove experimental from the API.
>>
>>>
>>> +#define RTE_LOG_REGISTER(token, name, level, fallback) \
>>> +RTE_INIT(token##_init) \
>>
>> Does it still need to be an init time call?
>> Since it is dynamic now it can be during probe, even log name can be a paramter
>> to the "struct rte_driver" and log can be registered automatically during probe,
>> not sure how complex it becomes.
>
> This would not work with non driver components built as shared
> libraries (unless they have an explicit init symbol in the dpdk init
> flow).
Right.
>
> The drivers can register multiple log types so this would have to be handled.
> We would touch the struct rte_driver which is embedded in other
> objects like rte_pci_driver, breaking the abi.
Yes they may require multiple logs, +abi break, so forget about it.
>
>
>>
>>> +{ \
>>> + token = rte_log_register_type_and_pick_level(name, level); \
>>> + if (token < 0) \
>>
>> The failure can be because component can try to register existing log name, or
>> there is no enough memory, do you think does it worth to do log, or some
>> additional work if component is trying to register existing log name?
>
> Yes, I can raise a warning log (using RTE_LOGTYPE_EAL type), since
> duplicates are not supposed to happen.
I was checking if we can detect the error from duplication, there can be a
defect it that logic:
Call trace is:
rte_log_register_type_and_pick_level
type = rte_log_register(name);
id = rte_log_lookup(name);
if (id >= 0)
return id
if (type < 0)
return type
"type > 0" for the duplication case but error check only checks if "type < 0"
>
>
>>
>>> + token = fallback; \
>>
>> Does the 'fallback' needs to be provided by user, it looks like everyone will
>> just copy/paste 'RTE_LOGTYPE_PMD' for drivers, and does it really needs to be
>> configurable since it is fallback.
>
> This series only touches drivers, but I expected other components
> would use this macro later.
> I can add a RTE_PMD_REGISTER_LOG macro that hides the RTE_LOGTYPE_PMD
> fallback value.
>
>
>>
>> Why not provide a hardcoded type for the failure case? And for that case perhaps
>> create a more generic logtype, something like "RTE_LOGTYPE_FALLBACK" so that it
>> can be as it is from all components?
>>
>
> I prefer to map all drivers to a logtype that means something, like
> RTE_LOGTYPE_PMD.
In that manner it make sense agreed, but previously drivers were using
'RTE_LOGTYPE_PMD' instead of having their own log types, Stephen did some work
to replace the 'RTE_LOGTYPE_PMD' so that it can be deprecated,
starting to use it again as fallback may lead drivers using it again as log type
in their drivers, may they wouldn't but this is what I concern. Something with
name 'RTE_LOGTYPE_FALLBACK' clear to not use as default logtype in drivers.
>
> Having a "fallback" could be used for all components, but this would
> have to be a static logtype and adding one is not possible without
> breaking the abi (static entries are < 32 and all values are used).
There is a gap between 'RTE_LOGTYPE_GSO' & 'RTE_LOGTYPE_USER1' ...
>
>
> --
> David Marchand
>
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] [PATCH 02/11] log: define logtype register wrapper for drivers
@ 2019-09-03 8:06 4% ` David Marchand
2019-09-03 8:47 3% ` Ferruh Yigit
0 siblings, 1 reply; 200+ results
From: David Marchand @ 2019-09-03 8:06 UTC (permalink / raw)
To: Ferruh Yigit; +Cc: dev
On Mon, Sep 2, 2019 at 4:29 PM Ferruh Yigit <ferruh.yigit@intel.com> wrote:
>
> On 8/19/2019 12:41 PM, David Marchand wrote:
> > The function rte_log_register_type_and_pick_level() fills a gap for
> > dynamically loaded code (especially drivers) who would not pick up
> > the log level passed at startup.
> >
> > Let's promote it to stable and export it for use by drivers via
> > a wrapper.
> >
> > Signed-off-by: David Marchand <david.marchand@redhat.com>
> > ---
> > lib/librte_eal/common/include/rte_log.h | 12 ++++++++----
> > lib/librte_eal/rte_eal_version.map | 8 +++++++-
> > 2 files changed, 15 insertions(+), 5 deletions(-)
> >
> > diff --git a/lib/librte_eal/common/include/rte_log.h b/lib/librte_eal/common/include/rte_log.h
> > index cbb4184..c3aff00 100644
> > --- a/lib/librte_eal/common/include/rte_log.h
> > +++ b/lib/librte_eal/common/include/rte_log.h
> > @@ -209,9 +209,6 @@ int rte_log_cur_msg_logtype(void);
> > int rte_log_register(const char *name);
> >
> > /**
> > - * @warning
> > - * @b EXPERIMENTAL: this API may change without prior notice
> > - *
> > * Register a dynamic log type and try to pick its level from EAL options
> > *
> > * rte_log_register() is called inside. If successful, the function tries
> > @@ -227,9 +224,16 @@ int rte_log_register(const char *name);
> > * - >=0: the newly registered log type
> > * - <0: rte_log_register() error value
> > */
> > -__rte_experimental
> > int rte_log_register_type_and_pick_level(const char *name, uint32_t level_def);
>
> +1 to remove experimental from the API.
>
> >
> > +#define RTE_LOG_REGISTER(token, name, level, fallback) \
> > +RTE_INIT(token##_init) \
>
> Does it still need to be an init time call?
> Since it is dynamic now it can be during probe, even log name can be a paramter
> to the "struct rte_driver" and log can be registered automatically during probe,
> not sure how complex it becomes.
This would not work with non driver components built as shared
libraries (unless they have an explicit init symbol in the dpdk init
flow).
The drivers can register multiple log types so this would have to be handled.
We would touch the struct rte_driver which is embedded in other
objects like rte_pci_driver, breaking the abi.
>
> > +{ \
> > + token = rte_log_register_type_and_pick_level(name, level); \
> > + if (token < 0) \
>
> The failure can be because component can try to register existing log name, or
> there is no enough memory, do you think does it worth to do log, or some
> additional work if component is trying to register existing log name?
Yes, I can raise a warning log (using RTE_LOGTYPE_EAL type), since
duplicates are not supposed to happen.
>
> > + token = fallback; \
>
> Does the 'fallback' needs to be provided by user, it looks like everyone will
> just copy/paste 'RTE_LOGTYPE_PMD' for drivers, and does it really needs to be
> configurable since it is fallback.
This series only touches drivers, but I expected other components
would use this macro later.
I can add a RTE_PMD_REGISTER_LOG macro that hides the RTE_LOGTYPE_PMD
fallback value.
>
> Why not provide a hardcoded type for the failure case? And for that case perhaps
> create a more generic logtype, something like "RTE_LOGTYPE_FALLBACK" so that it
> can be as it is from all components?
>
I prefer to map all drivers to a logtype that means something, like
RTE_LOGTYPE_PMD.
Having a "fallback" could be used for all components, but this would
have to be a static logtype and adding one is not possible without
breaking the abi (static entries are < 32 and all values are used).
--
David Marchand
^ permalink raw reply [relevance 4%]
* Re: [dpdk-dev] [PATCH 04/15] net/virtio: add virtio PCI subsystem device ID declaration
2019-09-02 6:14 0% ` Tiwei Bie
@ 2019-09-03 7:25 0% ` Maxime Coquelin
0 siblings, 0 replies; 200+ results
From: Maxime Coquelin @ 2019-09-03 7:25 UTC (permalink / raw)
To: Tiwei Bie; +Cc: zhihong.wang, amorenoz, xiao.w.wang, dev, jfreimann, stable
On 9/2/19 8:14 AM, Tiwei Bie wrote:
> On Thu, Aug 29, 2019 at 09:59:49AM +0200, Maxime Coquelin wrote:
>> The Virtio PCI susbsytem IDs need to be specified to
>> prevent it to probe IFC vDPA VFs.
>>
>> Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
>> ---
>> drivers/net/virtio/virtio_pci.h | 1 +
>> 1 file changed, 1 insertion(+)
>>
>> diff --git a/drivers/net/virtio/virtio_pci.h b/drivers/net/virtio/virtio_pci.h
>> index a38cb45ad..56f89a454 100644
>> --- a/drivers/net/virtio/virtio_pci.h
>> +++ b/drivers/net/virtio/virtio_pci.h
>> @@ -19,6 +19,7 @@ struct virtnet_ctl;
>> #define VIRTIO_PCI_VENDORID 0x1AF4
>> #define VIRTIO_PCI_LEGACY_DEVICEID_NET 0x1000
>> #define VIRTIO_PCI_MODERN_DEVICEID_NET 0x1041
>> +#define VIRTIO_PCI_SUBSY_DEVICEID_NET 0x1100
>
> 0x1100 is the subsystem device ID used by QEMU.
> Maybe naming it VIRTIO_PCI_SUBSYS_DEVICEID_QEMU is better?
Indeed, will do.
Thanks,
Maxime
> Regards,
> Tiwei
>
>>
>> /* VirtIO ABI version, this must match exactly. */
>> #define VIRTIO_PCI_ABI_VERSION 0
>> --
>> 2.21.0
>>
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [PATCH 04/15] net/virtio: add virtio PCI subsystem device ID declaration
2019-08-29 7:59 4% ` [dpdk-dev] [PATCH 04/15] net/virtio: add virtio PCI subsystem device ID declaration Maxime Coquelin
@ 2019-09-02 6:14 0% ` Tiwei Bie
2019-09-03 7:25 0% ` Maxime Coquelin
0 siblings, 1 reply; 200+ results
From: Tiwei Bie @ 2019-09-02 6:14 UTC (permalink / raw)
To: Maxime Coquelin
Cc: zhihong.wang, amorenoz, xiao.w.wang, dev, jfreimann, stable
On Thu, Aug 29, 2019 at 09:59:49AM +0200, Maxime Coquelin wrote:
> The Virtio PCI susbsytem IDs need to be specified to
> prevent it to probe IFC vDPA VFs.
>
> Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
> ---
> drivers/net/virtio/virtio_pci.h | 1 +
> 1 file changed, 1 insertion(+)
>
> diff --git a/drivers/net/virtio/virtio_pci.h b/drivers/net/virtio/virtio_pci.h
> index a38cb45ad..56f89a454 100644
> --- a/drivers/net/virtio/virtio_pci.h
> +++ b/drivers/net/virtio/virtio_pci.h
> @@ -19,6 +19,7 @@ struct virtnet_ctl;
> #define VIRTIO_PCI_VENDORID 0x1AF4
> #define VIRTIO_PCI_LEGACY_DEVICEID_NET 0x1000
> #define VIRTIO_PCI_MODERN_DEVICEID_NET 0x1041
> +#define VIRTIO_PCI_SUBSY_DEVICEID_NET 0x1100
0x1100 is the subsystem device ID used by QEMU.
Maybe naming it VIRTIO_PCI_SUBSYS_DEVICEID_QEMU is better?
Regards,
Tiwei
>
> /* VirtIO ABI version, this must match exactly. */
> #define VIRTIO_PCI_ABI_VERSION 0
> --
> 2.21.0
>
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [PATCH v3 2/4] doc: changes to abi policy introducing major abi versions
2019-08-15 10:23 31% ` [dpdk-dev] [PATCH v3 2/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
@ 2019-08-30 16:20 10% ` Kevin Traynor
2019-09-24 11:32 10% ` Ray Kinsella
0 siblings, 1 reply; 200+ results
From: Kevin Traynor @ 2019-08-30 16:20 UTC (permalink / raw)
To: Ray Kinsella, dev
Cc: thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, nhorman,
maxime.coquelin, john.mcnamara, marko.kovacevic, hemant.agrawal,
Luca Boccassi, David Marchand
Hi Ray,
On 15/08/2019 11:23, Ray Kinsella wrote:
> This policy change introduces major ABI versions, these are
> declared every year, typically aligned with the LTS release
> and are supported by subsequent releases in the following year.
> This change is intended to improve ABI stabilty for those projects
> consuming DPDK.
>
> Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
> ---
> doc/guides/contributing/abi_policy.rst | 308 ++++++++++++++++++++++++---------
> doc/guides/contributing/stable.rst | 38 ++--
> 2 files changed, 245 insertions(+), 101 deletions(-)
>
> diff --git a/doc/guides/contributing/abi_policy.rst b/doc/guides/contributing/abi_policy.rst
> index 55bacb4..6190bdc 100644
> --- a/doc/guides/contributing/abi_policy.rst
> +++ b/doc/guides/contributing/abi_policy.rst
> @@ -1,33 +1,46 @@
> .. SPDX-License-Identifier: BSD-3-Clause
> - Copyright 2018 The DPDK contributors
> + Copyright 2019 The DPDK contributors
>
> -.. abi_api_policy:
> +.. _abi_policy:
>
> -DPDK ABI/API policy
> -===================
> +ABI Policy
> +==========
>
> Description
> -----------
>
> -This document details some methods for handling ABI management in the DPDK.
> +This document details the management policy that ensures the long-term stability
> +of the DPDK ABI and API.
>
> General Guidelines
> ------------------
>
> -#. Whenever possible, ABI should be preserved
> -#. ABI/API may be changed with a deprecation process
> -#. The modification of symbols can generally be managed with versioning
> -#. Libraries or APIs marked in ``experimental`` state may change without constraint
> -#. New APIs will be marked as ``experimental`` for at least one release to allow
> - any issues found by users of the new API to be fixed quickly
> -#. The addition of symbols is generally not problematic
> -#. The removal of symbols generally is an ABI break and requires bumping of the
> - LIBABIVER macro
> -#. Updates to the minimum hardware requirements, which drop support for hardware which
> - was previously supported, should be treated as an ABI change.
> -
> -What is an ABI
> -~~~~~~~~~~~~~~
> +#. Major ABI versions are declared every **year** and are then supported for one
> + year, typically aligned with the :ref:`LTS release <stable_lts_releases>`.
> +#. The ABI version is managed at a project level in DPDK, with the ABI version
> + reflected in all :ref:`library's soname <what_is_soname>`.
> +#. The ABI should be preserved and not changed lightly. ABI changes must follow
> + the outlined :ref:`deprecation process <abi_changes>`.
> +#. The addition of symbols is generally not problematic. The modification of
> + symbols is managed with :ref:`ABI Versioning <abi_versioning>`.
> +#. The removal of symbols is considered an :ref:`ABI breakage <abi_breakages>`,
> + once approved these will form part of the next ABI version.
> +#. Libraries or APIs marked as :ref:`Experimental <experimental_apis>` are not
> + considered part of an ABI version and may change without constraint.
> +#. Updates to the :ref:`minimum hardware requirements <hw_rqmts>`, which drop
> + support for hardware which was previously supported, should be treated as an
> + ABI change.
> +
> +.. note::
> +
> + In 2019, the DPDK community stated it's intention to move to ABI stable
> + releases, over a number of release cycles. Beginning with maintaining ABI
> + stability through one year of DPDK releases starting from DPDK 19.11. This
> + policy will be reviewed in 2020, with intention of lengthening the stability
> + period.
> +
> +What is an ABI?
> +~~~~~~~~~~~~~~~
>
> An ABI (Application Binary Interface) is the set of runtime interfaces exposed
> by a library. It is similar to an API (Application Programming Interface) but
> @@ -39,30 +52,67 @@ Therefore, in the case of dynamic linking, it is critical that an ABI is
> preserved, or (when modified), done in such a way that the application is unable
> to behave improperly or in an unexpected fashion.
>
> +What is an ABI version?
> +~~~~~~~~~~~~~~~~~~~~~~~
>
> -ABI/API Deprecation
> --------------------
> +An ABI version is an instance of a library's ABI at a specific release. Certain
> +releases are considered by the community to be milestone releases, the yearly
> +LTS for example. Supporting those milestone release's ABI for some number of
> +subsequent releases is desirable to facilitate application upgrade. Those ABI
> +version's aligned with milestones release are therefore called 'ABI major
> +versions' and are supported for some number of releases.
> +
> +More details on major ABI version can be found in the :ref:`ABI versioning
> +<major_abi_versions>` guide.
>
> The DPDK ABI policy
> -~~~~~~~~~~~~~~~~~~~
> +-------------------
> +
> +A major ABI version is declared every year, aligned with that year's LTS
> +release, e.g. v19.11. This ABI version is then supported for one year by all
> +subsequent releases within that time period, until the next LTS release, e.g.
> +v20.11.
> +
> +At the declaration of a major ABI version, major version numbers encoded in
> +libraries soname's are bumped to indicate the new version, with the minor
> +version reset to ``0``. An example would be ``librte_eal.so.20.3`` would become
> +``librte_eal.so.21.0``.
> +
> +The ABI may then change multiple times, without warning, between the last major
> +ABI version increment and the HEAD label of the git tree, with the condition
> +that ABI compatibility with the major ABI version is preserved and therefore
> +soname's do not change.
> +
> +Minor versions are incremented to indicate the release of a new ABI compatible
> +DPDK release, typically the DPDK quarterly releases. An example of this, might
> +be that ``librte_eal.so.20.1`` would indicate the first ABI compatible DPDK
> +release, following the declaration of the new major ABI version ``20``.
> +
> +ABI versions, are supported by each release until such time as the next major
> +ABI version is declared. At that time, the deprecation of the previous major ABI
> +version will be noted in the Release Notes with guidance on individual symbol
> +depreciation and upgrade notes provided.
>
> -ABI versions are set at the time of major release labeling, and the ABI may
> -change multiple times, without warning, between the last release label and the
> -HEAD label of the git tree.
> +.. _abi_changes:
>
> -ABI versions, once released, are available until such time as their
> -deprecation has been noted in the Release Notes for at least one major release
> -cycle. For example consider the case where the ABI for DPDK 2.0 has been
> -shipped and then a decision is made to modify it during the development of
> -DPDK 2.1. The decision will be recorded in the Release Notes for the DPDK 2.1
> -release and the modification will be made available in the DPDK 2.2 release.
> +ABI Changes
> +~~~~~~~~~~~
>
> -ABI versions may be deprecated in whole or in part as needed by a given
> -update.
> +The ABI may still change after the declaration of a major ABI version, that is
> +new APIs may be still added or existing APIs may be modified.
>
> -Some ABI changes may be too significant to reasonably maintain multiple
> -versions. In those cases ABI's may be updated without backward compatibility
> -being provided. The requirements for doing so are:
> +.. Warning::
> +
> + Note that, the process for ABI deprecation should not be undertaken lightly.
> + ABI stability is extremely important for downstream consumers of the DPDK,
> + especially when distributed in shared object form. Every effort should be
> + made to preserve the ABI whenever possible. The ABI should only be changed
> + for significant reasons, such as performance enhancements. ABI breakage due
> + to changes such as reorganizing public structure fields for aesthetic or
> + readability purposes should be avoided.
> +
This text is not changed and it reads like *any* performance enhancement
is a good enough reason for an ABI break. Can't obviously quantify it,
but maybe "major performance enhancement" is closer to the intended
tone? Sorry for nit-picking over one word!
> +
> +The requirements for changing the ABI are:
>
> #. At least 3 acknowledgments of the need to do so must be made on the
> dpdk.org mailing list.
> @@ -71,34 +121,119 @@ being provided. The requirements for doing so are:
> no maintainer is available for the component, the tree/sub-tree maintainer
> for that component must acknowledge the ABI change instead.
>
> + - The acknowledgment of a member of the technical board, as a delegate of the
> + `technical board <https://core.dpdk.org/techboard/>`_ acknowledging the
> + need for the ABI change, is also mandatory.
> +
> - It is also recommended that acknowledgments from different "areas of
> interest" be sought for each deprecation, for example: from NIC vendors,
> CPU vendors, end-users, etc.
>
> -#. The changes (including an alternative map file) can be included with
> - deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option,
> - to provide more details about oncoming changes.
> - ``RTE_NEXT_ABI`` wrapper will be removed when it become the default ABI.
> - More preferred way to provide this information is sending the feature
> - as a separate patch and reference it in deprecation notice.
> +#. Backward compatibly with the major ABI version must be maintained through
s/compatibly/compatibility/
> + :ref:`abi_versioning`, with :ref:`forward-only <forward-only>` compatibility
> + offered for any ABI changes that are indicated to be part of the next ABI
> + version.
>
> -#. A full deprecation cycle, as explained above, must be made to offer
> - downstream consumers sufficient warning of the change.
> + - In situations were backward compatibility is not possible, read the
s/were/where/
> + section on :ref:`abi_breakages`.
>
> -Note that the above process for ABI deprecation should not be undertaken
> -lightly. ABI stability is extremely important for downstream consumers of the
> -DPDK, especially when distributed in shared object form. Every effort should
> -be made to preserve the ABI whenever possible. The ABI should only be changed
> -for significant reasons, such as performance enhancements. ABI breakage due to
> -changes such as reorganizing public structure fields for aesthetic or
> -readability purposes should be avoided.
> + - No backward or forward compatibility is offered for API changes marked as
> + ``experimental``, as described in the section on :ref:`Experimental APIs
> + and Libraries <experimental_apis>`.
>
> -.. note::
> +#. If a newly proposed API functionally replaces an existing one, when the new
> + API becomes non-experimental, then the old one is marked with
> + ``__rte_deprecated``.
> +
> + - The depreciated API should follow the notification process to be removed,
> + see :ref:`deprecation_notices`.
> +
> + - At the declaration of the next major ABI version, those ABI changes then
> + become a formal part of the new ABI and the requirement to preserve ABI
> + compatibility with the last major ABI version is then dropped.
> +
> + - The responsibility for removing redundant ABI compatibility code rests
> + with the original contributor of the ABI changes, failing that, then with
> + the contributor's company and then finally with the maintainer.
> +
> +.. _forward-only:
> +
> +.. Note::
> +
> + Note that forward-only compatibility is offered for those changes made
> + between major ABI versions. As a library's soname can only describe
> + compatibility with the last major ABI version, until the next major ABI
> + version is declared, these changes therefore cannot be resolved as a runtime
> + dependency through the soname. Therefore any application wishing to make use
> + of these ABI changes can only ensure that it's runtime dependencies are met
> + through Operating System package versioning.
> +
> +.. _hw_rqmts:
> +
> +.. Note::
>
> Updates to the minimum hardware requirements, which drop support for hardware
> which was previously supported, should be treated as an ABI change, and
> - follow the relevant deprecation policy procedures as above: 3 acks and
> - announcement at least one release in advance.
> + follow the relevant deprecation policy procedures as above: 3 acks, technical
> + board approval and announcement at least one release in advance.
> +
> +.. _abi_breakages:
> +
> +ABI Breakages
> +~~~~~~~~~~~~~
> +
> +For those ABI changes that are too significant to reasonably maintain multiple
> +symbol versions, there is an amended process. In these cases, ABIs may be
> +updated without the requirement of backward compatibility being provided. These
> +changes must follow the `same process :ref:`described above <abi_changes>` as non-breaking
> +changes, however with the following additional requirements:
> +
> +#. ABI breaking changes (including an alternative map file) can be included with
> + deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option, to provide
> + more details about oncoming changes. ``RTE_NEXT_ABI`` wrapper will be removed
> + at the declaration of the next major ABI version.
> +
> +#. Once approved, and after the depreciation notice has been observed these
> + changes will form part of the next declared major ABI version.
> +
> +Examples of ABI Changes
> +~~~~~~~~~~~~~~~~~~~~~~~
> +
> +The following are examples of allowable ABI changes occurring between
> +declarations of major ABI versions.
> +
> +* DPDK 19.11 release, defines the function ``rte_foo()``, and ``rte_foo()``
> + as part of the major ABI version ``20``.
> +
> +* DPDK 20.02 release defines a new function ``rte_foo(uint8_t bar)``, and
> + this is not a problem as long as the symbol ``rte_foo@DPDK20`` is
> + preserved through :ref:`abi_versioning`.
> +
> + - The new function may be marked with the ``__rte_experimental`` tag for a
> + number of releases, as described in the section :ref:`experimental_apis`.
> +
> + - Once ``rte_foo(uint8_t bar)`` becomes non-experimental ``rte_foo()`` is then
> + declared as ``__rte_depreciated``, with an associated deprecation notice
> + provided.
> +
> +* DPDK 19.11 is not re-released to include ``rte_foo(uint8_t bar)``, the new
> + version of ``rte_foo`` only exists from DPDK 20.02 onwards as described in the
> + :ref:`note on forward-only compatibility<forward-only>`.
> +
> +* DPDK 20.02 release defines the experimental function ``__rte_experimental
> + rte_baz()``. This function may or may not exist in the DPDK 20.05 release.
> +
> +* An application ``dPacket`` wishes to use ``rte_foo(uint8_t bar)``, before the
> + declaration of the DPDK ``21`` major API version. The application can only
> + ensure it's runtime dependencies are met by specifying ``DPDK (>= 20.2)`` as
> + an explicit package dependency, as the soname only may only indicate the
> + supported major ABI version.
> +
> +* At the release of DPDK 20.11, the function ``rte_foo(uint8_t bar)`` becomes
> + formally part of then new major ABI version DPDK 21.0 and ``rte_foo()`` may be
> + removed.
> +
> +.. _deprecation_notices:
>
> Examples of Deprecation Notices
> ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
> @@ -106,46 +241,42 @@ Examples of Deprecation Notices
> The following are some examples of ABI deprecation notices which would be
> added to the Release Notes:
>
> -* The Macro ``#RTE_FOO`` is deprecated and will be removed with version 2.0,
> - to be replaced with the inline function ``rte_foo()``.
> +* The Macro ``#RTE_FOO`` is deprecated and will be removed with ABI version
> + 21, to be replaced with the inline function ``rte_foo()``.
>
> * The function ``rte_mbuf_grok()`` has been updated to include a new parameter
> - in version 2.0. Backwards compatibility will be maintained for this function
> - until the release of version 2.1
> + in version 20.2. Backwards compatibility will be maintained for this function
> + until the release of the new DPDK major ABI version 21, in DPDK version
> + 20.11.
>
> -* The members of ``struct rte_foo`` have been reorganized in release 2.0 for
> +* The members of ``struct rte_foo`` have been reorganized in DPDK 20.02 for
> performance reasons. Existing binary applications will have backwards
> - compatibility in release 2.0, while newly built binaries will need to
> - reference the new structure variant ``struct rte_foo2``. Compatibility will
> - be removed in release 2.2, and all applications will require updating and
> + compatibility in release 20.02, while newly built binaries will need to
> + reference the new structure variant ``struct rte_foo2``. Compatibility will be
> + removed in release 20.11, and all applications will require updating and
> rebuilding to the new structure at that time, which will be renamed to the
> original ``struct rte_foo``.
>
> * Significant ABI changes are planned for the ``librte_dostuff`` library. The
> - upcoming release 2.0 will not contain these changes, but release 2.1 will,
> + upcoming release 20.02 will not contain these changes, but release 20.11 will,
> and no backwards compatibility is planned due to the extensive nature of
> - these changes. Binaries using this library built prior to version 2.1 will
> + these changes. Binaries using this library built prior to ABI version 21 will
> require updating and recompilation.
>
> -New API replacing previous one
> -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
> -
> -If a new API proposed functionally replaces an existing one, when the new API
> -becomes non-experimental then the old one is marked with ``__rte_deprecated``.
> -Deprecated APIs are removed completely just after the next LTS.
> -
> -Reminder that old API should follow deprecation process to be removed.
> +.. _experimental_apis:
>
> +Experimental
> +------------
>
> -Experimental APIs
> ------------------
> +APIs
> +~~~~
>
> -APIs marked as ``experimental`` are not considered part of the ABI and may
> -change without warning at any time. Since changes to APIs are most likely
> -immediately after their introduction, as users begin to take advantage of
> -those new APIs and start finding issues with them, new DPDK APIs will be
> -automatically marked as ``experimental`` to allow for a period of stabilization
> -before they become part of a tracked ABI.
> +APIs marked as ``experimental`` are not considered part of an ABI version and
> +may change without warning at any time. Since changes to APIs are most likely
> +immediately after their introduction, as users begin to take advantage of those
> +new APIs and start finding issues with them, new DPDK APIs will be automatically
> +marked as ``experimental`` to allow for a period of stabilization before they
> +become part of a tracked ABI version.
>
> Note that marking an API as experimental is a multi step process.
> To mark an API as experimental, the symbols which are desired to be exported
> @@ -163,7 +294,16 @@ In addition to tagging the code with ``__rte_experimental``,
> the doxygen markup must also contain the EXPERIMENTAL string,
> and the MAINTAINERS file should note the EXPERIMENTAL libraries.
>
> -For removing the experimental tag associated with an API, deprecation notice
> -is not required. Though, an API should remain in experimental state for at least
> -one release. Thereafter, normal process of posting patch for review to mailing
> -list can be followed.
> +For removing the experimental tag associated with an API, deprecation notice is
> +not required. Though, an API should remain in experimental state for at least
> +one release. Thereafter, the normal process of posting patch for review to
> +mailing list can be followed.
> +
> +Libraries
> +~~~~~~~~~
> +
> +Libraries marked as ``experimental`` are entirely not considered part of an ABI
> +version, and may change without warning at any time. Experimental libraries
> +always have a major version of ``0`` to indicate they exist outside of
> +:ref:`abi_versioning` , with the minor version incremented with each ABI change
> +to library.
> diff --git a/doc/guides/contributing/stable.rst b/doc/guides/contributing/stable.rst
> index 6a5eee9..d95c200 100644
> --- a/doc/guides/contributing/stable.rst
> +++ b/doc/guides/contributing/stable.rst
> @@ -1,7 +1,7 @@
> .. SPDX-License-Identifier: BSD-3-Clause
> Copyright 2018 The DPDK contributors
>
> -.. stable_lts_releases:
> +.. _stable_lts_releases:
>
> DPDK Stable Releases and Long Term Support
> ==========================================
> @@ -53,6 +53,9 @@ year's November (X.11) release will be maintained as an LTS for 2 years.
> After the X.11 release, an LTS branch will be created for it at
> http://git.dpdk.org/dpdk-stable where bugfixes will be backported to.
>
> +A LTS release may align with the declaration of a new major ABI version,
> +please read the :ref:`abi_policy` for more information.
> +
Above is worth to mention, but as discussed on call earlier today, the
changes below should be dropped from this patchset. At present each LTS
minor release (e.g. 18.11.2) maintains the API/ABI of the original LTS
release (e.g. 18.11) and that is not changing.
What type of non-ABI breaking things are backported to LTS branches can
be discussed during the LTS presentation in DPDK userspace.
thanks,
Kevin.
> It is anticipated that there will be at least 4 releases per year of the LTS
> or approximately 1 every 3 months. However, the cadence can be shorter or
> longer depending on the number and criticality of the backported
> @@ -68,10 +71,13 @@ point the LTS branch will no longer be maintained with no further releases.
> What changes should be backported
> ---------------------------------
>
> -Backporting should be limited to bug fixes. All patches accepted on the master
> -branch with a Fixes: tag should be backported to the relevant stable/LTS
> -branches, unless the submitter indicates otherwise. If there are exceptions,
> -they will be discussed on the mailing lists.
> +Backporting is a naturally conservative activity, and therefore should only
> +include bug fixes and support for new hardware, were adding support does not
> +necessitate DPDK ABI/API changes.
> +
> +All patches accepted on the master branch with a Fixes: tag should be backported
> +to the relevant stable/LTS branches, unless the submitter indicates otherwise.
> +If there are exceptions, they will be discussed on the mailing lists.
>
> Fixes suitable for backport should have a ``Cc: stable@dpdk.org`` tag in the
> commit message body as follows::
> @@ -86,13 +92,18 @@ commit message body as follows::
> Signed-off-by: Alex Smith <alex.smith@example.com>
>
>
> -Fixes not suitable for backport should not include the ``Cc: stable@dpdk.org`` tag.
> +Fixes not suitable for backport should not include the ``Cc: stable@dpdk.org``
> +tag.
>
> -Features should not be backported to stable releases. It may be acceptable, in
> -limited cases, to back port features for the LTS release where:
> +New features, with the exception of new hardware support, should not be
> +backported to stable releases. In the case of new hardware support or any other
> +exceptional circumstances limited backporting maybe permitted to the LTS release
> +where:
>
> -* There is a justifiable use case (for example a new PMD).
> -* The change is non-invasive.
> +* There is a justifiable use case, for example the change is required to support
> + a new platform or device (for example a new PMD).
> +* The change is ABI/API preserving, it does not present an obvious "new feature"
> + to end consumer.
> * The work of preparing the backport is done by the proposer.
> * There is support within the community.
>
> @@ -119,10 +130,3 @@ A Stable Release will be released by:
> list.
>
> Stable releases are available on the `dpdk.org download page <http://core.dpdk.org/download/>`_.
> -
> -
> -ABI
> ----
> -
> -The Stable Release should not be seen as a way of breaking or circumventing
> -the DPDK ABI policy.
>
^ permalink raw reply [relevance 10%]
* [dpdk-dev] [PATCH 04/15] net/virtio: add virtio PCI subsystem device ID declaration
@ 2019-08-29 7:59 4% ` Maxime Coquelin
2019-09-02 6:14 0% ` Tiwei Bie
0 siblings, 1 reply; 200+ results
From: Maxime Coquelin @ 2019-08-29 7:59 UTC (permalink / raw)
To: tiwei.bie, zhihong.wang, amorenoz, xiao.w.wang, dev, jfreimann
Cc: stable, Maxime Coquelin
The Virtio PCI susbsytem IDs need to be specified to
prevent it to probe IFC vDPA VFs.
Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
---
drivers/net/virtio/virtio_pci.h | 1 +
1 file changed, 1 insertion(+)
diff --git a/drivers/net/virtio/virtio_pci.h b/drivers/net/virtio/virtio_pci.h
index a38cb45ad..56f89a454 100644
--- a/drivers/net/virtio/virtio_pci.h
+++ b/drivers/net/virtio/virtio_pci.h
@@ -19,6 +19,7 @@ struct virtnet_ctl;
#define VIRTIO_PCI_VENDORID 0x1AF4
#define VIRTIO_PCI_LEGACY_DEVICEID_NET 0x1000
#define VIRTIO_PCI_MODERN_DEVICEID_NET 0x1041
+#define VIRTIO_PCI_SUBSY_DEVICEID_NET 0x1100
/* VirtIO ABI version, this must match exactly. */
#define VIRTIO_PCI_ABI_VERSION 0
--
2.21.0
^ permalink raw reply [relevance 4%]
* Re: [dpdk-dev] [RFC] rte_hash: introduce hash list into hash lib
@ 2019-08-28 11:53 3% ` Stephen Hemminger
0 siblings, 0 replies; 200+ results
From: Stephen Hemminger @ 2019-08-28 11:53 UTC (permalink / raw)
To: Bing Zhao
Cc: yipeng1.wang, sameh.gobriel, bruce.richardson, pablo.de.lara.guarch, dev
On Wed, 28 Aug 2019 14:51:49 +0800
Bing Zhao <bingz@mellanox.com> wrote:
> +
> +/** Node element structure on the LIST of the link */
> +struct rte_hlist_node_entry {
> + LIST_ENTRY(rte_hlist_node_entry) next; /**< Next element pointer. */
> + /**< Data element inside this noed. */
> + struct rte_hlist_data_element d;
> + char key[]; /**< Copied and stored key. */
> +};
> +
> +/** Head of all the nodes with the same hash value */
> +struct rte_hlist_head_entry {
> + LIST_HEAD(, rte_hlist_node_entry) head; /**< Head for each hash list. */
> + /**< Current items in the list. */
> + uint16_t entries_in_bucket;
> + /**< Shift number for extension */
> + uint16_t bucket_shift;
> +};
> +
> +/** The hlist table structure. */
> +struct rte_hlist_table {
> + char name[RTE_HLIST_NAMESIZE]; /**< Name of the hash. */
> + uint32_t entries; /**< Total number of entries. */
> + uint32_t entries_per_bucket; /**< Number of entries in a list. */
> + /**< Number of entries with data from customer. */
> + uint32_t custom_entries;
> + uint16_t key_len; /**< Length of the key. */
> + /**< Shift number of the whole table. */
> + uint16_t bucket_shift;
> + /**< To find which list the key is in. */
> + uint32_t bucket_mask;
> + rte_hlist_calc_fn hash_func; /**< The hash function to calcuate. */
> + /**< The function to free the custom data. */
> + rte_hlist_free_fn free_func;
> + uint32_t init_val; /**< For initializing hash function. */
> + /**< Reserved for fast shrinking of the table. */
> + char *map;
You probably should use void * for that.
> + /**< A flat and extendible table of all lists. */
> + struct rte_hlist_head_entry *t;
> +};
> +
Since API/ABI considerations are important.
You will save yourself a lot of pain if these structures can be made
private and only part of rte_hlist.c.
^ permalink raw reply [relevance 3%]
* [dpdk-dev] [PATCH v6] eal: add tsc_hz to rte_mem_config
@ 2019-08-27 16:16 3% ` Jim Harris
0 siblings, 0 replies; 200+ results
From: Jim Harris @ 2019-08-27 16:16 UTC (permalink / raw)
To: dev, bruce.richardson, anatoly.burakov
This ensures secondary processes never have to
calculate the TSC rate themselves, which can be
noticeable in VMs that don't have access to
arch-specific detection mechanism (such as
CPUID leaf 0x15 or MSR 0xCE on x86).
Since rte_mem_config is now internal to the rte_eal
library, we can add tsc_hz without ABI breakage
concerns.
Reduces rte_eal_init() execution time in a secondary
process from 165ms to 66ms on my test system.
Signed-off-by: Jim Harris <james.r.harris@intel.com>
---
lib/librte_eal/common/eal_common_timer.c | 15 +++++++++++++++
lib/librte_eal/common/eal_memcfg.h | 3 +++
2 files changed, 18 insertions(+)
diff --git a/lib/librte_eal/common/eal_common_timer.c b/lib/librte_eal/common/eal_common_timer.c
index 145543de7..fa9ee1b22 100644
--- a/lib/librte_eal/common/eal_common_timer.c
+++ b/lib/librte_eal/common/eal_common_timer.c
@@ -15,8 +15,10 @@
#include <rte_log.h>
#include <rte_cycles.h>
#include <rte_pause.h>
+#include <rte_eal.h>
#include "eal_private.h"
+#include "eal_memcfg.h"
/* The frequency of the RDTSC timer resolution */
static uint64_t eal_tsc_resolution_hz;
@@ -77,8 +79,20 @@ estimate_tsc_freq(void)
void
set_tsc_freq(void)
{
+ struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
uint64_t freq;
+ if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
+ /*
+ * Just use the primary process calculated TSC rate in any
+ * secondary process. It avoids any unnecessary overhead on
+ * systems where arch-specific frequency detection is not
+ * available.
+ */
+ eal_tsc_resolution_hz = mcfg->tsc_hz;
+ return;
+ }
+
freq = get_tsc_freq_arch();
if (!freq)
freq = get_tsc_freq();
@@ -87,6 +101,7 @@ set_tsc_freq(void)
RTE_LOG(DEBUG, EAL, "TSC frequency is ~%" PRIu64 " KHz\n", freq / 1000);
eal_tsc_resolution_hz = freq;
+ mcfg->tsc_hz = freq;
}
void rte_delay_us_callback_register(void (*userfunc)(unsigned int))
diff --git a/lib/librte_eal/common/eal_memcfg.h b/lib/librte_eal/common/eal_memcfg.h
index 359beb216..73be6fbae 100644
--- a/lib/librte_eal/common/eal_memcfg.h
+++ b/lib/librte_eal/common/eal_memcfg.h
@@ -70,6 +70,9 @@ struct rte_mem_config {
uint32_t single_file_segments;
/**< stored single file segments parameter. */
+ uint64_t tsc_hz;
+ /**< TSC rate */
+
uint8_t dma_maskbits; /**< Keeps the more restricted dma mask. */
};
^ permalink raw reply [relevance 3%]
* [dpdk-dev] [PATCH 01/51] ethdev: change rte_eth_dev_info_get() return value to int
@ 2019-08-27 14:25 3% ` Andrew Rybchenko
` (2 subsequent siblings)
3 siblings, 0 replies; 200+ results
From: Andrew Rybchenko @ 2019-08-27 14:25 UTC (permalink / raw)
To: Neil Horman, John McNamara, Marko Kovacevic, Thomas Monjalon,
Ferruh Yigit
Cc: dev, Ivan Ilchenko
From: Ivan Ilchenko <Ivan.Ilchenko@oktetlabs.ru>
Change rte_eth_dev_info_get() return value from void to int and return
negative errno values in case of error conditions.
Modify rte_eth_dev_info_get() usage across the ethdev according
to new return type.
Signed-off-by: Ivan Ilchenko <Ivan.Ilchenko@oktetlabs.ru>
Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
---
doc/guides/rel_notes/deprecation.rst | 1 -
doc/guides/rel_notes/release_19_11.rst | 5 ++-
lib/librte_ethdev/rte_ethdev.c | 71 ++++++++++++++++++++++++----------
lib/librte_ethdev/rte_ethdev.h | 6 ++-
4 files changed, 60 insertions(+), 23 deletions(-)
diff --git a/doc/guides/rel_notes/deprecation.rst b/doc/guides/rel_notes/deprecation.rst
index 0ee8533..cbb4c34 100644
--- a/doc/guides/rel_notes/deprecation.rst
+++ b/doc/guides/rel_notes/deprecation.rst
@@ -88,7 +88,6 @@ Deprecation Notices
negative errno values to indicate various error conditions (e.g.
invalid port ID, unsupported operation, failed operation):
- - ``rte_eth_dev_info_get``
- ``rte_eth_promiscuous_enable`` and ``rte_eth_promiscuous_disable``
- ``rte_eth_allmulticast_enable`` and ``rte_eth_allmulticast_disable``
- ``rte_eth_link_get`` and ``rte_eth_link_get_nowait``
diff --git a/doc/guides/rel_notes/release_19_11.rst b/doc/guides/rel_notes/release_19_11.rst
index 8490d89..5424b6a 100644
--- a/doc/guides/rel_notes/release_19_11.rst
+++ b/doc/guides/rel_notes/release_19_11.rst
@@ -85,6 +85,9 @@ API Changes
Also, make sure to start the actual text at the margin.
=========================================================
+* ethdev: changed ``rte_eth_dev_infos_get`` return value from ``void`` to
+ ``int`` to provide a way to report various error conditions.
+
ABI Changes
-----------
@@ -136,7 +139,7 @@ The libraries prepended with a plus sign were incremented in this version.
librte_distributor.so.1
librte_eal.so.11
librte_efd.so.1
- librte_ethdev.so.12
+ + librte_ethdev.so.13
librte_eventdev.so.7
librte_flow_classify.so.1
librte_gro.so.1
diff --git a/lib/librte_ethdev/rte_ethdev.c b/lib/librte_ethdev/rte_ethdev.c
index 17d183e..4b6cbe2 100644
--- a/lib/librte_ethdev/rte_ethdev.c
+++ b/lib/librte_ethdev/rte_ethdev.c
@@ -1125,7 +1125,6 @@ struct rte_eth_dev *
dev = &rte_eth_devices[port_id];
- RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP);
if (dev->data->dev_started) {
@@ -1144,7 +1143,9 @@ struct rte_eth_dev *
*/
memcpy(&dev->data->dev_conf, dev_conf, sizeof(dev->data->dev_conf));
- rte_eth_dev_info_get(port_id, &dev_info);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
/* If number of queues specified by application for both Rx and Tx is
* zero, use driver preferred values. This cannot be done individually
@@ -1406,6 +1407,7 @@ struct rte_eth_dev *
struct rte_eth_dev *dev;
struct rte_eth_dev_info dev_info;
int diag;
+ int ret;
RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
@@ -1420,7 +1422,9 @@ struct rte_eth_dev *
return 0;
}
- rte_eth_dev_info_get(port_id, &dev_info);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
/* Lets restore MAC now if device does not support live change */
if (*dev_info.dev_flags & RTE_ETH_DEV_NOLIVE_MAC_ADDR)
@@ -1584,7 +1588,6 @@ struct rte_eth_dev *
return -EINVAL;
}
- RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_setup, -ENOTSUP);
/*
@@ -1592,7 +1595,10 @@ struct rte_eth_dev *
* This value must be provided in the private data of the memory pool.
* First check that the memory pool has a valid private data.
*/
- rte_eth_dev_info_get(port_id, &dev_info);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
+
if (mp->private_data_size < sizeof(struct rte_pktmbuf_pool_private)) {
RTE_ETHDEV_LOG(ERR, "%s private_data_size %d < %d\n",
mp->name, (int)mp->private_data_size,
@@ -1703,6 +1709,7 @@ struct rte_eth_dev *
struct rte_eth_dev_info dev_info;
struct rte_eth_txconf local_conf;
void **txq;
+ int ret;
RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
@@ -1712,10 +1719,11 @@ struct rte_eth_dev *
return -EINVAL;
}
- RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_setup, -ENOTSUP);
- rte_eth_dev_info_get(port_id, &dev_info);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
/* Use default specified by driver, if nb_tx_desc is zero */
if (nb_tx_desc == 0) {
@@ -2540,7 +2548,7 @@ struct rte_eth_dev *
fw_version, fw_size));
}
-void
+int
rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info)
{
struct rte_eth_dev *dev;
@@ -2558,7 +2566,7 @@ struct rte_eth_dev *
*/
memset(dev_info, 0, sizeof(struct rte_eth_dev_info));
- RTE_ETH_VALID_PORTID_OR_RET(port_id);
+ RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
dev = &rte_eth_devices[port_id];
dev_info->rx_desc_lim = lim;
@@ -2567,13 +2575,15 @@ struct rte_eth_dev *
dev_info->min_mtu = RTE_ETHER_MIN_MTU;
dev_info->max_mtu = UINT16_MAX;
- RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_infos_get);
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
(*dev->dev_ops->dev_infos_get)(dev, dev_info);
dev_info->driver_name = dev->device->driver->name;
dev_info->nb_rx_queues = dev->data->nb_rx_queues;
dev_info->nb_tx_queues = dev->data->nb_tx_queues;
dev_info->dev_flags = &dev->data->dev_flags;
+
+ return 0;
}
int
@@ -2643,7 +2653,10 @@ struct rte_eth_dev *
* which relies on dev->dev_ops->dev_infos_get.
*/
if (*dev->dev_ops->dev_infos_get != NULL) {
- rte_eth_dev_info_get(port_id, &dev_info);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
+
if (mtu < dev_info.min_mtu || mtu > dev_info.max_mtu)
return -EINVAL;
}
@@ -2991,10 +3004,15 @@ struct rte_eth_dev *
{
struct rte_eth_dev *dev;
struct rte_eth_dev_info dev_info = { .flow_type_rss_offloads = 0, };
+ int ret;
RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
+
dev = &rte_eth_devices[port_id];
- rte_eth_dev_info_get(port_id, &dev_info);
if ((dev_info.flow_type_rss_offloads | rss_conf->rss_hf) !=
dev_info.flow_type_rss_offloads) {
RTE_ETHDEV_LOG(ERR,
@@ -3100,9 +3118,13 @@ struct rte_eth_dev *
struct rte_eth_dev_info dev_info;
struct rte_eth_dev *dev = &rte_eth_devices[port_id];
unsigned i;
+ int ret;
RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
- rte_eth_dev_info_get(port_id, &dev_info);
+
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
for (i = 0; i < dev_info.max_mac_addrs; i++)
if (memcmp(addr, &dev->data->mac_addrs[i],
@@ -3233,8 +3255,14 @@ struct rte_eth_dev *
struct rte_eth_dev_info dev_info;
struct rte_eth_dev *dev = &rte_eth_devices[port_id];
unsigned i;
+ int ret;
+
+ RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
- rte_eth_dev_info_get(port_id, &dev_info);
if (!dev->data->hash_mac_addrs)
return -1;
@@ -3319,11 +3347,15 @@ int rte_eth_set_queue_rate_limit(uint16_t port_id, uint16_t queue_idx,
struct rte_eth_dev *dev;
struct rte_eth_dev_info dev_info;
struct rte_eth_link link;
+ int ret;
RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
+
dev = &rte_eth_devices[port_id];
- rte_eth_dev_info_get(port_id, &dev_info);
link = dev->data->dev_link;
if (queue_idx > dev_info.max_tx_queues) {
@@ -4363,15 +4395,14 @@ int rte_eth_set_queue_rate_limit(uint16_t port_id, uint16_t queue_idx,
uint16_t *nb_rx_desc,
uint16_t *nb_tx_desc)
{
- struct rte_eth_dev *dev;
struct rte_eth_dev_info dev_info;
+ int ret;
RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
- dev = &rte_eth_devices[port_id];
- RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
-
- rte_eth_dev_info_get(port_id, &dev_info);
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
if (nb_rx_desc != NULL)
rte_eth_dev_adjust_nb_desc(nb_rx_desc, &dev_info.rx_desc_lim);
diff --git a/lib/librte_ethdev/rte_ethdev.h b/lib/librte_ethdev/rte_ethdev.h
index dc6596b..09c278d 100644
--- a/lib/librte_ethdev/rte_ethdev.h
+++ b/lib/librte_ethdev/rte_ethdev.h
@@ -2366,8 +2366,12 @@ int rte_eth_dev_set_rx_queue_stats_mapping(uint16_t port_id,
* @param dev_info
* A pointer to a structure of type *rte_eth_dev_info* to be filled with
* the contextual information of the Ethernet device.
+ * @return
+ * - (0) if successful.
+ * - (-ENOTSUP) if support for dev_infos_get() does not exist for the device.
+ * - (-ENODEV) if *port_id* invalid.
*/
-void rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info);
+int rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info);
/**
* Retrieve the firmware version of a device.
--
1.8.3.1
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] [PATCH v2 0/2] add abi version testing to app/test
2019-08-27 8:28 8% ` Ray Kinsella
@ 2019-08-27 14:19 7% ` Ray Kinsella
0 siblings, 0 replies; 200+ results
From: Ray Kinsella @ 2019-08-27 14:19 UTC (permalink / raw)
To: Bruce Richardson
Cc: Aaron Conole, dev, vladimir.medvedkin, john.mcnamara, marko.kovacevic
On 27/08/2019 09:28, Ray Kinsella wrote:
>
>
> On 27/08/2019 09:17, Bruce Richardson wrote:
>> On Mon, Aug 26, 2019 at 05:45:55PM +0100, Ray Kinsella wrote:
>>>
>>>
>>> On 23/08/2019 16:49, Aaron Conole wrote:
>>>> Ray Kinsella <mdr@ashroe.eu> writes:
>>>>
>>>>> This patchset adds ABI version testing to the app/test unit test framework,
>>>>> addressing two issues previously raised during ML conversations on ABI
>>>>> stability;
>>>>>
>>>>> 1. How do we unit test still supported previous ABI versions?
>>>>> 2. How to we unit test inline functions from still supported previous ABI
>>>>> versions?
>>>>>
>>>>> Starting with rte_lpm, I did the following:-
>>>>>
>>>>> * I reproduced mostly unmodified unit tests for the v2.0 ABI, taken from DPDK
>>>>> 2.2 and 17.02.
>>>>> * I reproduced the rte_lpm interface header from v2.0, including the inline
>>>>> functions and remapping symbols to their appropriate versions.
>>>>> * I added support for multiple abi versions to the app/test unit test framework
>>>>> to allow users to switch between abi versions (set_abi_version), without
>>>>> further polluting the already long list of unit tests available in app/test.
>>>>>
>>>>> The intention here is that in future as developers need to deprecate APIs, the
>>>>> associated unit tests may move into the ABI version testing mechanism of the
>>>>> app/test instead of being replaced by the latest set of unit tests as would be
>>>>> the case today.
>>>>>
>>>>> v2:
>>>>>
>>>>> * Added LPM IPv6 test cases for the v2.0 ABI.
>>>>> * Fixed a number of checkpatch errors, stop short of substantially reworking
>>>>> the test code from the v2.0 ABI.
>>>>> * Removed duplicating test cases published in the original v1 patch.
>>>>
>>>> Thanks for this work. I think it's useful.
>>>>
>>>> I see an error under aarch64 builds because there are some x86_64
>>>> specific types being used in the testing.
>>>
>>> So the problem is that LPM didn't fully support ARM until DPDK v16.04.
>>> The ABI versioning code in the LPM library is there to support the 2.0 ABI.
>>>
>>> The intention of this unit test is to test backward's compatibility with
>>> an inline LPM function from DPDK v2.2.0, which was essentially x86 only
>>> at that time.
>>>
>>> Unless we want to get into the business of backporting ARM support to
>>> DPDK 2.2.0 (from where this test cases came from) - we should probably
>>> restrict these ABI versioning test cases to CONFIG_RTE_ARCH_X86_64 only.
>>>
>>> The other option is forget about testing this the LPM ABI versioning
>>> support, which then asks the question should be perhaps excise that code
>>> altogether.
>>>
>>
>> I think function versioning is great and should be widely used.
>> Unfortunately, though, in our case since we break the ABI so consistently,
>> this old code is pretty useless. Therefore, I think we should remove all
>> old versionned code from e.g. pre-18.11, since no app is realistically
>> going to work from that far back anyway.
>>
>> /Bruce
>>
>
> I had come to a similar conclusion, that we likely need to deprecate
> much or all of the existing ABI Compatibility code, it needs a wider
> review.
>
> BIND_VERSION_SYMBOL and friends, are still needed to unit test ABI
> Versioning, the general idea is sound. And I liked LPM as an example,
> because it is well understood and contained, but I will look for
> something more recent we could use instead.
>
Only recent example I can find of ABI versioning is the Timer Library,
changed in April 2019. After that are the distributor and the lpm
library both changes in 2017, does this seem right?
Ray K
root@xxx:/build/dpdk# find lib -name *.map | xargs -I{} -- git log -1
--format="%ai {}" {} | sort -k 1,2 | tail -n 10
2019-04-03 18:20:13 -0500 lib/librte_stack/rte_stack_version.map
2019-04-15 16:41:28 -0500 lib/librte_timer/rte_timer_version.map
2019-04-30 22:54:16 -0500 lib/librte_rcu/rte_rcu_version.map
2019-06-25 04:46:02 +0530 lib/librte_eventdev/rte_eventdev_version.map
2019-07-05 10:16:17 -0700 lib/librte_net/rte_net_version.map
2019-07-11 09:26:05 +0000 lib/librte_metrics/rte_metrics_version.map
2019-07-17 03:23:55 +0800 lib/librte_ring/rte_ring_version.map
2019-07-26 15:10:19 +0100 lib/librte_security/rte_security_version.map
2019-07-27 09:21:33 +0200 lib/librte_eal/rte_eal_version.map
2019-07-31 14:27:16 +0200 lib/librte_ethdev/rte_ethdev_version.map
root@xxx:/build/dpdk# find lib -name *.map | xargs -I{} -- git log -1
--format="%ai {}" {} | sort -k 1,2 | tail -n 50 | awk '{print $4}' |
xargs sort | uniq -c | sort -k 1
...
2 rte_distributor_clear_returns;
2 rte_distributor_create;
2 rte_distributor_flush;
2 rte_distributor_get_pkt;
2 rte_distributor_poll_pkt;
2 rte_distributor_process;
2 rte_distributor_request_pkt;
2 rte_distributor_returned_pkts;
2 rte_distributor_return_pkt;
2 rte_lpm6_add;
2 rte_lpm6_is_rule_present;
2 rte_lpm6_lookup;
2 rte_lpm6_lookup_bulk_func;
2 rte_lpm_add;
2 rte_lpm_create;
2 rte_lpm_delete;
2 rte_lpm_delete_all;
2 rte_lpm_find_existing;
2 rte_lpm_free;
2 rte_lpm_is_rule_present;
2 rte_timer_dump_stats;
2 rte_timer_manage;
2 rte_timer_reset;
2 rte_timer_stop;
2 rte_timer_subsystem_init;
...
^ permalink raw reply [relevance 7%]
* Re: [dpdk-dev] [PATCH v2 0/2] add abi version testing to app/test
2019-08-27 8:17 7% ` Bruce Richardson
@ 2019-08-27 8:28 8% ` Ray Kinsella
2019-08-27 14:19 7% ` Ray Kinsella
0 siblings, 1 reply; 200+ results
From: Ray Kinsella @ 2019-08-27 8:28 UTC (permalink / raw)
To: Bruce Richardson
Cc: Aaron Conole, dev, vladimir.medvedkin, john.mcnamara, marko.kovacevic
On 27/08/2019 09:17, Bruce Richardson wrote:
> On Mon, Aug 26, 2019 at 05:45:55PM +0100, Ray Kinsella wrote:
>>
>>
>> On 23/08/2019 16:49, Aaron Conole wrote:
>>> Ray Kinsella <mdr@ashroe.eu> writes:
>>>
>>>> This patchset adds ABI version testing to the app/test unit test framework,
>>>> addressing two issues previously raised during ML conversations on ABI
>>>> stability;
>>>>
>>>> 1. How do we unit test still supported previous ABI versions?
>>>> 2. How to we unit test inline functions from still supported previous ABI
>>>> versions?
>>>>
>>>> Starting with rte_lpm, I did the following:-
>>>>
>>>> * I reproduced mostly unmodified unit tests for the v2.0 ABI, taken from DPDK
>>>> 2.2 and 17.02.
>>>> * I reproduced the rte_lpm interface header from v2.0, including the inline
>>>> functions and remapping symbols to their appropriate versions.
>>>> * I added support for multiple abi versions to the app/test unit test framework
>>>> to allow users to switch between abi versions (set_abi_version), without
>>>> further polluting the already long list of unit tests available in app/test.
>>>>
>>>> The intention here is that in future as developers need to deprecate APIs, the
>>>> associated unit tests may move into the ABI version testing mechanism of the
>>>> app/test instead of being replaced by the latest set of unit tests as would be
>>>> the case today.
>>>>
>>>> v2:
>>>>
>>>> * Added LPM IPv6 test cases for the v2.0 ABI.
>>>> * Fixed a number of checkpatch errors, stop short of substantially reworking
>>>> the test code from the v2.0 ABI.
>>>> * Removed duplicating test cases published in the original v1 patch.
>>>
>>> Thanks for this work. I think it's useful.
>>>
>>> I see an error under aarch64 builds because there are some x86_64
>>> specific types being used in the testing.
>>
>> So the problem is that LPM didn't fully support ARM until DPDK v16.04.
>> The ABI versioning code in the LPM library is there to support the 2.0 ABI.
>>
>> The intention of this unit test is to test backward's compatibility with
>> an inline LPM function from DPDK v2.2.0, which was essentially x86 only
>> at that time.
>>
>> Unless we want to get into the business of backporting ARM support to
>> DPDK 2.2.0 (from where this test cases came from) - we should probably
>> restrict these ABI versioning test cases to CONFIG_RTE_ARCH_X86_64 only.
>>
>> The other option is forget about testing this the LPM ABI versioning
>> support, which then asks the question should be perhaps excise that code
>> altogether.
>>
>
> I think function versioning is great and should be widely used.
> Unfortunately, though, in our case since we break the ABI so consistently,
> this old code is pretty useless. Therefore, I think we should remove all
> old versionned code from e.g. pre-18.11, since no app is realistically
> going to work from that far back anyway.
>
> /Bruce
>
I had come to a similar conclusion, that we likely need to deprecate
much or all of the existing ABI Compatibility code, it needs a wider
review.
BIND_VERSION_SYMBOL and friends, are still needed to unit test ABI
Versioning, the general idea is sound. And I liked LPM as an example,
because it is well understood and contained, but I will look for
something more recent we could use instead.
^ permalink raw reply [relevance 8%]
* Re: [dpdk-dev] [PATCH v2 0/2] add abi version testing to app/test
2019-08-26 16:45 9% ` Ray Kinsella
@ 2019-08-27 8:17 7% ` Bruce Richardson
2019-08-27 8:28 8% ` Ray Kinsella
0 siblings, 1 reply; 200+ results
From: Bruce Richardson @ 2019-08-27 8:17 UTC (permalink / raw)
To: Ray Kinsella
Cc: Aaron Conole, dev, vladimir.medvedkin, john.mcnamara, marko.kovacevic
On Mon, Aug 26, 2019 at 05:45:55PM +0100, Ray Kinsella wrote:
>
>
> On 23/08/2019 16:49, Aaron Conole wrote:
> > Ray Kinsella <mdr@ashroe.eu> writes:
> >
> >> This patchset adds ABI version testing to the app/test unit test framework,
> >> addressing two issues previously raised during ML conversations on ABI
> >> stability;
> >>
> >> 1. How do we unit test still supported previous ABI versions?
> >> 2. How to we unit test inline functions from still supported previous ABI
> >> versions?
> >>
> >> Starting with rte_lpm, I did the following:-
> >>
> >> * I reproduced mostly unmodified unit tests for the v2.0 ABI, taken from DPDK
> >> 2.2 and 17.02.
> >> * I reproduced the rte_lpm interface header from v2.0, including the inline
> >> functions and remapping symbols to their appropriate versions.
> >> * I added support for multiple abi versions to the app/test unit test framework
> >> to allow users to switch between abi versions (set_abi_version), without
> >> further polluting the already long list of unit tests available in app/test.
> >>
> >> The intention here is that in future as developers need to deprecate APIs, the
> >> associated unit tests may move into the ABI version testing mechanism of the
> >> app/test instead of being replaced by the latest set of unit tests as would be
> >> the case today.
> >>
> >> v2:
> >>
> >> * Added LPM IPv6 test cases for the v2.0 ABI.
> >> * Fixed a number of checkpatch errors, stop short of substantially reworking
> >> the test code from the v2.0 ABI.
> >> * Removed duplicating test cases published in the original v1 patch.
> >
> > Thanks for this work. I think it's useful.
> >
> > I see an error under aarch64 builds because there are some x86_64
> > specific types being used in the testing.
>
> So the problem is that LPM didn't fully support ARM until DPDK v16.04.
> The ABI versioning code in the LPM library is there to support the 2.0 ABI.
>
> The intention of this unit test is to test backward's compatibility with
> an inline LPM function from DPDK v2.2.0, which was essentially x86 only
> at that time.
>
> Unless we want to get into the business of backporting ARM support to
> DPDK 2.2.0 (from where this test cases came from) - we should probably
> restrict these ABI versioning test cases to CONFIG_RTE_ARCH_X86_64 only.
>
> The other option is forget about testing this the LPM ABI versioning
> support, which then asks the question should be perhaps excise that code
> altogether.
>
I think function versioning is great and should be widely used.
Unfortunately, though, in our case since we break the ABI so consistently,
this old code is pretty useless. Therefore, I think we should remove all
old versionned code from e.g. pre-18.11, since no app is realistically
going to work from that far back anyway.
/Bruce
^ permalink raw reply [relevance 7%]
* Re: [dpdk-dev] [PATCH v2 0/2] add abi version testing to app/test
2019-08-23 15:49 4% ` [dpdk-dev] [PATCH v2 0/2] add abi version testing to app/test Aaron Conole
@ 2019-08-26 16:45 9% ` Ray Kinsella
2019-08-27 8:17 7% ` Bruce Richardson
0 siblings, 1 reply; 200+ results
From: Ray Kinsella @ 2019-08-26 16:45 UTC (permalink / raw)
To: Aaron Conole
Cc: dev, bruce.richardson, vladimir.medvedkin, john.mcnamara,
marko.kovacevic
On 23/08/2019 16:49, Aaron Conole wrote:
> Ray Kinsella <mdr@ashroe.eu> writes:
>
>> This patchset adds ABI version testing to the app/test unit test framework,
>> addressing two issues previously raised during ML conversations on ABI
>> stability;
>>
>> 1. How do we unit test still supported previous ABI versions?
>> 2. How to we unit test inline functions from still supported previous ABI
>> versions?
>>
>> Starting with rte_lpm, I did the following:-
>>
>> * I reproduced mostly unmodified unit tests for the v2.0 ABI, taken from DPDK
>> 2.2 and 17.02.
>> * I reproduced the rte_lpm interface header from v2.0, including the inline
>> functions and remapping symbols to their appropriate versions.
>> * I added support for multiple abi versions to the app/test unit test framework
>> to allow users to switch between abi versions (set_abi_version), without
>> further polluting the already long list of unit tests available in app/test.
>>
>> The intention here is that in future as developers need to deprecate APIs, the
>> associated unit tests may move into the ABI version testing mechanism of the
>> app/test instead of being replaced by the latest set of unit tests as would be
>> the case today.
>>
>> v2:
>>
>> * Added LPM IPv6 test cases for the v2.0 ABI.
>> * Fixed a number of checkpatch errors, stop short of substantially reworking
>> the test code from the v2.0 ABI.
>> * Removed duplicating test cases published in the original v1 patch.
>
> Thanks for this work. I think it's useful.
>
> I see an error under aarch64 builds because there are some x86_64
> specific types being used in the testing.
So the problem is that LPM didn't fully support ARM until DPDK v16.04.
The ABI versioning code in the LPM library is there to support the 2.0 ABI.
The intention of this unit test is to test backward's compatibility with
an inline LPM function from DPDK v2.2.0, which was essentially x86 only
at that time.
Unless we want to get into the business of backporting ARM support to
DPDK 2.2.0 (from where this test cases came from) - we should probably
restrict these ABI versioning test cases to CONFIG_RTE_ARCH_X86_64 only.
The other option is forget about testing this the LPM ABI versioning
support, which then asks the question should be perhaps excise that code
altogether.
Make sense?
Ray K
^ permalink raw reply [relevance 9%]
* Re: [dpdk-dev] [PATCH 0/2] IXGBE vPMD changes for aarch64
2019-08-26 10:39 3% ` Ferruh Yigit
@ 2019-08-26 10:53 0% ` Ruifeng Wang (Arm Technology China)
0 siblings, 0 replies; 200+ results
From: Ruifeng Wang (Arm Technology China) @ 2019-08-26 10:53 UTC (permalink / raw)
To: Ferruh Yigit, Ye Xiaolong
Cc: jerinj, Gavin Hu (Arm Technology China),
dev, Honnappa Nagarahalli, nd, Kevin Traynor, Luca Boccassi, nd
> -----Original Message-----
> From: Ferruh Yigit <ferruh.yigit@intel.com>
> Sent: Monday, August 26, 2019 18:40
> To: Ruifeng Wang (Arm Technology China) <Ruifeng.Wang@arm.com>; Ye
> Xiaolong <xiaolong.ye@intel.com>
> Cc: jerinj@marvell.com; Gavin Hu (Arm Technology China)
> <Gavin.Hu@arm.com>; dev@dpdk.org; Honnappa Nagarahalli
> <Honnappa.Nagarahalli@arm.com>; nd <nd@arm.com>; Kevin Traynor
> <ktraynor@redhat.com>; Luca Boccassi <bluca@debian.org>
> Subject: Re: [dpdk-dev] [PATCH 0/2] IXGBE vPMD changes for aarch64
>
> On 8/26/2019 3:52 AM, Ruifeng Wang (Arm Technology China) wrote:
> > Hi Xiaolong,
> >
> >> -----Original Message-----
> >> From: Ye Xiaolong <xiaolong.ye@intel.com>
> >> Sent: Sunday, August 25, 2019 09:34
> >> To: Ruifeng Wang (Arm Technology China) <Ruifeng.Wang@arm.com>
> >> Cc: jerinj@marvell.com; Gavin Hu (Arm Technology China)
> >> <Gavin.Hu@arm.com>; dev@dpdk.org; Honnappa Nagarahalli
> >> <Honnappa.Nagarahalli@arm.com>; nd <nd@arm.com>
> >> Subject: Re: [dpdk-dev] [PATCH 0/2] IXGBE vPMD changes for aarch64
> >>
> >> Hi,
> >>
> >> Thanks for the patches, could you also provide the Fixes tag and cc stable?
> >> The patchset looks good to me.
> >
> > Code changes in both patches are not for bug fixing.
> > Patch 1/2 includes fix for code comments. I don't think it deserves a Fixes
> tag or backporting. Can we skip the Fixes tag?
>
> In 1/2 a memory barrier is removed, it means it was wrong to add it at first
> place and you are fixing it, no?
>
>
> Performance improvements are in gray are, but if there is no ABI/API break
> why not take is performance fix and backport and have the performance
> improvement in LTS?
> Also I think taking as much as possible may help to maintain LTS, since it
> reduces the chance of conflict in later commits, LTS is two years and these
> small things can accumulate and make getting important fixes hard by time.
>
> Is there any specific reason not to backport these patches to LTS releases?
>
Thanks for your explanation.
Understand that. No objection to backporting.
I'll send out new version.
>
> >
> >>
> >> Thanks,
> >> Xiaolong
> >>
> >> On 08/13, Ruifeng Wang wrote:
> >>> Couple of changes to IXGBE vector PMD on aarch64 platform.
> >>> An unnecessary memory barrier was identified and removed.
> >>> Also part of processing was replaced with NEON intrinsics.
> >>> Both of the changes will help to improve performance.
> >>>
> >>> Ruifeng Wang (2):
> >>> net/ixgbe: remove barrier in vPMD for aarch64
> >>> net/ixgbe: use neon intrinsics to count packet for aarch64
> >>>
> >>> drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c | 32
> >>> ++++++++++++-------------
> >>> 1 file changed, 16 insertions(+), 16 deletions(-)
> >>>
> >>> --
> >>> 2.17.1
> >>>
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [PATCH 0/2] IXGBE vPMD changes for aarch64
@ 2019-08-26 10:39 3% ` Ferruh Yigit
2019-08-26 10:53 0% ` Ruifeng Wang (Arm Technology China)
0 siblings, 1 reply; 200+ results
From: Ferruh Yigit @ 2019-08-26 10:39 UTC (permalink / raw)
To: Ruifeng Wang (Arm Technology China), Ye Xiaolong
Cc: jerinj, Gavin Hu (Arm Technology China),
dev, Honnappa Nagarahalli, nd, Kevin Traynor, Luca Boccassi
On 8/26/2019 3:52 AM, Ruifeng Wang (Arm Technology China) wrote:
> Hi Xiaolong,
>
>> -----Original Message-----
>> From: Ye Xiaolong <xiaolong.ye@intel.com>
>> Sent: Sunday, August 25, 2019 09:34
>> To: Ruifeng Wang (Arm Technology China) <Ruifeng.Wang@arm.com>
>> Cc: jerinj@marvell.com; Gavin Hu (Arm Technology China)
>> <Gavin.Hu@arm.com>; dev@dpdk.org; Honnappa Nagarahalli
>> <Honnappa.Nagarahalli@arm.com>; nd <nd@arm.com>
>> Subject: Re: [dpdk-dev] [PATCH 0/2] IXGBE vPMD changes for aarch64
>>
>> Hi,
>>
>> Thanks for the patches, could you also provide the Fixes tag and cc stable?
>> The patchset looks good to me.
>
> Code changes in both patches are not for bug fixing.
> Patch 1/2 includes fix for code comments. I don't think it deserves a Fixes tag or backporting. Can we skip the Fixes tag?
In 1/2 a memory barrier is removed, it means it was wrong to add it at first
place and you are fixing it, no?
Performance improvements are in gray are, but if there is no ABI/API break why
not take is performance fix and backport and have the performance improvement in
LTS?
Also I think taking as much as possible may help to maintain LTS, since it
reduces the chance of conflict in later commits, LTS is two years and these
small things can accumulate and make getting important fixes hard by time.
Is there any specific reason not to backport these patches to LTS releases?
>
>>
>> Thanks,
>> Xiaolong
>>
>> On 08/13, Ruifeng Wang wrote:
>>> Couple of changes to IXGBE vector PMD on aarch64 platform.
>>> An unnecessary memory barrier was identified and removed.
>>> Also part of processing was replaced with NEON intrinsics.
>>> Both of the changes will help to improve performance.
>>>
>>> Ruifeng Wang (2):
>>> net/ixgbe: remove barrier in vPMD for aarch64
>>> net/ixgbe: use neon intrinsics to count packet for aarch64
>>>
>>> drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c | 32 ++++++++++++-------------
>>> 1 file changed, 16 insertions(+), 16 deletions(-)
>>>
>>> --
>>> 2.17.1
>>>
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] [PATCH v2 0/2] add abi version testing to app/test
2019-08-22 16:07 9% [dpdk-dev] [PATCH v2 0/2] add abi version testing to app/test Ray Kinsella
2019-08-22 16:07 14% ` [dpdk-dev] [PATCH v2 1/2] app/test: add abi version testing functionality Ray Kinsella
2019-08-22 16:07 4% ` [dpdk-dev] [PATCH v2 2/2] app/test: lpm abi version testing Ray Kinsella
@ 2019-08-23 15:49 4% ` Aaron Conole
2019-08-26 16:45 9% ` Ray Kinsella
2 siblings, 1 reply; 200+ results
From: Aaron Conole @ 2019-08-23 15:49 UTC (permalink / raw)
To: Ray Kinsella
Cc: dev, bruce.richardson, vladimir.medvedkin, john.mcnamara,
marko.kovacevic
Ray Kinsella <mdr@ashroe.eu> writes:
> This patchset adds ABI version testing to the app/test unit test framework,
> addressing two issues previously raised during ML conversations on ABI
> stability;
>
> 1. How do we unit test still supported previous ABI versions?
> 2. How to we unit test inline functions from still supported previous ABI
> versions?
>
> Starting with rte_lpm, I did the following:-
>
> * I reproduced mostly unmodified unit tests for the v2.0 ABI, taken from DPDK
> 2.2 and 17.02.
> * I reproduced the rte_lpm interface header from v2.0, including the inline
> functions and remapping symbols to their appropriate versions.
> * I added support for multiple abi versions to the app/test unit test framework
> to allow users to switch between abi versions (set_abi_version), without
> further polluting the already long list of unit tests available in app/test.
>
> The intention here is that in future as developers need to deprecate APIs, the
> associated unit tests may move into the ABI version testing mechanism of the
> app/test instead of being replaced by the latest set of unit tests as would be
> the case today.
>
> v2:
>
> * Added LPM IPv6 test cases for the v2.0 ABI.
> * Fixed a number of checkpatch errors, stop short of substantially reworking
> the test code from the v2.0 ABI.
> * Removed duplicating test cases published in the original v1 patch.
Thanks for this work. I think it's useful.
I see an error under aarch64 builds because there are some x86_64
specific types being used in the testing.
See:
https://travis-ci.com/ovsrobot/dpdk/builds/124254699
> Ray Kinsella (2):
> app/test: add abi version testing functionality
> app/test: lpm abi version testing
>
> app/test/Makefile | 12 +-
> app/test/commands.c | 131 +-
> app/test/meson.build | 6 +
> app/test/test.c | 2 +
> app/test/test.h | 48 +-
> app/test/test_lpm.c | 3 +-
> app/test/test_lpm6.c | 2 +-
> app/test/test_lpm_perf.c | 293 +---
> app/test/test_lpm_routes.c | 287 ++++
> app/test/test_lpm_routes.h | 25 +
> app/test/v2.0/dcompat.h | 30 +
> app/test/v2.0/rte_lpm.h | 451 +++++
> app/test/v2.0/rte_lpm6.h | 198 +++
> app/test/v2.0/test_lpm.c | 1139 +++++++++++++
> app/test/v2.0/test_lpm6.c | 1748 ++++++++++++++++++++
> app/test/v2.0/test_lpm6_perf.c | 179 ++
> app/test/v2.0/test_lpm_perf.c | 212 +++
> app/test/v2.0/test_v20.c | 14 +
> doc/guides/contributing/versioning.rst | 4 +
> lib/librte_eal/common/include/rte_compat.h | 7 +
> 20 files changed, 4471 insertions(+), 320 deletions(-)
> create mode 100644 app/test/test_lpm_routes.c
> create mode 100644 app/test/test_lpm_routes.h
> create mode 100644 app/test/v2.0/dcompat.h
> create mode 100644 app/test/v2.0/rte_lpm.h
> create mode 100644 app/test/v2.0/rte_lpm6.h
> create mode 100644 app/test/v2.0/test_lpm.c
> create mode 100644 app/test/v2.0/test_lpm6.c
> create mode 100644 app/test/v2.0/test_lpm6_perf.c
> create mode 100644 app/test/v2.0/test_lpm_perf.c
> create mode 100644 app/test/v2.0/test_v20.c
^ permalink raw reply [relevance 4%]
* [dpdk-dev] [PATCH 15/15] sched: remove redundant code
@ 2019-08-23 14:46 4% ` Jasvinder Singh
1 sibling, 0 replies; 200+ results
From: Jasvinder Singh @ 2019-08-23 14:46 UTC (permalink / raw)
To: dev; +Cc: cristian.dumitrescu
Remove redundant data structure fields from port level data
structures and update release notes.
---
doc/guides/rel_notes/release_19_11.rst | 6 +++-
lib/librte_sched/rte_sched.c | 43 +-------------------------
lib/librte_sched/rte_sched.h | 22 -------------
3 files changed, 6 insertions(+), 65 deletions(-)
diff --git a/doc/guides/rel_notes/release_19_11.rst b/doc/guides/rel_notes/release_19_11.rst
index 8490d897c..746aeb0ea 100644
--- a/doc/guides/rel_notes/release_19_11.rst
+++ b/doc/guides/rel_notes/release_19_11.rst
@@ -85,6 +85,10 @@ API Changes
Also, make sure to start the actual text at the margin.
=========================================================
+* sched: The pipe nodes configuration parameters such as number of pipes,
+ pipe queue sizes, pipe profiles, etc., are moved from port level structure
+ to subport level. This allows different subports of the same port to
+ have different configuration for the pipe nodes.
ABI Changes
-----------
@@ -172,7 +176,7 @@ The libraries prepended with a plus sign were incremented in this version.
librte_rcu.so.1
librte_reorder.so.1
librte_ring.so.2
- librte_sched.so.3
+ + librte_sched.so.4
librte_security.so.2
librte_stack.so.1
librte_table.so.3
diff --git a/lib/librte_sched/rte_sched.c b/lib/librte_sched/rte_sched.c
index 8a7727286..ba504da1e 100644
--- a/lib/librte_sched/rte_sched.c
+++ b/lib/librte_sched/rte_sched.c
@@ -207,10 +207,8 @@ struct rte_sched_subport {
struct rte_sched_port {
/* User parameters */
uint32_t n_subports_per_port;
- uint32_t n_pipes_per_subport;
uint32_t n_max_pipes_per_subport;
uint32_t n_max_pipes_per_subport_log2;
- uint32_t n_pipes_per_subport_log2;
uint16_t pipe_queue[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
uint8_t pipe_tc[RTE_SCHED_QUEUES_PER_PIPE];
uint8_t tc_queue[RTE_SCHED_QUEUES_PER_PIPE];
@@ -218,13 +216,7 @@ struct rte_sched_port {
uint32_t mtu;
uint32_t frame_overhead;
int socket;
- uint16_t qsize[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
- uint32_t n_pipe_profiles;
uint32_t n_max_pipe_profiles;
- uint32_t pipe_tc_be_rate_max;
-#ifdef RTE_SCHED_RED
- struct rte_red_config red_config[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE][RTE_COLORS];
-#endif
/* Timing */
uint64_t time_cpu_cycles; /* Current CPU time measured in CPU cyles */
@@ -232,48 +224,15 @@ struct rte_sched_port {
uint64_t time; /* Current NIC TX time measured in bytes */
struct rte_reciprocal inv_cycles_per_byte; /* CPU cycles per byte */
- /* Scheduling loop detection */
- uint32_t pipe_loop;
- uint32_t pipe_exhaustion;
-
- /* Bitmap */
- struct rte_bitmap *bmp;
- uint32_t grinder_base_bmp_pos[RTE_SCHED_PORT_N_GRINDERS] __rte_aligned_16;
-
/* Grinders */
- struct rte_sched_grinder grinder[RTE_SCHED_PORT_N_GRINDERS];
- uint32_t busy_grinders;
struct rte_mbuf **pkts_out;
uint32_t n_pkts_out;
uint32_t subport_id;
- /* Queue base calculation */
- uint32_t qsize_add[RTE_SCHED_QUEUES_PER_PIPE];
- uint32_t qsize_sum;
-
/* Large data structures */
- struct rte_sched_subport *subports[0];
- struct rte_sched_subport *subport;
- struct rte_sched_pipe *pipe;
- struct rte_sched_queue *queue;
- struct rte_sched_queue_extra *queue_extra;
- struct rte_sched_pipe_profile *pipe_profiles;
- uint8_t *bmp_array;
- struct rte_mbuf **queue_array;
- uint8_t memory[0] __rte_cache_aligned;
+ struct rte_sched_subport *subports[0] __rte_cache_aligned;
} __rte_cache_aligned;
-enum rte_sched_port_array {
- e_RTE_SCHED_PORT_ARRAY_SUBPORT = 0,
- e_RTE_SCHED_PORT_ARRAY_PIPE,
- e_RTE_SCHED_PORT_ARRAY_QUEUE,
- e_RTE_SCHED_PORT_ARRAY_QUEUE_EXTRA,
- e_RTE_SCHED_PORT_ARRAY_PIPE_PROFILES,
- e_RTE_SCHED_PORT_ARRAY_BMP_ARRAY,
- e_RTE_SCHED_PORT_ARRAY_QUEUE_ARRAY,
- e_RTE_SCHED_PORT_ARRAY_TOTAL,
-};
-
enum rte_sched_subport_array {
e_RTE_SCHED_SUBPORT_ARRAY_PIPE = 0,
e_RTE_SCHED_SUBPORT_ARRAY_QUEUE,
diff --git a/lib/librte_sched/rte_sched.h b/lib/librte_sched/rte_sched.h
index ea2b07448..db1b0232f 100644
--- a/lib/librte_sched/rte_sched.h
+++ b/lib/librte_sched/rte_sched.h
@@ -246,33 +246,11 @@ struct rte_sched_port_params {
/** Number of subports */
uint32_t n_subports_per_port;
- /** Number of subport_pipes */
- uint32_t n_pipes_per_subport;
-
/** Maximum number of subport_pipes */
uint32_t n_max_pipes_per_subport;
- /** Packet queue size for each traffic class.
- * All the pipes within the same subport share the similar
- * configuration for the queues.
- */
- uint16_t qsize[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
-
- /** Pipe profile table.
- * Every pipe is configured using one of the profiles from this table.
- */
- struct rte_sched_pipe_params *pipe_profiles;
-
- /** Profiles in the pipe profile table */
- uint32_t n_pipe_profiles;
-
/** Max profiles allowed in the pipe profile table */
uint32_t n_max_pipe_profiles;
-
-#ifdef RTE_SCHED_RED
- /** RED parameters */
- struct rte_red_params red_params[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE][RTE_COLORS];
-#endif
};
/*
--
2.21.0
^ permalink raw reply [relevance 4%]
* [dpdk-dev] [PATCH v2 2/2] app/test: lpm abi version testing
2019-08-22 16:07 9% [dpdk-dev] [PATCH v2 0/2] add abi version testing to app/test Ray Kinsella
2019-08-22 16:07 14% ` [dpdk-dev] [PATCH v2 1/2] app/test: add abi version testing functionality Ray Kinsella
@ 2019-08-22 16:07 4% ` Ray Kinsella
2019-08-23 15:49 4% ` [dpdk-dev] [PATCH v2 0/2] add abi version testing to app/test Aaron Conole
2 siblings, 0 replies; 200+ results
From: Ray Kinsella @ 2019-08-22 16:07 UTC (permalink / raw)
To: dev
Cc: mdr, bruce.richardson, vladimir.medvedkin, john.mcnamara,
marko.kovacevic
This second patch adds the LPM ABI version Unit Tests, comprised of
1. Registering DPDK v2.0 ABI versions with the infrastructure.
2. Forward Porting the DPDK v2.0 LPM Unit Test cases, remapping the LPM
Library symbols to the appropriate versions. The unit tests are forward
ported more or less on an as-it-was basis.
3. Refactoring the lpm perf routes table to make this functionality
available to the v2.0 unit tests, forwarding porting this code also from
v2.0 would have increased the DPDK codebase several MLoC.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
app/test/Makefile | 12 +-
app/test/meson.build | 6 +
app/test/test.h | 4 -
app/test/test_lpm.c | 3 +-
app/test/test_lpm6.c | 2 +-
app/test/test_lpm_perf.c | 293 +-----
app/test/test_lpm_routes.c | 287 ++++++
app/test/test_lpm_routes.h | 25 +
app/test/v2.0/dcompat.h | 30 +
app/test/v2.0/rte_lpm.h | 451 ++++++++
app/test/v2.0/rte_lpm6.h | 198 ++++
app/test/v2.0/test_lpm.c | 1139 +++++++++++++++++++++
app/test/v2.0/test_lpm6.c | 1748 ++++++++++++++++++++++++++++++++
app/test/v2.0/test_lpm6_perf.c | 179 ++++
app/test/v2.0/test_lpm_perf.c | 212 ++++
app/test/v2.0/test_v20.c | 14 +
16 files changed, 4305 insertions(+), 298 deletions(-)
create mode 100644 app/test/test_lpm_routes.c
create mode 100644 app/test/test_lpm_routes.h
create mode 100644 app/test/v2.0/dcompat.h
create mode 100644 app/test/v2.0/rte_lpm.h
create mode 100644 app/test/v2.0/rte_lpm6.h
create mode 100644 app/test/v2.0/test_lpm.c
create mode 100644 app/test/v2.0/test_lpm6.c
create mode 100644 app/test/v2.0/test_lpm6_perf.c
create mode 100644 app/test/v2.0/test_lpm_perf.c
create mode 100644 app/test/v2.0/test_v20.c
diff --git a/app/test/Makefile b/app/test/Makefile
index 26ba6fe2b..8b2e0ca61 100644
--- a/app/test/Makefile
+++ b/app/test/Makefile
@@ -80,6 +80,8 @@ SRCS-y += test_ring.c
SRCS-y += test_ring_perf.c
SRCS-y += test_pmd_perf.c
+#ABI Version Testing
+SRCS-$(CONFIG_RTE_BUILD_SHARED_LIB) += v2.0/test_v20.c
ifeq ($(CONFIG_RTE_LIBRTE_TABLE),y)
SRCS-y += test_table.c
SRCS-$(CONFIG_RTE_LIBRTE_PIPELINE) += test_table_pipeline.c
@@ -109,7 +111,6 @@ SRCS-y += test_logs.c
SRCS-y += test_memcpy.c
SRCS-y += test_memcpy_perf.c
-
SRCS-$(CONFIG_RTE_LIBRTE_MEMBER) += test_member.c
SRCS-$(CONFIG_RTE_LIBRTE_MEMBER) += test_member_perf.c
@@ -124,11 +125,20 @@ SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_multiwriter.c
SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_readwrite.c
SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_readwrite_lf.c
+SRCS-$(CONFIG_RTE_LIBRTE_LPM) += test_lpm_routes.c
SRCS-$(CONFIG_RTE_LIBRTE_LPM) += test_lpm.c
SRCS-$(CONFIG_RTE_LIBRTE_LPM) += test_lpm_perf.c
SRCS-$(CONFIG_RTE_LIBRTE_LPM) += test_lpm6.c
SRCS-$(CONFIG_RTE_LIBRTE_LPM) += test_lpm6_perf.c
+#LPM ABI Testing
+ifeq ($(CONFIG_RTE_BUILD_SHARED_LIB),y)
+SRCS-$(CONFIG_RTE_LIBRTE_LPM) += v2.0/test_lpm.c
+SRCS-$(CONFIG_RTE_LIBRTE_LPM) += v2.0/test_lpm_perf.c
+SRCS-$(CONFIG_RTE_LIBRTE_LPM) += v2.0/test_lpm6.c
+SRCS-$(CONFIG_RTE_LIBRTE_LPM) += v2.0/test_lpm6_perf.c
+endif
+
SRCS-y += test_debug.c
SRCS-y += test_errno.c
SRCS-y += test_tailq.c
diff --git a/app/test/meson.build b/app/test/meson.build
index ec40943bd..a0c8d4611 100644
--- a/app/test/meson.build
+++ b/app/test/meson.build
@@ -8,6 +8,7 @@ endif
test_sources = files('commands.c',
'packet_burst_generator.c',
'sample_packet_forward.c',
+ 'v2.0/test_v20.c',
'test.c',
'test_acl.c',
'test_alarm.c',
@@ -68,6 +69,11 @@ test_sources = files('commands.c',
'test_lpm6.c',
'test_lpm6_perf.c',
'test_lpm_perf.c',
+ 'test_lpm_routes.c',
+ 'v2.0/test_lpm.c',
+ 'v2.0/test_lpm_perf.c',
+ 'v2.0/test_lpm6.c',
+ 'v2.0/test_lpm6_perf.c',
'test_malloc.c',
'test_mbuf.c',
'test_member.c',
diff --git a/app/test/test.h b/app/test/test.h
index 5ec3728d0..dc95c4059 100644
--- a/app/test/test.h
+++ b/app/test/test.h
@@ -162,11 +162,7 @@ int test_set_rxtx_conf(cmdline_fixed_string_t mode);
int test_set_rxtx_anchor(cmdline_fixed_string_t type);
int test_set_rxtx_sc(cmdline_fixed_string_t type);
-#define MAP_ABI_SYMBOL_VERSION(name, abi_version) \
- __asm(".symver "RTE_STR(name)","RTE_STR(name)"@"RTE_STR(abi_version))
-
#define TEST_DPDK_ABI_VERSION_DEFAULT 0
-#define TEST_DPDK_ABI_VERSION_V1604 1
#define TEST_DPDK_ABI_VERSION_V20 2
#define TEST_DPDK_ABI_VERSION_MAX 3
diff --git a/app/test/test_lpm.c b/app/test/test_lpm.c
index e969fe051..0a3233220 100644
--- a/app/test/test_lpm.c
+++ b/app/test/test_lpm.c
@@ -41,7 +41,7 @@ static int32_t test16(void);
static int32_t test17(void);
static int32_t test18(void);
-rte_lpm_test tests[] = {
+static rte_lpm_test tests[] = {
/* Test Cases */
test0,
test1,
@@ -1277,6 +1277,7 @@ test_lpm(void)
int status, global_status = 0;
for (i = 0; i < NUM_LPM_TESTS; i++) {
+ printf("# test %02d\n", i);
status = tests[i]();
if (status < 0) {
printf("ERROR: LPM Test %u: FAIL\n", i);
diff --git a/app/test/test_lpm6.c b/app/test/test_lpm6.c
index 670aadb40..474e4014c 100644
--- a/app/test/test_lpm6.c
+++ b/app/test/test_lpm6.c
@@ -52,7 +52,7 @@ static int32_t test26(void);
static int32_t test27(void);
static int32_t test28(void);
-rte_lpm6_test tests6[] = {
+static rte_lpm6_test tests6[] = {
/* Test Cases */
test0,
test1,
diff --git a/app/test/test_lpm_perf.c b/app/test/test_lpm_perf.c
index 77eea66ad..a6b8b35c2 100644
--- a/app/test/test_lpm_perf.c
+++ b/app/test/test_lpm_perf.c
@@ -5,7 +5,6 @@
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
-#include <math.h>
#include <rte_cycles.h>
#include <rte_random.h>
@@ -13,6 +12,7 @@
#include <rte_ip.h>
#include <rte_lpm.h>
+#include "test_lpm_routes.h"
#include "test.h"
#include "test_xmmt_ops.h"
@@ -27,295 +27,6 @@
#define BATCH_SIZE (1 << 12)
#define BULK_SIZE 32
-#define MAX_RULE_NUM (1200000)
-
-struct route_rule {
- uint32_t ip;
- uint8_t depth;
-};
-
-struct route_rule large_route_table[MAX_RULE_NUM];
-
-static uint32_t num_route_entries;
-#define NUM_ROUTE_ENTRIES num_route_entries
-
-enum {
- IP_CLASS_A,
- IP_CLASS_B,
- IP_CLASS_C
-};
-
-/* struct route_rule_count defines the total number of rules in following a/b/c
- * each item in a[]/b[]/c[] is the number of common IP address class A/B/C, not
- * including the ones for private local network.
- */
-struct route_rule_count {
- uint32_t a[RTE_LPM_MAX_DEPTH];
- uint32_t b[RTE_LPM_MAX_DEPTH];
- uint32_t c[RTE_LPM_MAX_DEPTH];
-};
-
-/* All following numbers of each depth of each common IP class are just
- * got from previous large constant table in app/test/test_lpm_routes.h .
- * In order to match similar performance, they keep same depth and IP
- * address coverage as previous constant table. These numbers don't
- * include any private local IP address. As previous large const rule
- * table was just dumped from a real router, there are no any IP address
- * in class C or D.
- */
-static struct route_rule_count rule_count = {
- .a = { /* IP class A in which the most significant bit is 0 */
- 0, /* depth = 1 */
- 0, /* depth = 2 */
- 1, /* depth = 3 */
- 0, /* depth = 4 */
- 2, /* depth = 5 */
- 1, /* depth = 6 */
- 3, /* depth = 7 */
- 185, /* depth = 8 */
- 26, /* depth = 9 */
- 16, /* depth = 10 */
- 39, /* depth = 11 */
- 144, /* depth = 12 */
- 233, /* depth = 13 */
- 528, /* depth = 14 */
- 866, /* depth = 15 */
- 3856, /* depth = 16 */
- 3268, /* depth = 17 */
- 5662, /* depth = 18 */
- 17301, /* depth = 19 */
- 22226, /* depth = 20 */
- 11147, /* depth = 21 */
- 16746, /* depth = 22 */
- 17120, /* depth = 23 */
- 77578, /* depth = 24 */
- 401, /* depth = 25 */
- 656, /* depth = 26 */
- 1107, /* depth = 27 */
- 1121, /* depth = 28 */
- 2316, /* depth = 29 */
- 717, /* depth = 30 */
- 10, /* depth = 31 */
- 66 /* depth = 32 */
- },
- .b = { /* IP class A in which the most 2 significant bits are 10 */
- 0, /* depth = 1 */
- 0, /* depth = 2 */
- 0, /* depth = 3 */
- 0, /* depth = 4 */
- 1, /* depth = 5 */
- 1, /* depth = 6 */
- 1, /* depth = 7 */
- 3, /* depth = 8 */
- 3, /* depth = 9 */
- 30, /* depth = 10 */
- 25, /* depth = 11 */
- 168, /* depth = 12 */
- 305, /* depth = 13 */
- 569, /* depth = 14 */
- 1129, /* depth = 15 */
- 50800, /* depth = 16 */
- 1645, /* depth = 17 */
- 1820, /* depth = 18 */
- 3506, /* depth = 19 */
- 3258, /* depth = 20 */
- 3424, /* depth = 21 */
- 4971, /* depth = 22 */
- 6885, /* depth = 23 */
- 39771, /* depth = 24 */
- 424, /* depth = 25 */
- 170, /* depth = 26 */
- 433, /* depth = 27 */
- 92, /* depth = 28 */
- 366, /* depth = 29 */
- 377, /* depth = 30 */
- 2, /* depth = 31 */
- 200 /* depth = 32 */
- },
- .c = { /* IP class A in which the most 3 significant bits are 110 */
- 0, /* depth = 1 */
- 0, /* depth = 2 */
- 0, /* depth = 3 */
- 0, /* depth = 4 */
- 0, /* depth = 5 */
- 0, /* depth = 6 */
- 0, /* depth = 7 */
- 12, /* depth = 8 */
- 8, /* depth = 9 */
- 9, /* depth = 10 */
- 33, /* depth = 11 */
- 69, /* depth = 12 */
- 237, /* depth = 13 */
- 1007, /* depth = 14 */
- 1717, /* depth = 15 */
- 14663, /* depth = 16 */
- 8070, /* depth = 17 */
- 16185, /* depth = 18 */
- 48261, /* depth = 19 */
- 36870, /* depth = 20 */
- 33960, /* depth = 21 */
- 50638, /* depth = 22 */
- 61422, /* depth = 23 */
- 466549, /* depth = 24 */
- 1829, /* depth = 25 */
- 4824, /* depth = 26 */
- 4927, /* depth = 27 */
- 5914, /* depth = 28 */
- 10254, /* depth = 29 */
- 4905, /* depth = 30 */
- 1, /* depth = 31 */
- 716 /* depth = 32 */
- }
-};
-
-static void generate_random_rule_prefix(uint32_t ip_class, uint8_t depth)
-{
-/* IP address class A, the most significant bit is 0 */
-#define IP_HEAD_MASK_A 0x00000000
-#define IP_HEAD_BIT_NUM_A 1
-
-/* IP address class B, the most significant 2 bits are 10 */
-#define IP_HEAD_MASK_B 0x80000000
-#define IP_HEAD_BIT_NUM_B 2
-
-/* IP address class C, the most significant 3 bits are 110 */
-#define IP_HEAD_MASK_C 0xC0000000
-#define IP_HEAD_BIT_NUM_C 3
-
- uint32_t class_depth;
- uint32_t range;
- uint32_t mask;
- uint32_t step;
- uint32_t start;
- uint32_t fixed_bit_num;
- uint32_t ip_head_mask;
- uint32_t rule_num;
- uint32_t k;
- struct route_rule *ptr_rule;
-
- if (ip_class == IP_CLASS_A) { /* IP Address class A */
- fixed_bit_num = IP_HEAD_BIT_NUM_A;
- ip_head_mask = IP_HEAD_MASK_A;
- rule_num = rule_count.a[depth - 1];
- } else if (ip_class == IP_CLASS_B) { /* IP Address class B */
- fixed_bit_num = IP_HEAD_BIT_NUM_B;
- ip_head_mask = IP_HEAD_MASK_B;
- rule_num = rule_count.b[depth - 1];
- } else { /* IP Address class C */
- fixed_bit_num = IP_HEAD_BIT_NUM_C;
- ip_head_mask = IP_HEAD_MASK_C;
- rule_num = rule_count.c[depth - 1];
- }
-
- if (rule_num == 0)
- return;
-
- /* the number of rest bits which don't include the most significant
- * fixed bits for this IP address class
- */
- class_depth = depth - fixed_bit_num;
-
- /* range is the maximum number of rules for this depth and
- * this IP address class
- */
- range = 1 << class_depth;
-
- /* only mask the most depth significant generated bits
- * except fixed bits for IP address class
- */
- mask = range - 1;
-
- /* Widen coverage of IP address in generated rules */
- if (range <= rule_num)
- step = 1;
- else
- step = round((double)range / rule_num);
-
- /* Only generate rest bits except the most significant
- * fixed bits for IP address class
- */
- start = lrand48() & mask;
- ptr_rule = &large_route_table[num_route_entries];
- for (k = 0; k < rule_num; k++) {
- ptr_rule->ip = (start << (RTE_LPM_MAX_DEPTH - depth))
- | ip_head_mask;
- ptr_rule->depth = depth;
- ptr_rule++;
- start = (start + step) & mask;
- }
- num_route_entries += rule_num;
-}
-
-static void insert_rule_in_random_pos(uint32_t ip, uint8_t depth)
-{
- uint32_t pos;
- int try_count = 0;
- struct route_rule tmp;
-
- do {
- pos = lrand48();
- try_count++;
- } while ((try_count < 10) && (pos > num_route_entries));
-
- if ((pos > num_route_entries) || (pos >= MAX_RULE_NUM))
- pos = num_route_entries >> 1;
-
- tmp = large_route_table[pos];
- large_route_table[pos].ip = ip;
- large_route_table[pos].depth = depth;
- if (num_route_entries < MAX_RULE_NUM)
- large_route_table[num_route_entries++] = tmp;
-}
-
-static void generate_large_route_rule_table(void)
-{
- uint32_t ip_class;
- uint8_t depth;
-
- num_route_entries = 0;
- memset(large_route_table, 0, sizeof(large_route_table));
-
- for (ip_class = IP_CLASS_A; ip_class <= IP_CLASS_C; ip_class++) {
- for (depth = 1; depth <= RTE_LPM_MAX_DEPTH; depth++) {
- generate_random_rule_prefix(ip_class, depth);
- }
- }
-
- /* Add following rules to keep same as previous large constant table,
- * they are 4 rules with private local IP address and 1 all-zeros prefix
- * with depth = 8.
- */
- insert_rule_in_random_pos(RTE_IPV4(0, 0, 0, 0), 8);
- insert_rule_in_random_pos(RTE_IPV4(10, 2, 23, 147), 32);
- insert_rule_in_random_pos(RTE_IPV4(192, 168, 100, 10), 24);
- insert_rule_in_random_pos(RTE_IPV4(192, 168, 25, 100), 24);
- insert_rule_in_random_pos(RTE_IPV4(192, 168, 129, 124), 32);
-}
-
-static void
-print_route_distribution(const struct route_rule *table, uint32_t n)
-{
- unsigned i, j;
-
- printf("Route distribution per prefix width: \n");
- printf("DEPTH QUANTITY (PERCENT)\n");
- printf("--------------------------- \n");
-
- /* Count depths. */
- for (i = 1; i <= 32; i++) {
- unsigned depth_counter = 0;
- double percent_hits;
-
- for (j = 0; j < n; j++)
- if (table[j].depth == (uint8_t) i)
- depth_counter++;
-
- percent_hits = ((double)depth_counter)/((double)n) * 100;
- printf("%.2u%15u (%.2f)\n", i, depth_counter, percent_hits);
- }
- printf("\n");
-}
-
static int
test_lpm_perf(void)
{
@@ -375,7 +86,7 @@ test_lpm_perf(void)
(unsigned) cache_line_counter, (unsigned) cache_line_counter * 64);
printf("Average LPM Add: %g cycles\n",
- (double)total_time / NUM_ROUTE_ENTRIES);
+ (double)total_time / NUM_ROUTE_ENTRIES);
/* Measure single Lookup */
total_time = 0;
diff --git a/app/test/test_lpm_routes.c b/app/test/test_lpm_routes.c
new file mode 100644
index 000000000..edd1eba1b
--- /dev/null
+++ b/app/test/test_lpm_routes.c
@@ -0,0 +1,287 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2019 Intel Corporation
+ */
+
+#include <math.h>
+
+#include "rte_lpm.h"
+#include "test_lpm_routes.h"
+
+uint32_t num_route_entries;
+struct route_rule large_route_table[MAX_RULE_NUM];
+
+enum {
+ IP_CLASS_A,
+ IP_CLASS_B,
+ IP_CLASS_C
+};
+
+/* struct route_rule_count defines the total number of rules in following a/b/c
+ * each item in a[]/b[]/c[] is the number of common IP address class A/B/C, not
+ * including the ones for private local network.
+ */
+struct route_rule_count {
+ uint32_t a[RTE_LPM_MAX_DEPTH];
+ uint32_t b[RTE_LPM_MAX_DEPTH];
+ uint32_t c[RTE_LPM_MAX_DEPTH];
+};
+
+/* All following numbers of each depth of each common IP class are just
+ * got from previous large constant table in app/test/test_lpm_routes.h .
+ * In order to match similar performance, they keep same depth and IP
+ * address coverage as previous constant table. These numbers don't
+ * include any private local IP address. As previous large const rule
+ * table was just dumped from a real router, there are no any IP address
+ * in class C or D.
+ */
+static struct route_rule_count rule_count = {
+ .a = { /* IP class A in which the most significant bit is 0 */
+ 0, /* depth = 1 */
+ 0, /* depth = 2 */
+ 1, /* depth = 3 */
+ 0, /* depth = 4 */
+ 2, /* depth = 5 */
+ 1, /* depth = 6 */
+ 3, /* depth = 7 */
+ 185, /* depth = 8 */
+ 26, /* depth = 9 */
+ 16, /* depth = 10 */
+ 39, /* depth = 11 */
+ 144, /* depth = 12 */
+ 233, /* depth = 13 */
+ 528, /* depth = 14 */
+ 866, /* depth = 15 */
+ 3856, /* depth = 16 */
+ 3268, /* depth = 17 */
+ 5662, /* depth = 18 */
+ 17301, /* depth = 19 */
+ 22226, /* depth = 20 */
+ 11147, /* depth = 21 */
+ 16746, /* depth = 22 */
+ 17120, /* depth = 23 */
+ 77578, /* depth = 24 */
+ 401, /* depth = 25 */
+ 656, /* depth = 26 */
+ 1107, /* depth = 27 */
+ 1121, /* depth = 28 */
+ 2316, /* depth = 29 */
+ 717, /* depth = 30 */
+ 10, /* depth = 31 */
+ 66 /* depth = 32 */
+ },
+ .b = { /* IP class A in which the most 2 significant bits are 10 */
+ 0, /* depth = 1 */
+ 0, /* depth = 2 */
+ 0, /* depth = 3 */
+ 0, /* depth = 4 */
+ 1, /* depth = 5 */
+ 1, /* depth = 6 */
+ 1, /* depth = 7 */
+ 3, /* depth = 8 */
+ 3, /* depth = 9 */
+ 30, /* depth = 10 */
+ 25, /* depth = 11 */
+ 168, /* depth = 12 */
+ 305, /* depth = 13 */
+ 569, /* depth = 14 */
+ 1129, /* depth = 15 */
+ 50800, /* depth = 16 */
+ 1645, /* depth = 17 */
+ 1820, /* depth = 18 */
+ 3506, /* depth = 19 */
+ 3258, /* depth = 20 */
+ 3424, /* depth = 21 */
+ 4971, /* depth = 22 */
+ 6885, /* depth = 23 */
+ 39771, /* depth = 24 */
+ 424, /* depth = 25 */
+ 170, /* depth = 26 */
+ 433, /* depth = 27 */
+ 92, /* depth = 28 */
+ 366, /* depth = 29 */
+ 377, /* depth = 30 */
+ 2, /* depth = 31 */
+ 200 /* depth = 32 */
+ },
+ .c = { /* IP class A in which the most 3 significant bits are 110 */
+ 0, /* depth = 1 */
+ 0, /* depth = 2 */
+ 0, /* depth = 3 */
+ 0, /* depth = 4 */
+ 0, /* depth = 5 */
+ 0, /* depth = 6 */
+ 0, /* depth = 7 */
+ 12, /* depth = 8 */
+ 8, /* depth = 9 */
+ 9, /* depth = 10 */
+ 33, /* depth = 11 */
+ 69, /* depth = 12 */
+ 237, /* depth = 13 */
+ 1007, /* depth = 14 */
+ 1717, /* depth = 15 */
+ 14663, /* depth = 16 */
+ 8070, /* depth = 17 */
+ 16185, /* depth = 18 */
+ 48261, /* depth = 19 */
+ 36870, /* depth = 20 */
+ 33960, /* depth = 21 */
+ 50638, /* depth = 22 */
+ 61422, /* depth = 23 */
+ 466549, /* depth = 24 */
+ 1829, /* depth = 25 */
+ 4824, /* depth = 26 */
+ 4927, /* depth = 27 */
+ 5914, /* depth = 28 */
+ 10254, /* depth = 29 */
+ 4905, /* depth = 30 */
+ 1, /* depth = 31 */
+ 716 /* depth = 32 */
+ }
+};
+
+static void generate_random_rule_prefix(uint32_t ip_class, uint8_t depth)
+{
+/* IP address class A, the most significant bit is 0 */
+#define IP_HEAD_MASK_A 0x00000000
+#define IP_HEAD_BIT_NUM_A 1
+
+/* IP address class B, the most significant 2 bits are 10 */
+#define IP_HEAD_MASK_B 0x80000000
+#define IP_HEAD_BIT_NUM_B 2
+
+/* IP address class C, the most significant 3 bits are 110 */
+#define IP_HEAD_MASK_C 0xC0000000
+#define IP_HEAD_BIT_NUM_C 3
+
+ uint32_t class_depth;
+ uint32_t range;
+ uint32_t mask;
+ uint32_t step;
+ uint32_t start;
+ uint32_t fixed_bit_num;
+ uint32_t ip_head_mask;
+ uint32_t rule_num;
+ uint32_t k;
+ struct route_rule *ptr_rule;
+
+ if (ip_class == IP_CLASS_A) { /* IP Address class A */
+ fixed_bit_num = IP_HEAD_BIT_NUM_A;
+ ip_head_mask = IP_HEAD_MASK_A;
+ rule_num = rule_count.a[depth - 1];
+ } else if (ip_class == IP_CLASS_B) { /* IP Address class B */
+ fixed_bit_num = IP_HEAD_BIT_NUM_B;
+ ip_head_mask = IP_HEAD_MASK_B;
+ rule_num = rule_count.b[depth - 1];
+ } else { /* IP Address class C */
+ fixed_bit_num = IP_HEAD_BIT_NUM_C;
+ ip_head_mask = IP_HEAD_MASK_C;
+ rule_num = rule_count.c[depth - 1];
+ }
+
+ if (rule_num == 0)
+ return;
+
+ /* the number of rest bits which don't include the most significant
+ * fixed bits for this IP address class
+ */
+ class_depth = depth - fixed_bit_num;
+
+ /* range is the maximum number of rules for this depth and
+ * this IP address class
+ */
+ range = 1 << class_depth;
+
+ /* only mask the most depth significant generated bits
+ * except fixed bits for IP address class
+ */
+ mask = range - 1;
+
+ /* Widen coverage of IP address in generated rules */
+ if (range <= rule_num)
+ step = 1;
+ else
+ step = round((double)range / rule_num);
+
+ /* Only generate rest bits except the most significant
+ * fixed bits for IP address class
+ */
+ start = lrand48() & mask;
+ ptr_rule = &large_route_table[num_route_entries];
+ for (k = 0; k < rule_num; k++) {
+ ptr_rule->ip = (start << (RTE_LPM_MAX_DEPTH - depth))
+ | ip_head_mask;
+ ptr_rule->depth = depth;
+ ptr_rule++;
+ start = (start + step) & mask;
+ }
+ num_route_entries += rule_num;
+}
+
+static void insert_rule_in_random_pos(uint32_t ip, uint8_t depth)
+{
+ uint32_t pos;
+ int try_count = 0;
+ struct route_rule tmp;
+
+ do {
+ pos = lrand48();
+ try_count++;
+ } while ((try_count < 10) && (pos > num_route_entries));
+
+ if ((pos > num_route_entries) || (pos >= MAX_RULE_NUM))
+ pos = num_route_entries >> 1;
+
+ tmp = large_route_table[pos];
+ large_route_table[pos].ip = ip;
+ large_route_table[pos].depth = depth;
+ if (num_route_entries < MAX_RULE_NUM)
+ large_route_table[num_route_entries++] = tmp;
+}
+
+void generate_large_route_rule_table(void)
+{
+ uint32_t ip_class;
+ uint8_t depth;
+
+ num_route_entries = 0;
+ memset(large_route_table, 0, sizeof(large_route_table));
+
+ for (ip_class = IP_CLASS_A; ip_class <= IP_CLASS_C; ip_class++) {
+ for (depth = 1; depth <= RTE_LPM_MAX_DEPTH; depth++)
+ generate_random_rule_prefix(ip_class, depth);
+ }
+
+ /* Add following rules to keep same as previous large constant table,
+ * they are 4 rules with private local IP address and 1 all-zeros prefix
+ * with depth = 8.
+ */
+ insert_rule_in_random_pos(RTE_IPV4(0, 0, 0, 0), 8);
+ insert_rule_in_random_pos(RTE_IPV4(10, 2, 23, 147), 32);
+ insert_rule_in_random_pos(RTE_IPV4(192, 168, 100, 10), 24);
+ insert_rule_in_random_pos(RTE_IPV4(192, 168, 25, 100), 24);
+ insert_rule_in_random_pos(RTE_IPV4(192, 168, 129, 124), 32);
+}
+
+void
+print_route_distribution(const struct route_rule *table, uint32_t n)
+{
+ unsigned int i, j;
+
+ printf("Route distribution per prefix width:\n");
+ printf("DEPTH QUANTITY (PERCENT)\n");
+ printf("---------------------------\n");
+
+ /* Count depths. */
+ for (i = 1; i <= 32; i++) {
+ unsigned int depth_counter = 0;
+ double percent_hits;
+
+ for (j = 0; j < n; j++)
+ if (table[j].depth == (uint8_t) i)
+ depth_counter++;
+
+ percent_hits = ((double)depth_counter)/((double)n) * 100;
+ printf("%.2u%15u (%.2f)\n", i, depth_counter, percent_hits);
+ }
+ printf("\n");
+}
diff --git a/app/test/test_lpm_routes.h b/app/test/test_lpm_routes.h
new file mode 100644
index 000000000..c7874ea8f
--- /dev/null
+++ b/app/test/test_lpm_routes.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2019 Intel Corporation
+ */
+
+#ifndef _TEST_LPM_ROUTES_H_
+#define _TEST_LPM_ROUTES_H_
+
+#include <rte_ip.h>
+
+#define MAX_RULE_NUM (1200000)
+
+struct route_rule {
+ uint32_t ip;
+ uint8_t depth;
+};
+
+extern struct route_rule large_route_table[MAX_RULE_NUM];
+
+extern uint32_t num_route_entries;
+#define NUM_ROUTE_ENTRIES num_route_entries
+
+void generate_large_route_rule_table(void);
+void print_route_distribution(const struct route_rule *table, uint32_t n);
+
+#endif
diff --git a/app/test/v2.0/dcompat.h b/app/test/v2.0/dcompat.h
new file mode 100644
index 000000000..e66206156
--- /dev/null
+++ b/app/test/v2.0/dcompat.h
@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2019 Intel Corporation
+ */
+
+#ifndef _DCOMPAT_H_
+#define _DCOMPAT_H_
+
+#include <rte_compat.h>
+
+#define ABI_VERSION 2.0
+
+#define MAP_ABI_SYMBOL(name) \
+ BIND_VERSION_SYMBOL(name, ABI_VERSION)
+
+MAP_ABI_SYMBOL(rte_lpm_add);
+MAP_ABI_SYMBOL(rte_lpm_find_existing);
+MAP_ABI_SYMBOL(rte_lpm_create);
+MAP_ABI_SYMBOL(rte_lpm_free);
+MAP_ABI_SYMBOL(rte_lpm_is_rule_present);
+MAP_ABI_SYMBOL(rte_lpm_delete);
+MAP_ABI_SYMBOL(rte_lpm_delete_all);
+
+MAP_ABI_SYMBOL(rte_lpm6_add);
+MAP_ABI_SYMBOL(rte_lpm6_is_rule_present);
+MAP_ABI_SYMBOL(rte_lpm6_lookup);
+MAP_ABI_SYMBOL(rte_lpm6_lookup_bulk_func);
+
+#undef MAP_ABI_SYMBOL
+
+#endif
diff --git a/app/test/v2.0/rte_lpm.h b/app/test/v2.0/rte_lpm.h
new file mode 100644
index 000000000..5e05015cb
--- /dev/null
+++ b/app/test/v2.0/rte_lpm.h
@@ -0,0 +1,451 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2014 Intel Corporation
+ */
+
+#ifndef _RTE_LPM_H_
+#define _RTE_LPM_H_
+
+/**
+ * @file
+ * RTE Longest Prefix Match (LPM)
+ */
+
+#include <errno.h>
+#include <sys/queue.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <rte_branch_prediction.h>
+#include <rte_byteorder.h>
+#include <rte_memory.h>
+#include <rte_common.h>
+#include <rte_vect.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** Max number of characters in LPM name. */
+#define RTE_LPM_NAMESIZE 32
+
+/** Maximum depth value possible for IPv4 LPM. */
+#define RTE_LPM_MAX_DEPTH 32
+
+/** @internal Total number of tbl24 entries. */
+#define RTE_LPM_TBL24_NUM_ENTRIES (1 << 24)
+
+/** @internal Number of entries in a tbl8 group. */
+#define RTE_LPM_TBL8_GROUP_NUM_ENTRIES 256
+
+/** @internal Total number of tbl8 groups in the tbl8. */
+#define RTE_LPM_TBL8_NUM_GROUPS 256
+
+/** @internal Total number of tbl8 entries. */
+#define RTE_LPM_TBL8_NUM_ENTRIES (RTE_LPM_TBL8_NUM_GROUPS * \
+ RTE_LPM_TBL8_GROUP_NUM_ENTRIES)
+
+/** @internal Macro to enable/disable run-time checks. */
+#if defined(RTE_LIBRTE_LPM_DEBUG)
+#define RTE_LPM_RETURN_IF_TRUE(cond, retval) do { \
+ if (cond) return (retval); \
+} while (0)
+#else
+#define RTE_LPM_RETURN_IF_TRUE(cond, retval)
+#endif
+
+/** @internal bitmask with valid and ext_entry/valid_group fields set */
+#define RTE_LPM_VALID_EXT_ENTRY_BITMASK 0x0300
+
+/** Bitmask used to indicate successful lookup */
+#define RTE_LPM_LOOKUP_SUCCESS 0x0100
+
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+/** @internal Tbl24 entry structure. */
+struct rte_lpm_tbl24_entry {
+ /* Stores Next hop or group index (i.e. gindex)into tbl8. */
+ union {
+ uint8_t next_hop;
+ uint8_t tbl8_gindex;
+ };
+ /* Using single uint8_t to store 3 values. */
+ uint8_t valid :1; /**< Validation flag. */
+ uint8_t ext_entry :1; /**< External entry. */
+ uint8_t depth :6; /**< Rule depth. */
+};
+
+/** @internal Tbl8 entry structure. */
+struct rte_lpm_tbl8_entry {
+ uint8_t next_hop; /**< next hop. */
+ /* Using single uint8_t to store 3 values. */
+ uint8_t valid :1; /**< Validation flag. */
+ uint8_t valid_group :1; /**< Group validation flag. */
+ uint8_t depth :6; /**< Rule depth. */
+};
+#else
+struct rte_lpm_tbl24_entry {
+ uint8_t depth :6;
+ uint8_t ext_entry :1;
+ uint8_t valid :1;
+ union {
+ uint8_t tbl8_gindex;
+ uint8_t next_hop;
+ };
+};
+
+struct rte_lpm_tbl8_entry {
+ uint8_t depth :6;
+ uint8_t valid_group :1;
+ uint8_t valid :1;
+ uint8_t next_hop;
+};
+#endif
+
+/** @internal Rule structure. */
+struct rte_lpm_rule {
+ uint32_t ip; /**< Rule IP address. */
+ uint8_t next_hop; /**< Rule next hop. */
+};
+
+/** @internal Contains metadata about the rules table. */
+struct rte_lpm_rule_info {
+ uint32_t used_rules; /**< Used rules so far. */
+ uint32_t first_rule; /**< Indexes the first rule of a given depth. */
+};
+
+/** @internal LPM structure. */
+struct rte_lpm {
+ /* LPM metadata. */
+ char name[RTE_LPM_NAMESIZE]; /**< Name of the lpm. */
+ uint32_t max_rules; /**< Max. balanced rules per lpm. */
+ struct rte_lpm_rule_info rule_info[RTE_LPM_MAX_DEPTH]; /**< Rule info table. */
+
+ /* LPM Tables. */
+ struct rte_lpm_tbl24_entry tbl24[RTE_LPM_TBL24_NUM_ENTRIES] \
+ __rte_cache_aligned; /**< LPM tbl24 table. */
+ struct rte_lpm_tbl8_entry tbl8[RTE_LPM_TBL8_NUM_ENTRIES] \
+ __rte_cache_aligned; /**< LPM tbl8 table. */
+ struct rte_lpm_rule rules_tbl[0] \
+ __rte_cache_aligned; /**< LPM rules. */
+};
+
+/**
+ * Create an LPM object.
+ *
+ * @param name
+ * LPM object name
+ * @param socket_id
+ * NUMA socket ID for LPM table memory allocation
+ * @param max_rules
+ * Maximum number of LPM rules that can be added
+ * @param flags
+ * This parameter is currently unused
+ * @return
+ * Handle to LPM object on success, NULL otherwise with rte_errno set
+ * to an appropriate values. Possible rte_errno values include:
+ * - E_RTE_NO_CONFIG - function could not get pointer to rte_config structure
+ * - E_RTE_SECONDARY - function was called from a secondary process instance
+ * - EINVAL - invalid parameter passed to function
+ * - ENOSPC - the maximum number of memzones has already been allocated
+ * - EEXIST - a memzone with the same name already exists
+ * - ENOMEM - no appropriate memory area found in which to create memzone
+ */
+struct rte_lpm *
+rte_lpm_create(const char *name, int socket_id, int max_rules, int flags);
+
+/**
+ * Find an existing LPM object and return a pointer to it.
+ *
+ * @param name
+ * Name of the lpm object as passed to rte_lpm_create()
+ * @return
+ * Pointer to lpm object or NULL if object not found with rte_errno
+ * set appropriately. Possible rte_errno values include:
+ * - ENOENT - required entry not available to return.
+ */
+struct rte_lpm *
+rte_lpm_find_existing(const char *name);
+
+/**
+ * Free an LPM object.
+ *
+ * @param lpm
+ * LPM object handle
+ * @return
+ * None
+ */
+void
+rte_lpm_free(struct rte_lpm *lpm);
+
+/**
+ * Add a rule to the LPM table.
+ *
+ * @param lpm
+ * LPM object handle
+ * @param ip
+ * IP of the rule to be added to the LPM table
+ * @param depth
+ * Depth of the rule to be added to the LPM table
+ * @param next_hop
+ * Next hop of the rule to be added to the LPM table
+ * @return
+ * 0 on success, negative value otherwise
+ */
+int
+rte_lpm_add(struct rte_lpm *lpm, uint32_t ip, uint8_t depth, uint8_t next_hop);
+
+/**
+ * Check if a rule is present in the LPM table,
+ * and provide its next hop if it is.
+ *
+ * @param lpm
+ * LPM object handle
+ * @param ip
+ * IP of the rule to be searched
+ * @param depth
+ * Depth of the rule to searched
+ * @param next_hop
+ * Next hop of the rule (valid only if it is found)
+ * @return
+ * 1 if the rule exists, 0 if it does not, a negative value on failure
+ */
+int
+rte_lpm_is_rule_present(struct rte_lpm *lpm, uint32_t ip, uint8_t depth,
+uint8_t *next_hop);
+
+/**
+ * Delete a rule from the LPM table.
+ *
+ * @param lpm
+ * LPM object handle
+ * @param ip
+ * IP of the rule to be deleted from the LPM table
+ * @param depth
+ * Depth of the rule to be deleted from the LPM table
+ * @return
+ * 0 on success, negative value otherwise
+ */
+int
+rte_lpm_delete(struct rte_lpm *lpm, uint32_t ip, uint8_t depth);
+
+/**
+ * Delete all rules from the LPM table.
+ *
+ * @param lpm
+ * LPM object handle
+ */
+void
+rte_lpm_delete_all(struct rte_lpm *lpm);
+
+/**
+ * Lookup an IP into the LPM table.
+ *
+ * @param lpm
+ * LPM object handle
+ * @param ip
+ * IP to be looked up in the LPM table
+ * @param next_hop
+ * Next hop of the most specific rule found for IP (valid on lookup hit only)
+ * @return
+ * -EINVAL for incorrect arguments, -ENOENT on lookup miss, 0 on lookup hit
+ */
+static inline int
+rte_lpm_lookup(struct rte_lpm *lpm, uint32_t ip, uint8_t *next_hop)
+{
+ unsigned tbl24_index = (ip >> 8);
+ uint16_t tbl_entry;
+
+ /* DEBUG: Check user input arguments. */
+ RTE_LPM_RETURN_IF_TRUE(((lpm == NULL) || (next_hop == NULL)), -EINVAL);
+
+ /* Copy tbl24 entry */
+ tbl_entry = *(const uint16_t *)&lpm->tbl24[tbl24_index];
+
+ /* Copy tbl8 entry (only if needed) */
+ if (unlikely((tbl_entry & RTE_LPM_VALID_EXT_ENTRY_BITMASK) ==
+ RTE_LPM_VALID_EXT_ENTRY_BITMASK)) {
+
+ unsigned tbl8_index = (uint8_t)ip +
+ ((uint8_t)tbl_entry * RTE_LPM_TBL8_GROUP_NUM_ENTRIES);
+
+ tbl_entry = *(const uint16_t *)&lpm->tbl8[tbl8_index];
+ }
+
+ *next_hop = (uint8_t)tbl_entry;
+ return (tbl_entry & RTE_LPM_LOOKUP_SUCCESS) ? 0 : -ENOENT;
+}
+
+/**
+ * Lookup multiple IP addresses in an LPM table. This may be implemented as a
+ * macro, so the address of the function should not be used.
+ *
+ * @param lpm
+ * LPM object handle
+ * @param ips
+ * Array of IPs to be looked up in the LPM table
+ * @param next_hops
+ * Next hop of the most specific rule found for IP (valid on lookup hit only).
+ * This is an array of two byte values. The most significant byte in each
+ * value says whether the lookup was successful (bitmask
+ * RTE_LPM_LOOKUP_SUCCESS is set). The least significant byte is the
+ * actual next hop.
+ * @param n
+ * Number of elements in ips (and next_hops) array to lookup. This should be a
+ * compile time constant, and divisible by 8 for best performance.
+ * @return
+ * -EINVAL for incorrect arguments, otherwise 0
+ */
+#define rte_lpm_lookup_bulk(lpm, ips, next_hops, n) \
+ rte_lpm_lookup_bulk_func(lpm, ips, next_hops, n)
+
+static inline int
+rte_lpm_lookup_bulk_func(const struct rte_lpm *lpm, const uint32_t * ips,
+ uint16_t * next_hops, const unsigned n)
+{
+ unsigned i;
+ unsigned tbl24_indexes[n];
+
+ /* DEBUG: Check user input arguments. */
+ RTE_LPM_RETURN_IF_TRUE(((lpm == NULL) || (ips == NULL) ||
+ (next_hops == NULL)), -EINVAL);
+
+ for (i = 0; i < n; i++) {
+ tbl24_indexes[i] = ips[i] >> 8;
+ }
+
+ for (i = 0; i < n; i++) {
+ /* Simply copy tbl24 entry to output */
+ next_hops[i] = *(const uint16_t *)&lpm->tbl24[tbl24_indexes[i]];
+
+ /* Overwrite output with tbl8 entry if needed */
+ if (unlikely((next_hops[i] & RTE_LPM_VALID_EXT_ENTRY_BITMASK) ==
+ RTE_LPM_VALID_EXT_ENTRY_BITMASK)) {
+
+ unsigned tbl8_index = (uint8_t)ips[i] +
+ ((uint8_t)next_hops[i] *
+ RTE_LPM_TBL8_GROUP_NUM_ENTRIES);
+
+ next_hops[i] = *(const uint16_t *)&lpm->tbl8[tbl8_index];
+ }
+ }
+ return 0;
+}
+
+/* Mask four results. */
+#define RTE_LPM_MASKX4_RES UINT64_C(0x00ff00ff00ff00ff)
+
+/**
+ * Lookup four IP addresses in an LPM table.
+ *
+ * @param lpm
+ * LPM object handle
+ * @param ip
+ * Four IPs to be looked up in the LPM table
+ * @param hop
+ * Next hop of the most specific rule found for IP (valid on lookup hit only).
+ * This is an 4 elements array of two byte values.
+ * If the lookup was succesfull for the given IP, then least significant byte
+ * of the corresponding element is the actual next hop and the most
+ * significant byte is zero.
+ * If the lookup for the given IP failed, then corresponding element would
+ * contain default value, see description of then next parameter.
+ * @param defv
+ * Default value to populate into corresponding element of hop[] array,
+ * if lookup would fail.
+ */
+static inline void
+rte_lpm_lookupx4(const struct rte_lpm *lpm, __m128i ip, uint16_t hop[4],
+ uint16_t defv)
+{
+ __m128i i24;
+ rte_xmm_t i8;
+ uint16_t tbl[4];
+ uint64_t idx, pt;
+
+ const __m128i mask8 =
+ _mm_set_epi32(UINT8_MAX, UINT8_MAX, UINT8_MAX, UINT8_MAX);
+
+ /*
+ * RTE_LPM_VALID_EXT_ENTRY_BITMASK for 4 LPM entries
+ * as one 64-bit value (0x0300030003000300).
+ */
+ const uint64_t mask_xv =
+ ((uint64_t)RTE_LPM_VALID_EXT_ENTRY_BITMASK |
+ (uint64_t)RTE_LPM_VALID_EXT_ENTRY_BITMASK << 16 |
+ (uint64_t)RTE_LPM_VALID_EXT_ENTRY_BITMASK << 32 |
+ (uint64_t)RTE_LPM_VALID_EXT_ENTRY_BITMASK << 48);
+
+ /*
+ * RTE_LPM_LOOKUP_SUCCESS for 4 LPM entries
+ * as one 64-bit value (0x0100010001000100).
+ */
+ const uint64_t mask_v =
+ ((uint64_t)RTE_LPM_LOOKUP_SUCCESS |
+ (uint64_t)RTE_LPM_LOOKUP_SUCCESS << 16 |
+ (uint64_t)RTE_LPM_LOOKUP_SUCCESS << 32 |
+ (uint64_t)RTE_LPM_LOOKUP_SUCCESS << 48);
+
+ /* get 4 indexes for tbl24[]. */
+ i24 = _mm_srli_epi32(ip, CHAR_BIT);
+
+ /* extract values from tbl24[] */
+ idx = _mm_cvtsi128_si64(i24);
+ i24 = _mm_srli_si128(i24, sizeof(uint64_t));
+
+ tbl[0] = *(const uint16_t *)&lpm->tbl24[(uint32_t)idx];
+ tbl[1] = *(const uint16_t *)&lpm->tbl24[idx >> 32];
+
+ idx = _mm_cvtsi128_si64(i24);
+
+ tbl[2] = *(const uint16_t *)&lpm->tbl24[(uint32_t)idx];
+ tbl[3] = *(const uint16_t *)&lpm->tbl24[idx >> 32];
+
+ /* get 4 indexes for tbl8[]. */
+ i8.x = _mm_and_si128(ip, mask8);
+
+ pt = (uint64_t)tbl[0] |
+ (uint64_t)tbl[1] << 16 |
+ (uint64_t)tbl[2] << 32 |
+ (uint64_t)tbl[3] << 48;
+
+ /* search successfully finished for all 4 IP addresses. */
+ if (likely((pt & mask_xv) == mask_v)) {
+ uintptr_t ph = (uintptr_t)hop;
+ *(uint64_t *)ph = pt & RTE_LPM_MASKX4_RES;
+ return;
+ }
+
+ if (unlikely((pt & RTE_LPM_VALID_EXT_ENTRY_BITMASK) ==
+ RTE_LPM_VALID_EXT_ENTRY_BITMASK)) {
+ i8.u32[0] = i8.u32[0] +
+ (uint8_t)tbl[0] * RTE_LPM_TBL8_GROUP_NUM_ENTRIES;
+ tbl[0] = *(const uint16_t *)&lpm->tbl8[i8.u32[0]];
+ }
+ if (unlikely((pt >> 16 & RTE_LPM_VALID_EXT_ENTRY_BITMASK) ==
+ RTE_LPM_VALID_EXT_ENTRY_BITMASK)) {
+ i8.u32[1] = i8.u32[1] +
+ (uint8_t)tbl[1] * RTE_LPM_TBL8_GROUP_NUM_ENTRIES;
+ tbl[1] = *(const uint16_t *)&lpm->tbl8[i8.u32[1]];
+ }
+ if (unlikely((pt >> 32 & RTE_LPM_VALID_EXT_ENTRY_BITMASK) ==
+ RTE_LPM_VALID_EXT_ENTRY_BITMASK)) {
+ i8.u32[2] = i8.u32[2] +
+ (uint8_t)tbl[2] * RTE_LPM_TBL8_GROUP_NUM_ENTRIES;
+ tbl[2] = *(const uint16_t *)&lpm->tbl8[i8.u32[2]];
+ }
+ if (unlikely((pt >> 48 & RTE_LPM_VALID_EXT_ENTRY_BITMASK) ==
+ RTE_LPM_VALID_EXT_ENTRY_BITMASK)) {
+ i8.u32[3] = i8.u32[3] +
+ (uint8_t)tbl[3] * RTE_LPM_TBL8_GROUP_NUM_ENTRIES;
+ tbl[3] = *(const uint16_t *)&lpm->tbl8[i8.u32[3]];
+ }
+
+ hop[0] = (tbl[0] & RTE_LPM_LOOKUP_SUCCESS) ? (uint8_t)tbl[0] : defv;
+ hop[1] = (tbl[1] & RTE_LPM_LOOKUP_SUCCESS) ? (uint8_t)tbl[1] : defv;
+ hop[2] = (tbl[2] & RTE_LPM_LOOKUP_SUCCESS) ? (uint8_t)tbl[2] : defv;
+ hop[3] = (tbl[3] & RTE_LPM_LOOKUP_SUCCESS) ? (uint8_t)tbl[3] : defv;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_LPM_H_ */
diff --git a/app/test/v2.0/rte_lpm6.h b/app/test/v2.0/rte_lpm6.h
new file mode 100644
index 000000000..4892a84e4
--- /dev/null
+++ b/app/test/v2.0/rte_lpm6.h
@@ -0,0 +1,198 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2014 Intel Corporation
+ */
+#ifndef _RTE_LPM6_H_
+#define _RTE_LPM6_H_
+
+/**
+ * @file
+ * RTE Longest Prefix Match for IPv6 (LPM6)
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+#define RTE_LPM6_MAX_DEPTH 128
+#define RTE_LPM6_IPV6_ADDR_SIZE 16
+/** Max number of characters in LPM name. */
+#define RTE_LPM6_NAMESIZE 32
+
+/** LPM structure. */
+struct rte_lpm6;
+
+/** LPM configuration structure. */
+struct rte_lpm6_config {
+ uint32_t max_rules; /**< Max number of rules. */
+ uint32_t number_tbl8s; /**< Number of tbl8s to allocate. */
+ int flags; /**< This field is currently unused. */
+};
+
+/**
+ * Create an LPM object.
+ *
+ * @param name
+ * LPM object name
+ * @param socket_id
+ * NUMA socket ID for LPM table memory allocation
+ * @param config
+ * Structure containing the configuration
+ * @return
+ * Handle to LPM object on success, NULL otherwise with rte_errno set
+ * to an appropriate values. Possible rte_errno values include:
+ * - E_RTE_NO_CONFIG - function could not get pointer to rte_config structure
+ * - E_RTE_SECONDARY - function was called from a secondary process instance
+ * - EINVAL - invalid parameter passed to function
+ * - ENOSPC - the maximum number of memzones has already been allocated
+ * - EEXIST - a memzone with the same name already exists
+ * - ENOMEM - no appropriate memory area found in which to create memzone
+ */
+struct rte_lpm6 *
+rte_lpm6_create(const char *name, int socket_id,
+ const struct rte_lpm6_config *config);
+
+/**
+ * Find an existing LPM object and return a pointer to it.
+ *
+ * @param name
+ * Name of the lpm object as passed to rte_lpm6_create()
+ * @return
+ * Pointer to lpm object or NULL if object not found with rte_errno
+ * set appropriately. Possible rte_errno values include:
+ * - ENOENT - required entry not available to return.
+ */
+struct rte_lpm6 *
+rte_lpm6_find_existing(const char *name);
+
+/**
+ * Free an LPM object.
+ *
+ * @param lpm
+ * LPM object handle
+ * @return
+ * None
+ */
+void
+rte_lpm6_free(struct rte_lpm6 *lpm);
+
+/**
+ * Add a rule to the LPM table.
+ *
+ * @param lpm
+ * LPM object handle
+ * @param ip
+ * IP of the rule to be added to the LPM table
+ * @param depth
+ * Depth of the rule to be added to the LPM table
+ * @param next_hop
+ * Next hop of the rule to be added to the LPM table
+ * @return
+ * 0 on success, negative value otherwise
+ */
+int
+rte_lpm6_add(struct rte_lpm6 *lpm, uint8_t *ip, uint8_t depth,
+ uint8_t next_hop);
+
+/**
+ * Check if a rule is present in the LPM table,
+ * and provide its next hop if it is.
+ *
+ * @param lpm
+ * LPM object handle
+ * @param ip
+ * IP of the rule to be searched
+ * @param depth
+ * Depth of the rule to searched
+ * @param next_hop
+ * Next hop of the rule (valid only if it is found)
+ * @return
+ * 1 if the rule exists, 0 if it does not, a negative value on failure
+ */
+int
+rte_lpm6_is_rule_present(struct rte_lpm6 *lpm, uint8_t *ip, uint8_t depth,
+uint8_t *next_hop);
+
+/**
+ * Delete a rule from the LPM table.
+ *
+ * @param lpm
+ * LPM object handle
+ * @param ip
+ * IP of the rule to be deleted from the LPM table
+ * @param depth
+ * Depth of the rule to be deleted from the LPM table
+ * @return
+ * 0 on success, negative value otherwise
+ */
+int
+rte_lpm6_delete(struct rte_lpm6 *lpm, uint8_t *ip, uint8_t depth);
+
+/**
+ * Delete a rule from the LPM table.
+ *
+ * @param lpm
+ * LPM object handle
+ * @param ips
+ * Array of IPs to be deleted from the LPM table
+ * @param depths
+ * Array of depths of the rules to be deleted from the LPM table
+ * @param n
+ * Number of rules to be deleted from the LPM table
+ * @return
+ * 0 on success, negative value otherwise.
+ */
+int
+rte_lpm6_delete_bulk_func(struct rte_lpm6 *lpm,
+ uint8_t ips[][RTE_LPM6_IPV6_ADDR_SIZE], uint8_t *depths, unsigned n);
+
+/**
+ * Delete all rules from the LPM table.
+ *
+ * @param lpm
+ * LPM object handle
+ */
+void
+rte_lpm6_delete_all(struct rte_lpm6 *lpm);
+
+/**
+ * Lookup an IP into the LPM table.
+ *
+ * @param lpm
+ * LPM object handle
+ * @param ip
+ * IP to be looked up in the LPM table
+ * @param next_hop
+ * Next hop of the most specific rule found for IP (valid on lookup hit only)
+ * @return
+ * -EINVAL for incorrect arguments, -ENOENT on lookup miss, 0 on lookup hit
+ */
+int
+rte_lpm6_lookup(const struct rte_lpm6 *lpm, uint8_t *ip, uint8_t *next_hop);
+
+/**
+ * Lookup multiple IP addresses in an LPM table.
+ *
+ * @param lpm
+ * LPM object handle
+ * @param ips
+ * Array of IPs to be looked up in the LPM table
+ * @param next_hops
+ * Next hop of the most specific rule found for IP (valid on lookup hit only).
+ * This is an array of two byte values. The next hop will be stored on
+ * each position on success; otherwise the position will be set to -1.
+ * @param n
+ * Number of elements in ips (and next_hops) array to lookup.
+ * @return
+ * -EINVAL for incorrect arguments, otherwise 0
+ */
+int
+rte_lpm6_lookup_bulk_func(const struct rte_lpm6 *lpm,
+ uint8_t ips[][RTE_LPM6_IPV6_ADDR_SIZE],
+ int16_t * next_hops, unsigned n);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/app/test/v2.0/test_lpm.c b/app/test/v2.0/test_lpm.c
new file mode 100644
index 000000000..3ae4a121a
--- /dev/null
+++ b/app/test/v2.0/test_lpm.c
@@ -0,0 +1,1139 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2019 Intel Corporation
+ *
+ * LPM Autotests from DPDK v2.2.0 for v2.0 abi compability testing.
+ *
+ */
+
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <sys/queue.h>
+
+#include <rte_common.h>
+#include <rte_cycles.h>
+#include <rte_memory.h>
+#include <rte_random.h>
+#include <rte_branch_prediction.h>
+#include <rte_ip.h>
+#include <time.h>
+
+#include "../test.h"
+
+/* remapping of DPDK v2.0 symbols */
+#include "dcompat.h"
+/* backported header from DPDK v2.0 */
+#include "rte_lpm.h"
+#include "../test_lpm_routes.h"
+
+
+#define TEST_LPM_ASSERT(cond) do { \
+ if (!(cond)) { \
+ printf("Error at line %d:\n", __LINE__); \
+ return -1; \
+ } \
+} while (0)
+
+typedef int32_t (*rte_lpm_test)(void);
+
+static int32_t test0(void);
+static int32_t test1(void);
+static int32_t test2(void);
+static int32_t test3(void);
+static int32_t test4(void);
+static int32_t test5(void);
+static int32_t test6(void);
+static int32_t test7(void);
+static int32_t test8(void);
+static int32_t test9(void);
+static int32_t test10(void);
+static int32_t test11(void);
+static int32_t test12(void);
+static int32_t test13(void);
+static int32_t test14(void);
+static int32_t test15(void);
+static int32_t test16(void);
+static int32_t test17(void);
+
+static rte_lpm_test tests[] = {
+/* Test Cases */
+ test0,
+ test1,
+ test2,
+ test3,
+ test4,
+ test5,
+ test6,
+ test7,
+ test8,
+ test9,
+ test10,
+ test11,
+ test12,
+ test13,
+ test14,
+ test15,
+ test16,
+ test17,
+};
+
+#define NUM_LPM_TESTS (sizeof(tests)/sizeof(tests[0]))
+#define MAX_DEPTH 32
+#define MAX_RULES 256
+#define PASS 0
+
+/*
+ * Check that rte_lpm_create fails gracefully for incorrect user input
+ * arguments
+ */
+int32_t
+test0(void)
+{
+ struct rte_lpm *lpm = NULL;
+
+ /* rte_lpm_create: lpm name == NULL */
+ lpm = rte_lpm_create(NULL, SOCKET_ID_ANY, MAX_RULES, 0);
+ TEST_LPM_ASSERT(lpm == NULL);
+
+ /* rte_lpm_create: max_rules = 0 */
+ /* Note: __func__ inserts the function name, in this case "test0". */
+ lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, 0, 0);
+ TEST_LPM_ASSERT(lpm == NULL);
+
+ /* socket_id < -1 is invalid */
+ lpm = rte_lpm_create(__func__, -2, MAX_RULES, 0);
+ TEST_LPM_ASSERT(lpm == NULL);
+
+ return PASS;
+}
+
+/*
+ * Create lpm table then delete lpm table 100 times
+ * Use a slightly different rules size each time
+ * */
+int32_t
+test1(void)
+{
+ struct rte_lpm *lpm = NULL;
+ int32_t i;
+
+ /* rte_lpm_free: Free NULL */
+ for (i = 0; i < 100; i++) {
+ lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, MAX_RULES - i, 0);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ rte_lpm_free(lpm);
+ }
+
+ /* Can not test free so return success */
+ return PASS;
+}
+
+/*
+ * Call rte_lpm_free for NULL pointer user input. Note: free has no return and
+ * therefore it is impossible to check for failure but this test is added to
+ * increase function coverage metrics and to validate that freeing null does
+ * not crash.
+ */
+int32_t
+test2(void)
+{
+ struct rte_lpm *lpm = NULL;
+
+ lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, MAX_RULES, 0);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ rte_lpm_free(lpm);
+ rte_lpm_free(NULL);
+ return PASS;
+}
+
+/*
+ * Check that rte_lpm_add fails gracefully for incorrect user input arguments
+ */
+int32_t
+test3(void)
+{
+ struct rte_lpm *lpm = NULL;
+ uint32_t ip = RTE_IPV4(0, 0, 0, 0);
+ uint8_t depth = 24, next_hop = 100;
+ int32_t status = 0;
+
+ /* rte_lpm_add: lpm == NULL */
+ status = rte_lpm_add(NULL, ip, depth, next_hop);
+ TEST_LPM_ASSERT(status < 0);
+
+ /*Create vaild lpm to use in rest of test. */
+ lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, MAX_RULES, 0);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ /* rte_lpm_add: depth < 1 */
+ status = rte_lpm_add(lpm, ip, 0, next_hop);
+ TEST_LPM_ASSERT(status < 0);
+
+ /* rte_lpm_add: depth > MAX_DEPTH */
+ status = rte_lpm_add(lpm, ip, (MAX_DEPTH + 1), next_hop);
+ TEST_LPM_ASSERT(status < 0);
+
+ rte_lpm_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Check that rte_lpm_delete fails gracefully for incorrect user input
+ * arguments
+ */
+int32_t
+test4(void)
+{
+ struct rte_lpm *lpm = NULL;
+ uint32_t ip = RTE_IPV4(0, 0, 0, 0);
+ uint8_t depth = 24;
+ int32_t status = 0;
+
+ /* rte_lpm_delete: lpm == NULL */
+ status = rte_lpm_delete(NULL, ip, depth);
+ TEST_LPM_ASSERT(status < 0);
+
+ /*Create vaild lpm to use in rest of test. */
+ lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, MAX_RULES, 0);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ /* rte_lpm_delete: depth < 1 */
+ status = rte_lpm_delete(lpm, ip, 0);
+ TEST_LPM_ASSERT(status < 0);
+
+ /* rte_lpm_delete: depth > MAX_DEPTH */
+ status = rte_lpm_delete(lpm, ip, (MAX_DEPTH + 1));
+ TEST_LPM_ASSERT(status < 0);
+
+ rte_lpm_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Check that rte_lpm_lookup fails gracefully for incorrect user input
+ * arguments
+ */
+int32_t
+test5(void)
+{
+#if defined(RTE_LIBRTE_LPM_DEBUG)
+ struct rte_lpm *lpm = NULL;
+ uint32_t ip = RTE_IPV4(0, 0, 0, 0);
+ uint8_t next_hop_return = 0;
+ int32_t status = 0;
+
+ /* rte_lpm_lookup: lpm == NULL */
+ status = rte_lpm_lookup(NULL, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status < 0);
+
+ /*Create vaild lpm to use in rest of test. */
+ lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, MAX_RULES, 0);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ /* rte_lpm_lookup: depth < 1 */
+ status = rte_lpm_lookup(lpm, ip, NULL);
+ TEST_LPM_ASSERT(status < 0);
+
+ rte_lpm_free(lpm);
+#endif
+ return PASS;
+}
+
+
+
+/*
+ * Call add, lookup and delete for a single rule with depth <= 24
+ */
+int32_t
+test6(void)
+{
+ struct rte_lpm *lpm = NULL;
+ uint32_t ip = RTE_IPV4(0, 0, 0, 0);
+ uint8_t depth = 24, next_hop_add = 100, next_hop_return = 0;
+ int32_t status = 0;
+
+ lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, MAX_RULES, 0);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ status = rte_lpm_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ status = rte_lpm_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ rte_lpm_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Call add, lookup and delete for a single rule with depth > 24
+ */
+
+int32_t
+test7(void)
+{
+ __m128i ipx4;
+ uint16_t hop[4];
+ struct rte_lpm *lpm = NULL;
+ uint32_t ip = RTE_IPV4(0, 0, 0, 0);
+ uint8_t depth = 32, next_hop_add = 100, next_hop_return = 0;
+ int32_t status = 0;
+
+ lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, MAX_RULES, 0);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ status = rte_lpm_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ ipx4 = _mm_set_epi32(ip, ip + 0x100, ip - 0x100, ip);
+ rte_lpm_lookupx4(lpm, ipx4, hop, UINT16_MAX);
+ TEST_LPM_ASSERT(hop[0] == next_hop_add);
+ TEST_LPM_ASSERT(hop[1] == UINT16_MAX);
+ TEST_LPM_ASSERT(hop[2] == UINT16_MAX);
+ TEST_LPM_ASSERT(hop[3] == next_hop_add);
+
+ status = rte_lpm_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ rte_lpm_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Use rte_lpm_add to add rules which effect only the second half of the lpm
+ * table. Use all possible depths ranging from 1..32. Set the next hop = to the
+ * depth. Check lookup hit for on every add and check for lookup miss on the
+ * first half of the lpm table after each add. Finally delete all rules going
+ * backwards (i.e. from depth = 32 ..1) and carry out a lookup after each
+ * delete. The lookup should return the next_hop_add value related to the
+ * previous depth value (i.e. depth -1).
+ */
+int32_t
+test8(void)
+{
+ __m128i ipx4;
+ uint16_t hop[4];
+ struct rte_lpm *lpm = NULL;
+ uint32_t ip1 = RTE_IPV4(127, 255, 255, 255), ip2 = RTE_IPV4(128, 0, 0, 0);
+ uint8_t depth, next_hop_add, next_hop_return;
+ int32_t status = 0;
+
+ lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, MAX_RULES, 0);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ /* Loop with rte_lpm_add. */
+ for (depth = 1; depth <= 32; depth++) {
+ /* Let the next_hop_add value = depth. Just for change. */
+ next_hop_add = depth;
+
+ status = rte_lpm_add(lpm, ip2, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ /* Check IP in first half of tbl24 which should be empty. */
+ status = rte_lpm_lookup(lpm, ip1, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ status = rte_lpm_lookup(lpm, ip2, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) &&
+ (next_hop_return == next_hop_add));
+
+ ipx4 = _mm_set_epi32(ip2, ip1, ip2, ip1);
+ rte_lpm_lookupx4(lpm, ipx4, hop, UINT16_MAX);
+ TEST_LPM_ASSERT(hop[0] == UINT16_MAX);
+ TEST_LPM_ASSERT(hop[1] == next_hop_add);
+ TEST_LPM_ASSERT(hop[2] == UINT16_MAX);
+ TEST_LPM_ASSERT(hop[3] == next_hop_add);
+ }
+
+ /* Loop with rte_lpm_delete. */
+ for (depth = 32; depth >= 1; depth--) {
+ next_hop_add = (uint8_t) (depth - 1);
+
+ status = rte_lpm_delete(lpm, ip2, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip2, &next_hop_return);
+
+ if (depth != 1) {
+ TEST_LPM_ASSERT((status == 0) &&
+ (next_hop_return == next_hop_add));
+ }
+ else {
+ TEST_LPM_ASSERT(status == -ENOENT);
+ }
+
+ status = rte_lpm_lookup(lpm, ip1, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ ipx4 = _mm_set_epi32(ip1, ip1, ip2, ip2);
+ rte_lpm_lookupx4(lpm, ipx4, hop, UINT16_MAX);
+ if (depth != 1) {
+ TEST_LPM_ASSERT(hop[0] == next_hop_add);
+ TEST_LPM_ASSERT(hop[1] == next_hop_add);
+ } else {
+ TEST_LPM_ASSERT(hop[0] == UINT16_MAX);
+ TEST_LPM_ASSERT(hop[1] == UINT16_MAX);
+ }
+ TEST_LPM_ASSERT(hop[2] == UINT16_MAX);
+ TEST_LPM_ASSERT(hop[3] == UINT16_MAX);
+ }
+
+ rte_lpm_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * - Add & lookup to hit invalid TBL24 entry
+ * - Add & lookup to hit valid TBL24 entry not extended
+ * - Add & lookup to hit valid extended TBL24 entry with invalid TBL8 entry
+ * - Add & lookup to hit valid extended TBL24 entry with valid TBL8 entry
+ *
+ */
+int32_t
+test9(void)
+{
+ struct rte_lpm *lpm = NULL;
+ uint32_t ip, ip_1, ip_2;
+ uint8_t depth, depth_1, depth_2, next_hop_add, next_hop_add_1,
+ next_hop_add_2, next_hop_return;
+ int32_t status = 0;
+
+ /* Add & lookup to hit invalid TBL24 entry */
+ ip = RTE_IPV4(128, 0, 0, 0);
+ depth = 24;
+ next_hop_add = 100;
+
+ lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, MAX_RULES, 0);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ status = rte_lpm_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ status = rte_lpm_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ rte_lpm_delete_all(lpm);
+
+ /* Add & lookup to hit valid TBL24 entry not extended */
+ ip = RTE_IPV4(128, 0, 0, 0);
+ depth = 23;
+ next_hop_add = 100;
+
+ status = rte_lpm_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ depth = 24;
+ next_hop_add = 101;
+
+ status = rte_lpm_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ depth = 24;
+
+ status = rte_lpm_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ depth = 23;
+
+ status = rte_lpm_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ rte_lpm_delete_all(lpm);
+
+ /* Add & lookup to hit valid extended TBL24 entry with invalid TBL8
+ * entry */
+ ip = RTE_IPV4(128, 0, 0, 0);
+ depth = 32;
+ next_hop_add = 100;
+
+ status = rte_lpm_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ ip = RTE_IPV4(128, 0, 0, 5);
+ depth = 32;
+ next_hop_add = 101;
+
+ status = rte_lpm_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ status = rte_lpm_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ ip = RTE_IPV4(128, 0, 0, 0);
+ depth = 32;
+ next_hop_add = 100;
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ status = rte_lpm_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ rte_lpm_delete_all(lpm);
+
+ /* Add & lookup to hit valid extended TBL24 entry with valid TBL8
+ * entry */
+ ip_1 = RTE_IPV4(128, 0, 0, 0);
+ depth_1 = 25;
+ next_hop_add_1 = 101;
+
+ ip_2 = RTE_IPV4(128, 0, 0, 5);
+ depth_2 = 32;
+ next_hop_add_2 = 102;
+
+ next_hop_return = 0;
+
+ status = rte_lpm_add(lpm, ip_1, depth_1, next_hop_add_1);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip_1, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add_1));
+
+ status = rte_lpm_add(lpm, ip_2, depth_2, next_hop_add_2);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip_2, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add_2));
+
+ status = rte_lpm_delete(lpm, ip_2, depth_2);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip_2, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add_1));
+
+ status = rte_lpm_delete(lpm, ip_1, depth_1);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip_1, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ rte_lpm_free(lpm);
+
+ return PASS;
+}
+
+
+/*
+ * - Add rule that covers a TBL24 range previously invalid & lookup (& delete &
+ * lookup)
+ * - Add rule that extends a TBL24 invalid entry & lookup (& delete & lookup)
+ * - Add rule that extends a TBL24 valid entry & lookup for both rules (&
+ * delete & lookup)
+ * - Add rule that updates the next hop in TBL24 & lookup (& delete & lookup)
+ * - Add rule that updates the next hop in TBL8 & lookup (& delete & lookup)
+ * - Delete a rule that is not present in the TBL24 & lookup
+ * - Delete a rule that is not present in the TBL8 & lookup
+ *
+ */
+int32_t
+test10(void)
+{
+
+ struct rte_lpm *lpm = NULL;
+ uint32_t ip;
+ uint8_t depth, next_hop_add, next_hop_return;
+ int32_t status = 0;
+
+ /* Add rule that covers a TBL24 range previously invalid & lookup
+ * (& delete & lookup) */
+ lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, MAX_RULES, 0);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ ip = RTE_IPV4(128, 0, 0, 0);
+ depth = 16;
+ next_hop_add = 100;
+
+ status = rte_lpm_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ status = rte_lpm_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ rte_lpm_delete_all(lpm);
+
+ ip = RTE_IPV4(128, 0, 0, 0);
+ depth = 25;
+ next_hop_add = 100;
+
+ status = rte_lpm_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ status = rte_lpm_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ rte_lpm_delete_all(lpm);
+
+ /* Add rule that extends a TBL24 valid entry & lookup for both rules
+ * (& delete & lookup) */
+
+ ip = RTE_IPV4(128, 0, 0, 0);
+ depth = 24;
+ next_hop_add = 100;
+
+ status = rte_lpm_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ ip = RTE_IPV4(128, 0, 0, 10);
+ depth = 32;
+ next_hop_add = 101;
+
+ status = rte_lpm_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ ip = RTE_IPV4(128, 0, 0, 0);
+ next_hop_add = 100;
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ ip = RTE_IPV4(128, 0, 0, 0);
+ depth = 24;
+
+ status = rte_lpm_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ ip = RTE_IPV4(128, 0, 0, 10);
+ depth = 32;
+
+ status = rte_lpm_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ rte_lpm_delete_all(lpm);
+
+ /* Add rule that updates the next hop in TBL24 & lookup
+ * (& delete & lookup) */
+
+ ip = RTE_IPV4(128, 0, 0, 0);
+ depth = 24;
+ next_hop_add = 100;
+
+ status = rte_lpm_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ next_hop_add = 101;
+
+ status = rte_lpm_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ status = rte_lpm_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ rte_lpm_delete_all(lpm);
+
+ /* Add rule that updates the next hop in TBL8 & lookup
+ * (& delete & lookup) */
+
+ ip = RTE_IPV4(128, 0, 0, 0);
+ depth = 32;
+ next_hop_add = 100;
+
+ status = rte_lpm_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ next_hop_add = 101;
+
+ status = rte_lpm_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ status = rte_lpm_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ rte_lpm_delete_all(lpm);
+
+ /* Delete a rule that is not present in the TBL24 & lookup */
+
+ ip = RTE_IPV4(128, 0, 0, 0);
+ depth = 24;
+
+ status = rte_lpm_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status < 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ rte_lpm_delete_all(lpm);
+
+ /* Delete a rule that is not present in the TBL8 & lookup */
+
+ ip = RTE_IPV4(128, 0, 0, 0);
+ depth = 32;
+
+ status = rte_lpm_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status < 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ rte_lpm_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Add two rules, lookup to hit the more specific one, lookup to hit the less
+ * specific one delete the less specific rule and lookup previous values again;
+ * add a more specific rule than the existing rule, lookup again
+ *
+ * */
+int32_t
+test11(void)
+{
+
+ struct rte_lpm *lpm = NULL;
+ uint32_t ip;
+ uint8_t depth, next_hop_add, next_hop_return;
+ int32_t status = 0;
+
+ lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, MAX_RULES, 0);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ ip = RTE_IPV4(128, 0, 0, 0);
+ depth = 24;
+ next_hop_add = 100;
+
+ status = rte_lpm_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ ip = RTE_IPV4(128, 0, 0, 10);
+ depth = 32;
+ next_hop_add = 101;
+
+ status = rte_lpm_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ ip = RTE_IPV4(128, 0, 0, 0);
+ next_hop_add = 100;
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ ip = RTE_IPV4(128, 0, 0, 0);
+ depth = 24;
+
+ status = rte_lpm_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ ip = RTE_IPV4(128, 0, 0, 10);
+ depth = 32;
+
+ status = rte_lpm_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ rte_lpm_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Add an extended rule (i.e. depth greater than 24, lookup (hit), delete,
+ * lookup (miss) in a for loop of 1000 times. This will check tbl8 extension
+ * and contraction.
+ *
+ * */
+
+int32_t
+test12(void)
+{
+ __m128i ipx4;
+ uint16_t hop[4];
+ struct rte_lpm *lpm = NULL;
+ uint32_t ip, i;
+ uint8_t depth, next_hop_add, next_hop_return;
+ int32_t status = 0;
+
+ lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, MAX_RULES, 0);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ ip = RTE_IPV4(128, 0, 0, 0);
+ depth = 32;
+ next_hop_add = 100;
+
+ for (i = 0; i < 1000; i++) {
+ status = rte_lpm_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) &&
+ (next_hop_return == next_hop_add));
+
+ ipx4 = _mm_set_epi32(ip, ip + 1, ip, ip - 1);
+ rte_lpm_lookupx4(lpm, ipx4, hop, UINT16_MAX);
+ TEST_LPM_ASSERT(hop[0] == UINT16_MAX);
+ TEST_LPM_ASSERT(hop[1] == next_hop_add);
+ TEST_LPM_ASSERT(hop[2] == UINT16_MAX);
+ TEST_LPM_ASSERT(hop[3] == next_hop_add);
+
+ status = rte_lpm_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+ }
+
+ rte_lpm_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Add a rule to tbl24, lookup (hit), then add a rule that will extend this
+ * tbl24 entry, lookup (hit). delete the rule that caused the tbl24 extension,
+ * lookup (miss) and repeat for loop of 1000 times. This will check tbl8
+ * extension and contraction.
+ *
+ * */
+
+int32_t
+test13(void)
+{
+ struct rte_lpm *lpm = NULL;
+ uint32_t ip, i;
+ uint8_t depth, next_hop_add_1, next_hop_add_2, next_hop_return;
+ int32_t status = 0;
+
+ lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, MAX_RULES, 0);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ ip = RTE_IPV4(128, 0, 0, 0);
+ depth = 24;
+ next_hop_add_1 = 100;
+
+ status = rte_lpm_add(lpm, ip, depth, next_hop_add_1);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add_1));
+
+ depth = 32;
+ next_hop_add_2 = 101;
+
+ for (i = 0; i < 1000; i++) {
+ status = rte_lpm_add(lpm, ip, depth, next_hop_add_2);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) &&
+ (next_hop_return == next_hop_add_2));
+
+ status = rte_lpm_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) &&
+ (next_hop_return == next_hop_add_1));
+ }
+
+ depth = 24;
+
+ status = rte_lpm_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ rte_lpm_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Fore TBL8 extension exhaustion. Add 256 rules that require a tbl8 extension.
+ * No more tbl8 extensions will be allowed. Now add one more rule that required
+ * a tbl8 extension and get fail.
+ * */
+int32_t
+test14(void)
+{
+
+ /* We only use depth = 32 in the loop below so we must make sure
+ * that we have enough storage for all rules at that depth*/
+
+ struct rte_lpm *lpm = NULL;
+ uint32_t ip;
+ uint8_t depth, next_hop_add, next_hop_return;
+ int32_t status = 0;
+
+ /* Add enough space for 256 rules for every depth */
+ lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, 256 * 32, 0);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ depth = 32;
+ next_hop_add = 100;
+ ip = RTE_IPV4(0, 0, 0, 0);
+
+ /* Add 256 rules that require a tbl8 extension */
+ for (; ip <= RTE_IPV4(0, 0, 255, 0); ip += 256) {
+ status = rte_lpm_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) &&
+ (next_hop_return == next_hop_add));
+ }
+
+ /* All tbl8 extensions have been used above. Try to add one more and
+ * we get a fail */
+ ip = RTE_IPV4(1, 0, 0, 0);
+ depth = 32;
+
+ status = rte_lpm_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status < 0);
+
+ rte_lpm_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Sequence of operations for find existing lpm table
+ *
+ * - create table
+ * - find existing table: hit
+ * - find non-existing table: miss
+ *
+ */
+int32_t
+test15(void)
+{
+ struct rte_lpm *lpm = NULL, *result = NULL;
+
+ /* Create lpm */
+ lpm = rte_lpm_create("lpm_find_existing", SOCKET_ID_ANY, 256 * 32, 0);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ /* Try to find existing lpm */
+ result = rte_lpm_find_existing("lpm_find_existing");
+ TEST_LPM_ASSERT(result == lpm);
+
+ /* Try to find non-existing lpm */
+ result = rte_lpm_find_existing("lpm_find_non_existing");
+ TEST_LPM_ASSERT(result == NULL);
+
+ /* Cleanup. */
+ rte_lpm_delete_all(lpm);
+ rte_lpm_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * test failure condition of overloading the tbl8 so no more will fit
+ * Check we get an error return value in that case
+ */
+int32_t
+test16(void)
+{
+ uint32_t ip;
+ struct rte_lpm *lpm = rte_lpm_create(__func__, SOCKET_ID_ANY,
+ 256 * 32, 0);
+
+ /* ip loops through all possibilities for top 24 bits of address */
+ for (ip = 0; ip < 0xFFFFFF; ip++){
+ /* add an entry within a different tbl8 each time, since
+ * depth >24 and the top 24 bits are different */
+ if (rte_lpm_add(lpm, (ip << 8) + 0xF0, 30, 0) < 0)
+ break;
+ }
+
+ if (ip != RTE_LPM_TBL8_NUM_GROUPS) {
+ printf("Error, unexpected failure with filling tbl8 groups\n");
+ printf("Failed after %u additions, expected after %u\n",
+ (unsigned)ip, (unsigned)RTE_LPM_TBL8_NUM_GROUPS);
+ }
+
+ rte_lpm_free(lpm);
+ return 0;
+}
+
+/*
+ * Test for overwriting of tbl8:
+ * - add rule /32 and lookup
+ * - add new rule /24 and lookup
+ * - add third rule /25 and lookup
+ * - lookup /32 and /24 rule to ensure the table has not been overwritten.
+ */
+int32_t
+test17(void)
+{
+ struct rte_lpm *lpm = NULL;
+ const uint32_t ip_10_32 = RTE_IPV4(10, 10, 10, 2);
+ const uint32_t ip_10_24 = RTE_IPV4(10, 10, 10, 0);
+ const uint32_t ip_20_25 = RTE_IPV4(10, 10, 20, 2);
+ const uint8_t d_ip_10_32 = 32,
+ d_ip_10_24 = 24,
+ d_ip_20_25 = 25;
+ const uint8_t next_hop_ip_10_32 = 100,
+ next_hop_ip_10_24 = 105,
+ next_hop_ip_20_25 = 111;
+ uint8_t next_hop_return = 0;
+ int32_t status = 0;
+
+ lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, MAX_RULES, 0);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ if ((status = rte_lpm_add(lpm, ip_10_32, d_ip_10_32,
+ next_hop_ip_10_32)) < 0)
+ return -1;
+
+ status = rte_lpm_lookup(lpm, ip_10_32, &next_hop_return);
+ uint8_t test_hop_10_32 = next_hop_return;
+ TEST_LPM_ASSERT(status == 0);
+ TEST_LPM_ASSERT(next_hop_return == next_hop_ip_10_32);
+
+ if ((status = rte_lpm_add(lpm, ip_10_24, d_ip_10_24,
+ next_hop_ip_10_24)) < 0)
+ return -1;
+
+ status = rte_lpm_lookup(lpm, ip_10_24, &next_hop_return);
+ uint8_t test_hop_10_24 = next_hop_return;
+ TEST_LPM_ASSERT(status == 0);
+ TEST_LPM_ASSERT(next_hop_return == next_hop_ip_10_24);
+
+ if ((status = rte_lpm_add(lpm, ip_20_25, d_ip_20_25,
+ next_hop_ip_20_25)) < 0)
+ return -1;
+
+ status = rte_lpm_lookup(lpm, ip_20_25, &next_hop_return);
+ uint8_t test_hop_20_25 = next_hop_return;
+ TEST_LPM_ASSERT(status == 0);
+ TEST_LPM_ASSERT(next_hop_return == next_hop_ip_20_25);
+
+ if (test_hop_10_32 == test_hop_10_24) {
+ printf("Next hop return equal\n");
+ return -1;
+ }
+
+ if (test_hop_10_24 == test_hop_20_25){
+ printf("Next hop return equal\n");
+ return -1;
+ }
+
+ status = rte_lpm_lookup(lpm, ip_10_32, &next_hop_return);
+ TEST_LPM_ASSERT(status == 0);
+ TEST_LPM_ASSERT(next_hop_return == next_hop_ip_10_32);
+
+ status = rte_lpm_lookup(lpm, ip_10_24, &next_hop_return);
+ TEST_LPM_ASSERT(status == 0);
+ TEST_LPM_ASSERT(next_hop_return == next_hop_ip_10_24);
+
+ rte_lpm_free(lpm);
+
+ return PASS;
+}
+
+
+/*
+ * Do all unit tests.
+ */
+
+static int
+test_lpm(void)
+{
+ unsigned int i;
+ int status, global_status = 0;
+
+ for (i = 0; i < NUM_LPM_TESTS; i++) {
+ status = tests[i]();
+ if (status < 0) {
+ printf("ERROR: LPM Test %s: FAIL\n", RTE_STR(tests[i]));
+ global_status = status;
+ }
+ }
+
+ return global_status;
+}
+
+REGISTER_TEST_COMMAND_VERSION(lpm_autotest,
+ test_lpm, TEST_DPDK_ABI_VERSION_V20);
diff --git a/app/test/v2.0/test_lpm6.c b/app/test/v2.0/test_lpm6.c
new file mode 100644
index 000000000..6c6694259
--- /dev/null
+++ b/app/test/v2.0/test_lpm6.c
@@ -0,0 +1,1748 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2019 Intel Corporation
+ *
+ * LPM6 Autotests from DPDK v17.02 for v2.0 abi compatibility testing.
+ *
+ */
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+
+#include <rte_memory.h>
+/* remapping of DPDK v2.0 symbols */
+#include "dcompat.h"
+/* backported header from DPDK v2.0 */
+#include "rte_lpm6.h"
+
+#include "../test.h"
+#include "../test_lpm6_data.h"
+
+#define TEST_LPM_ASSERT(cond) do { \
+ if (!(cond)) { \
+ printf("Error at line %d: \n", __LINE__); \
+ return -1; \
+ } \
+} while(0)
+
+typedef int32_t (* rte_lpm6_test)(void);
+
+static int32_t test0(void);
+static int32_t test1(void);
+static int32_t test2(void);
+static int32_t test3(void);
+static int32_t test4(void);
+static int32_t test5(void);
+static int32_t test6(void);
+static int32_t test7(void);
+static int32_t test8(void);
+static int32_t test9(void);
+static int32_t test10(void);
+static int32_t test11(void);
+static int32_t test12(void);
+static int32_t test13(void);
+static int32_t test14(void);
+static int32_t test15(void);
+static int32_t test16(void);
+static int32_t test17(void);
+static int32_t test18(void);
+static int32_t test19(void);
+static int32_t test20(void);
+static int32_t test21(void);
+static int32_t test22(void);
+static int32_t test23(void);
+static int32_t test24(void);
+static int32_t test25(void);
+static int32_t test26(void);
+static int32_t test27(void);
+
+static rte_lpm6_test tests6[] = {
+/* Test Cases */
+ test0,
+ test1,
+ test2,
+ test3,
+ test4,
+ test5,
+ test6,
+ test7,
+ test8,
+ test9,
+ test10,
+ test11,
+ test12,
+ test13,
+ test14,
+ test15,
+ test16,
+ test17,
+ test18,
+ test19,
+ test20,
+ test21,
+ test22,
+ test23,
+ test24,
+ test25,
+ test26,
+ test27,
+};
+
+#define NUM_LPM6_TESTS (sizeof(tests6)/sizeof(tests6[0]))
+#define MAX_DEPTH 128
+#define MAX_RULES 1000000
+#define NUMBER_TBL8S (1 << 16)
+#define MAX_NUM_TBL8S (1 << 21)
+#define PASS 0
+
+static void
+IPv6(uint8_t *ip, uint8_t b1, uint8_t b2, uint8_t b3, uint8_t b4, uint8_t b5,
+ uint8_t b6, uint8_t b7, uint8_t b8, uint8_t b9, uint8_t b10,
+ uint8_t b11, uint8_t b12, uint8_t b13, uint8_t b14, uint8_t b15,
+ uint8_t b16)
+{
+ ip[0] = b1;
+ ip[1] = b2;
+ ip[2] = b3;
+ ip[3] = b4;
+ ip[4] = b5;
+ ip[5] = b6;
+ ip[6] = b7;
+ ip[7] = b8;
+ ip[8] = b9;
+ ip[9] = b10;
+ ip[10] = b11;
+ ip[11] = b12;
+ ip[12] = b13;
+ ip[13] = b14;
+ ip[14] = b15;
+ ip[15] = b16;
+}
+
+/*
+ * Check that rte_lpm6_create fails gracefully for incorrect user input
+ * arguments
+ */
+int32_t
+test0(void)
+{
+ struct rte_lpm6 *lpm = NULL;
+ struct rte_lpm6_config config;
+
+ config.max_rules = MAX_RULES;
+ config.number_tbl8s = NUMBER_TBL8S;
+ config.flags = 0;
+
+ /* rte_lpm6_create: lpm name == NULL */
+ lpm = rte_lpm6_create(NULL, SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm == NULL);
+
+ /* rte_lpm6_create: max_rules = 0 */
+ /* Note: __func__ inserts the function name, in this case "test0". */
+ config.max_rules = 0;
+ lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm == NULL);
+
+ /* socket_id < -1 is invalid */
+ config.max_rules = MAX_RULES;
+ lpm = rte_lpm6_create(__func__, -2, &config);
+ TEST_LPM_ASSERT(lpm == NULL);
+
+ /* rte_lpm6_create: number_tbl8s is bigger than the maximum */
+ config.number_tbl8s = MAX_NUM_TBL8S + 1;
+ lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm == NULL);
+
+ /* rte_lpm6_create: config = NULL */
+ lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, NULL);
+ TEST_LPM_ASSERT(lpm == NULL);
+
+ return PASS;
+}
+
+/*
+ * Creates two different LPM tables. Tries to create a third one with the same
+ * name as the first one and expects the create function to return the same
+ * pointer.
+ */
+int32_t
+test1(void)
+{
+ struct rte_lpm6 *lpm1 = NULL, *lpm2 = NULL, *lpm3 = NULL;
+ struct rte_lpm6_config config;
+
+ config.max_rules = MAX_RULES;
+ config.number_tbl8s = NUMBER_TBL8S;
+ config.flags = 0;
+
+ /* rte_lpm6_create: lpm name == LPM1 */
+ lpm1 = rte_lpm6_create("LPM1", SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm1 != NULL);
+
+ /* rte_lpm6_create: lpm name == LPM2 */
+ lpm2 = rte_lpm6_create("LPM2", SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm2 != NULL);
+
+ /* rte_lpm6_create: lpm name == LPM2 */
+ lpm3 = rte_lpm6_create("LPM1", SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm3 == NULL);
+
+ rte_lpm6_free(lpm1);
+ rte_lpm6_free(lpm2);
+
+ return PASS;
+}
+
+/*
+ * Create lpm table then delete lpm table 20 times
+ * Use a slightly different rules size each time
+ */
+int32_t
+test2(void)
+{
+ struct rte_lpm6 *lpm = NULL;
+ struct rte_lpm6_config config;
+ int32_t i;
+
+ config.number_tbl8s = NUMBER_TBL8S;
+ config.flags = 0;
+
+ /* rte_lpm6_free: Free NULL */
+ for (i = 0; i < 20; i++) {
+ config.max_rules = MAX_RULES - i;
+ lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ rte_lpm6_free(lpm);
+ }
+
+ /* Can not test free so return success */
+ return PASS;
+}
+
+/*
+ * Call rte_lpm6_free for NULL pointer user input. Note: free has no return and
+ * therefore it is impossible to check for failure but this test is added to
+ * increase function coverage metrics and to validate that freeing null does
+ * not crash.
+ */
+int32_t
+test3(void)
+{
+ struct rte_lpm6 *lpm = NULL;
+ struct rte_lpm6_config config;
+
+ config.max_rules = MAX_RULES;
+ config.number_tbl8s = NUMBER_TBL8S;
+ config.flags = 0;
+
+ lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ rte_lpm6_free(lpm);
+ rte_lpm6_free(NULL);
+ return PASS;
+}
+
+/*
+ * Check that rte_lpm6_add fails gracefully for incorrect user input arguments
+ */
+int32_t
+test4(void)
+{
+ struct rte_lpm6 *lpm = NULL;
+ struct rte_lpm6_config config;
+
+ uint8_t ip[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+ uint8_t depth = 24, next_hop = 100;
+ int32_t status = 0;
+
+ config.max_rules = MAX_RULES;
+ config.number_tbl8s = NUMBER_TBL8S;
+ config.flags = 0;
+
+ /* rte_lpm6_add: lpm == NULL */
+ status = rte_lpm6_add(NULL, ip, depth, next_hop);
+ TEST_LPM_ASSERT(status < 0);
+
+ /*Create vaild lpm to use in rest of test. */
+ lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ /* rte_lpm6_add: depth < 1 */
+ status = rte_lpm6_add(lpm, ip, 0, next_hop);
+ TEST_LPM_ASSERT(status < 0);
+
+ /* rte_lpm6_add: depth > MAX_DEPTH */
+ status = rte_lpm6_add(lpm, ip, (MAX_DEPTH + 1), next_hop);
+ TEST_LPM_ASSERT(status < 0);
+
+ rte_lpm6_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Check that rte_lpm6_delete fails gracefully for incorrect user input
+ * arguments
+ */
+int32_t
+test5(void)
+{
+ struct rte_lpm6 *lpm = NULL;
+ struct rte_lpm6_config config;
+ uint8_t ip[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+ uint8_t depth = 24;
+ int32_t status = 0;
+
+ config.max_rules = MAX_RULES;
+ config.number_tbl8s = NUMBER_TBL8S;
+ config.flags = 0;
+
+ /* rte_lpm_delete: lpm == NULL */
+ status = rte_lpm6_delete(NULL, ip, depth);
+ TEST_LPM_ASSERT(status < 0);
+
+ /*Create vaild lpm to use in rest of test. */
+ lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ /* rte_lpm_delete: depth < 1 */
+ status = rte_lpm6_delete(lpm, ip, 0);
+ TEST_LPM_ASSERT(status < 0);
+
+ /* rte_lpm_delete: depth > MAX_DEPTH */
+ status = rte_lpm6_delete(lpm, ip, (MAX_DEPTH + 1));
+ TEST_LPM_ASSERT(status < 0);
+
+ rte_lpm6_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Check that rte_lpm6_lookup fails gracefully for incorrect user input
+ * arguments
+ */
+int32_t
+test6(void)
+{
+ struct rte_lpm6 *lpm = NULL;
+ struct rte_lpm6_config config;
+ uint8_t ip[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+ uint8_t next_hop_return = 0;
+ int32_t status = 0;
+
+ config.max_rules = MAX_RULES;
+ config.number_tbl8s = NUMBER_TBL8S;
+ config.flags = 0;
+
+ /* rte_lpm6_lookup: lpm == NULL */
+ status = rte_lpm6_lookup(NULL, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status < 0);
+
+ /*Create vaild lpm to use in rest of test. */
+ lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ /* rte_lpm6_lookup: ip = NULL */
+ status = rte_lpm6_lookup(lpm, NULL, &next_hop_return);
+ TEST_LPM_ASSERT(status < 0);
+
+ /* rte_lpm6_lookup: next_hop = NULL */
+ status = rte_lpm6_lookup(lpm, ip, NULL);
+ TEST_LPM_ASSERT(status < 0);
+
+ rte_lpm6_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Checks that rte_lpm6_lookup_bulk_func fails gracefully for incorrect user
+ * input arguments
+ */
+int32_t
+test7(void)
+{
+ struct rte_lpm6 *lpm = NULL;
+ struct rte_lpm6_config config;
+ uint8_t ip[10][16];
+ int16_t next_hop_return[10];
+ int32_t status = 0;
+
+ config.max_rules = MAX_RULES;
+ config.number_tbl8s = NUMBER_TBL8S;
+ config.flags = 0;
+
+ /* rte_lpm6_lookup: lpm == NULL */
+ status = rte_lpm6_lookup_bulk_func(NULL, ip, next_hop_return, 10);
+ TEST_LPM_ASSERT(status < 0);
+
+ /*Create vaild lpm to use in rest of test. */
+ lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ /* rte_lpm6_lookup: ip = NULL */
+ status = rte_lpm6_lookup_bulk_func(lpm, NULL, next_hop_return, 10);
+ TEST_LPM_ASSERT(status < 0);
+
+ /* rte_lpm6_lookup: next_hop = NULL */
+ status = rte_lpm6_lookup_bulk_func(lpm, ip, NULL, 10);
+ TEST_LPM_ASSERT(status < 0);
+
+ rte_lpm6_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Checks that rte_lpm6_delete_bulk_func fails gracefully for incorrect user
+ * input arguments
+ */
+int32_t
+test8(void)
+{
+ struct rte_lpm6 *lpm = NULL;
+ struct rte_lpm6_config config;
+ uint8_t ip[10][16];
+ uint8_t depth[10];
+ int32_t status = 0;
+
+ config.max_rules = MAX_RULES;
+ config.number_tbl8s = NUMBER_TBL8S;
+ config.flags = 0;
+
+ /* rte_lpm6_delete: lpm == NULL */
+ status = rte_lpm6_delete_bulk_func(NULL, ip, depth, 10);
+ TEST_LPM_ASSERT(status < 0);
+
+ /*Create vaild lpm to use in rest of test. */
+ lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ /* rte_lpm6_delete: ip = NULL */
+ status = rte_lpm6_delete_bulk_func(lpm, NULL, depth, 10);
+ TEST_LPM_ASSERT(status < 0);
+
+ /* rte_lpm6_delete: next_hop = NULL */
+ status = rte_lpm6_delete_bulk_func(lpm, ip, NULL, 10);
+ TEST_LPM_ASSERT(status < 0);
+
+ rte_lpm6_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Call add, lookup and delete for a single rule with depth < 24.
+ * Check all the combinations for the first three bytes that result in a hit.
+ * Delete the rule and check that the same test returs a miss.
+ */
+int32_t
+test9(void)
+{
+ struct rte_lpm6 *lpm = NULL;
+ struct rte_lpm6_config config;
+ uint8_t ip[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+ uint8_t depth = 16, next_hop_add = 100, next_hop_return = 0;
+ int32_t status = 0;
+ uint8_t i;
+
+ config.max_rules = MAX_RULES;
+ config.number_tbl8s = NUMBER_TBL8S;
+ config.flags = 0;
+
+ lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ for (i = 0; i < UINT8_MAX; i++) {
+ ip[2] = i;
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+ }
+
+ status = rte_lpm6_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ for (i = 0; i < UINT8_MAX; i++) {
+ ip[2] = i;
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+ }
+
+ rte_lpm6_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Adds max_rules + 1 and expects a failure. Deletes a rule, then adds
+ * another one and expects success.
+ */
+int32_t
+test10(void)
+{
+ struct rte_lpm6 *lpm = NULL;
+ struct rte_lpm6_config config;
+ uint8_t ip[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+ uint8_t depth, next_hop_add = 100;
+ int32_t status = 0;
+ int i;
+
+ config.max_rules = 127;
+ config.number_tbl8s = NUMBER_TBL8S;
+ config.flags = 0;
+
+ lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ for (i = 1; i < 128; i++) {
+ depth = (uint8_t)i;
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+ }
+
+ depth = 128;
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == -ENOSPC);
+
+ depth = 127;
+ status = rte_lpm6_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ depth = 128;
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ rte_lpm6_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Creates an LPM table with a small number of tbl8s and exhaust them in the
+ * middle of the process of creating a rule.
+ */
+int32_t
+test11(void)
+{
+ struct rte_lpm6 *lpm = NULL;
+ struct rte_lpm6_config config;
+ uint8_t ip[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+ uint8_t depth, next_hop_add = 100;
+ int32_t status = 0;
+
+ config.max_rules = MAX_RULES;
+ config.number_tbl8s = 16;
+ config.flags = 0;
+
+ lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ depth = 128;
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ ip[0] = 1;
+ depth = 25;
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ depth = 33;
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ depth = 41;
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ depth = 49;
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == -ENOSPC);
+
+ depth = 41;
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ rte_lpm6_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Creates an LPM table with a small number of tbl8s and exhaust them in the
+ * middle of the process of adding a rule when there is already an existing rule
+ * in that position and needs to be extended.
+ */
+int32_t
+test12(void)
+{
+ struct rte_lpm6 *lpm = NULL;
+ struct rte_lpm6_config config;
+ uint8_t ip[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+ uint8_t depth, next_hop_add = 100;
+ int32_t status = 0;
+
+ config.max_rules = MAX_RULES;
+ config.number_tbl8s = 16;
+ config.flags = 0;
+
+ lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ depth = 128;
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ ip[0] = 1;
+ depth = 41;
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ depth = 49;
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == -ENOSPC);
+
+ rte_lpm6_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Creates an LPM table with max_rules = 2 and tries to add 3 rules.
+ * Delete one of the rules and tries to add the third one again.
+ */
+int32_t
+test13(void)
+{
+ struct rte_lpm6 *lpm = NULL;
+ struct rte_lpm6_config config;
+ uint8_t ip[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+ uint8_t depth, next_hop_add = 100;
+ int32_t status = 0;
+
+ config.max_rules = 2;
+ config.number_tbl8s = NUMBER_TBL8S;
+ config.flags = 0;
+
+ lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ depth = 1;
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ depth = 2;
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ depth = 3;
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == -ENOSPC);
+
+ depth = 2;
+ status = rte_lpm6_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ depth = 3;
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ rte_lpm6_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Add 2^12 routes with different first 12 bits and depth 25.
+ * Add one more route with the same depth and check that results in a failure.
+ * After that delete the last rule and create the one that was attempted to be
+ * created. This checks tbl8 exhaustion.
+ */
+int32_t
+test14(void)
+{
+ struct rte_lpm6 *lpm = NULL;
+ struct rte_lpm6_config config;
+ uint8_t ip[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+ uint8_t depth = 25, next_hop_add = 100;
+ int32_t status = 0;
+ int i;
+
+ config.max_rules = MAX_RULES;
+ config.number_tbl8s = 256;
+ config.flags = 0;
+
+ lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ for (i = 0; i < 256; i++) {
+ ip[0] = (uint8_t)i;
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+ }
+
+ ip[0] = 255;
+ ip[1] = 1;
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == -ENOSPC);
+
+ ip[0] = 255;
+ ip[1] = 0;
+ status = rte_lpm6_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ ip[0] = 255;
+ ip[1] = 1;
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ rte_lpm6_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Call add, lookup and delete for a single rule with depth = 24
+ */
+int32_t
+test15(void)
+{
+ struct rte_lpm6 *lpm = NULL;
+ struct rte_lpm6_config config;
+ uint8_t ip[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+ uint8_t depth = 24, next_hop_add = 100, next_hop_return = 0;
+ int32_t status = 0;
+
+ config.max_rules = MAX_RULES;
+ config.number_tbl8s = NUMBER_TBL8S;
+ config.flags = 0;
+
+ lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ status = rte_lpm6_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ rte_lpm6_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Call add, lookup and delete for a single rule with depth > 24
+ */
+int32_t
+test16(void)
+{
+ struct rte_lpm6 *lpm = NULL;
+ struct rte_lpm6_config config;
+ uint8_t ip[] = {12,12,1,0,0,0,0,0,0,0,0,0,0,0,0,0};
+ uint8_t depth = 128, next_hop_add = 100, next_hop_return = 0;
+ int32_t status = 0;
+
+ config.max_rules = MAX_RULES;
+ config.number_tbl8s = NUMBER_TBL8S;
+ config.flags = 0;
+
+ lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ status = rte_lpm6_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ rte_lpm6_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Use rte_lpm6_add to add rules which effect only the second half of the lpm
+ * table. Use all possible depths ranging from 1..32. Set the next hop = to the
+ * depth. Check lookup hit for on every add and check for lookup miss on the
+ * first half of the lpm table after each add. Finally delete all rules going
+ * backwards (i.e. from depth = 32 ..1) and carry out a lookup after each
+ * delete. The lookup should return the next_hop_add value related to the
+ * previous depth value (i.e. depth -1).
+ */
+int32_t
+test17(void)
+{
+ struct rte_lpm6 *lpm = NULL;
+ struct rte_lpm6_config config;
+ uint8_t ip1[] = {127,255,255,255,255,255,255,255,255,
+ 255,255,255,255,255,255,255};
+ uint8_t ip2[] = {128,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+ uint8_t depth, next_hop_add, next_hop_return;
+ int32_t status = 0;
+
+ config.max_rules = MAX_RULES;
+ config.number_tbl8s = NUMBER_TBL8S;
+ config.flags = 0;
+
+ lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ /* Loop with rte_lpm6_add. */
+ for (depth = 1; depth <= 16; depth++) {
+ /* Let the next_hop_add value = depth. Just for change. */
+ next_hop_add = depth;
+
+ status = rte_lpm6_add(lpm, ip2, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ /* Check IP in first half of tbl24 which should be empty. */
+ status = rte_lpm6_lookup(lpm, ip1, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ status = rte_lpm6_lookup(lpm, ip2, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) &&
+ (next_hop_return == next_hop_add));
+ }
+
+ /* Loop with rte_lpm6_delete. */
+ for (depth = 16; depth >= 1; depth--) {
+ next_hop_add = (uint8_t) (depth - 1);
+
+ status = rte_lpm6_delete(lpm, ip2, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip2, &next_hop_return);
+
+ if (depth != 1) {
+ TEST_LPM_ASSERT((status == 0) &&
+ (next_hop_return == next_hop_add));
+ }
+ else {
+ TEST_LPM_ASSERT(status == -ENOENT);
+ }
+
+ status = rte_lpm6_lookup(lpm, ip1, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+ }
+
+ rte_lpm6_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * - Add & lookup to hit invalid TBL24 entry
+ * - Add & lookup to hit valid TBL24 entry not extended
+ * - Add & lookup to hit valid extended TBL24 entry with invalid TBL8 entry
+ * - Add & lookup to hit valid extended TBL24 entry with valid TBL8 entry
+ */
+int32_t
+test18(void)
+{
+ struct rte_lpm6 *lpm = NULL;
+ struct rte_lpm6_config config;
+ uint8_t ip[16], ip_1[16], ip_2[16];
+ uint8_t depth, depth_1, depth_2, next_hop_add, next_hop_add_1,
+ next_hop_add_2, next_hop_return;
+ int32_t status = 0;
+
+ config.max_rules = MAX_RULES;
+ config.number_tbl8s = NUMBER_TBL8S;
+ config.flags = 0;
+
+ /* Add & lookup to hit invalid TBL24 entry */
+ IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ depth = 24;
+ next_hop_add = 100;
+
+ lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ status = rte_lpm6_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ rte_lpm6_delete_all(lpm);
+
+ /* Add & lookup to hit valid TBL24 entry not extended */
+ IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ depth = 23;
+ next_hop_add = 100;
+
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ depth = 24;
+ next_hop_add = 101;
+
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ depth = 24;
+
+ status = rte_lpm6_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ depth = 23;
+
+ status = rte_lpm6_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ rte_lpm6_delete_all(lpm);
+
+ /* Add & lookup to hit valid extended TBL24 entry with invalid TBL8
+ * entry.
+ */
+ IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ depth = 32;
+ next_hop_add = 100;
+
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ IPv6(ip, 128, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ depth = 32;
+ next_hop_add = 101;
+
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ depth = 32;
+ next_hop_add = 100;
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ status = rte_lpm6_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ rte_lpm6_delete_all(lpm);
+
+ /* Add & lookup to hit valid extended TBL24 entry with valid TBL8
+ * entry
+ */
+ IPv6(ip_1, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ depth_1 = 25;
+ next_hop_add_1 = 101;
+
+ IPv6(ip_2, 128, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ depth_2 = 32;
+ next_hop_add_2 = 102;
+
+ next_hop_return = 0;
+
+ status = rte_lpm6_add(lpm, ip_1, depth_1, next_hop_add_1);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip_1, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add_1));
+
+ status = rte_lpm6_add(lpm, ip_2, depth_2, next_hop_add_2);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip_2, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add_2));
+
+ status = rte_lpm6_delete(lpm, ip_2, depth_2);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip_2, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add_1));
+
+ status = rte_lpm6_delete(lpm, ip_1, depth_1);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip_1, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ rte_lpm6_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * - Add rule that covers a TBL24 range previously invalid & lookup (& delete &
+ * lookup)
+ * - Add rule that extends a TBL24 invalid entry & lookup (& delete & lookup)
+ * - Add rule that extends a TBL24 valid entry & lookup for both rules (&
+ * delete & lookup)
+ * - Add rule that updates the next hop in TBL24 & lookup (& delete & lookup)
+ * - Add rule that updates the next hop in TBL8 & lookup (& delete & lookup)
+ * - Delete a rule that is not present in the TBL24 & lookup
+ * - Delete a rule that is not present in the TBL8 & lookup
+ */
+int32_t
+test19(void)
+{
+ struct rte_lpm6 *lpm = NULL;
+ struct rte_lpm6_config config;
+ uint8_t ip[16];
+ uint8_t depth, next_hop_add, next_hop_return;
+ int32_t status = 0;
+
+ config.max_rules = MAX_RULES;
+ config.number_tbl8s = NUMBER_TBL8S;
+ config.flags = 0;
+
+ /* Add rule that covers a TBL24 range previously invalid & lookup
+ * (& delete & lookup)
+ */
+ lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ depth = 16;
+ next_hop_add = 100;
+
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ status = rte_lpm6_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ rte_lpm6_delete_all(lpm);
+
+ IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ depth = 25;
+ next_hop_add = 100;
+
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ status = rte_lpm6_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ rte_lpm6_delete_all(lpm);
+
+ /*
+ * Add rule that extends a TBL24 valid entry & lookup for both rules
+ * (& delete & lookup)
+ */
+
+ IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ depth = 24;
+ next_hop_add = 100;
+
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ IPv6(ip, 128, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ depth = 32;
+ next_hop_add = 101;
+
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ next_hop_add = 100;
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ depth = 24;
+
+ status = rte_lpm6_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ IPv6(ip, 128, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ depth = 32;
+
+ status = rte_lpm6_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ rte_lpm6_delete_all(lpm);
+
+ /*
+ * Add rule that updates the next hop in TBL24 & lookup
+ * (& delete & lookup)
+ */
+
+ IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ depth = 24;
+ next_hop_add = 100;
+
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ next_hop_add = 101;
+
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ status = rte_lpm6_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ rte_lpm6_delete_all(lpm);
+
+ /*
+ * Add rule that updates the next hop in TBL8 & lookup
+ * (& delete & lookup)
+ */
+
+ IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ depth = 32;
+ next_hop_add = 100;
+
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ next_hop_add = 101;
+
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ status = rte_lpm6_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ rte_lpm6_delete_all(lpm);
+
+ /* Delete a rule that is not present in the TBL24 & lookup */
+
+ IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ depth = 24;
+ next_hop_add = 100;
+
+ status = rte_lpm6_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status < 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ rte_lpm6_delete_all(lpm);
+
+ /* Delete a rule that is not present in the TBL8 & lookup */
+
+ IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ depth = 32;
+ next_hop_add = 100;
+
+ status = rte_lpm6_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status < 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ rte_lpm6_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Add two rules, lookup to hit the more specific one, lookup to hit the less
+ * specific one delete the less specific rule and lookup previous values again;
+ * add a more specific rule than the existing rule, lookup again
+ */
+int32_t
+test20(void)
+{
+ struct rte_lpm6 *lpm = NULL;
+ struct rte_lpm6_config config;
+ uint8_t ip[16];
+ uint8_t depth, next_hop_add, next_hop_return;
+ int32_t status = 0;
+
+ config.max_rules = MAX_RULES;
+ config.number_tbl8s = NUMBER_TBL8S;
+ config.flags = 0;
+
+ lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ depth = 24;
+ next_hop_add = 100;
+
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10);
+ depth = 128;
+ next_hop_add = 101;
+
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ next_hop_add = 100;
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add));
+
+ IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ depth = 24;
+
+ status = rte_lpm6_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10);
+ depth = 128;
+
+ status = rte_lpm6_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+
+ rte_lpm6_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Adds 3 rules and look them up through the lookup_bulk function.
+ * Includes in the lookup a fourth IP address that won't match
+ * and checks that the result is as expected.
+ */
+int32_t
+test21(void)
+{
+ struct rte_lpm6 *lpm = NULL;
+ struct rte_lpm6_config config;
+ uint8_t ip_batch[4][16];
+ uint8_t depth, next_hop_add;
+ int16_t next_hop_return[4];
+ int32_t status = 0;
+
+ config.max_rules = MAX_RULES;
+ config.number_tbl8s = NUMBER_TBL8S;
+ config.flags = 0;
+
+ lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ IPv6(ip_batch[0], 128, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ depth = 48;
+ next_hop_add = 100;
+
+ status = rte_lpm6_add(lpm, ip_batch[0], depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ IPv6(ip_batch[1], 128, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ depth = 48;
+ next_hop_add = 101;
+
+ status = rte_lpm6_add(lpm, ip_batch[1], depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ IPv6(ip_batch[2], 128, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ depth = 48;
+ next_hop_add = 102;
+
+ status = rte_lpm6_add(lpm, ip_batch[2], depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ IPv6(ip_batch[3], 128, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+
+ status = rte_lpm6_lookup_bulk_func(lpm, ip_batch,
+ next_hop_return, 4);
+ TEST_LPM_ASSERT(status == 0 && next_hop_return[0] == 100
+ && next_hop_return[1] == 101 && next_hop_return[2] == 102
+ && next_hop_return[3] == -1);
+
+ rte_lpm6_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Adds 5 rules and look them up.
+ * Use the delete_bulk function to delete two of them. Lookup again.
+ * Use the delete_bulk function to delete one more. Lookup again.
+ * Use the delete_bulk function to delete two more, one invalid. Lookup again.
+ * Use the delete_bulk function to delete the remaining one. Lookup again.
+ */
+int32_t
+test22(void)
+{
+ struct rte_lpm6 *lpm = NULL;
+ struct rte_lpm6_config config;
+ uint8_t ip_batch[5][16];
+ uint8_t depth[5], next_hop_add;
+ int16_t next_hop_return[5];
+ int32_t status = 0;
+
+ config.max_rules = MAX_RULES;
+ config.number_tbl8s = NUMBER_TBL8S;
+ config.flags = 0;
+
+ lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ /* Adds 5 rules and look them up */
+
+ IPv6(ip_batch[0], 128, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ depth[0] = 48;
+ next_hop_add = 101;
+
+ status = rte_lpm6_add(lpm, ip_batch[0], depth[0], next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ IPv6(ip_batch[1], 128, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ depth[1] = 48;
+ next_hop_add = 102;
+
+ status = rte_lpm6_add(lpm, ip_batch[1], depth[1], next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ IPv6(ip_batch[2], 128, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ depth[2] = 48;
+ next_hop_add = 103;
+
+ status = rte_lpm6_add(lpm, ip_batch[2], depth[2], next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ IPv6(ip_batch[3], 128, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ depth[3] = 48;
+ next_hop_add = 104;
+
+ status = rte_lpm6_add(lpm, ip_batch[3], depth[3], next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ IPv6(ip_batch[4], 128, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ depth[4] = 48;
+ next_hop_add = 105;
+
+ status = rte_lpm6_add(lpm, ip_batch[4], depth[4], next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup_bulk_func(lpm, ip_batch,
+ next_hop_return, 5);
+ TEST_LPM_ASSERT(status == 0 && next_hop_return[0] == 101
+ && next_hop_return[1] == 102 && next_hop_return[2] == 103
+ && next_hop_return[3] == 104 && next_hop_return[4] == 105);
+
+ /* Use the delete_bulk function to delete two of them. Lookup again */
+
+ status = rte_lpm6_delete_bulk_func(lpm, &ip_batch[0], depth, 2);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup_bulk_func(lpm, ip_batch,
+ next_hop_return, 5);
+ TEST_LPM_ASSERT(status == 0 && next_hop_return[0] == -1
+ && next_hop_return[1] == -1 && next_hop_return[2] == 103
+ && next_hop_return[3] == 104 && next_hop_return[4] == 105);
+
+ /* Use the delete_bulk function to delete one more. Lookup again */
+
+ status = rte_lpm6_delete_bulk_func(lpm, &ip_batch[2], depth, 1);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup_bulk_func(lpm, ip_batch,
+ next_hop_return, 5);
+ TEST_LPM_ASSERT(status == 0 && next_hop_return[0] == -1
+ && next_hop_return[1] == -1 && next_hop_return[2] == -1
+ && next_hop_return[3] == 104 && next_hop_return[4] == 105);
+
+ /* Use the delete_bulk function to delete two, one invalid. Lookup again */
+
+ IPv6(ip_batch[4], 128, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ status = rte_lpm6_delete_bulk_func(lpm, &ip_batch[3], depth, 2);
+ TEST_LPM_ASSERT(status == 0);
+
+ IPv6(ip_batch[4], 128, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ status = rte_lpm6_lookup_bulk_func(lpm, ip_batch,
+ next_hop_return, 5);
+ TEST_LPM_ASSERT(status == 0 && next_hop_return[0] == -1
+ && next_hop_return[1] == -1 && next_hop_return[2] == -1
+ && next_hop_return[3] == -1 && next_hop_return[4] == 105);
+
+ /* Use the delete_bulk function to delete the remaining one. Lookup again */
+
+ status = rte_lpm6_delete_bulk_func(lpm, &ip_batch[4], depth, 1);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup_bulk_func(lpm, ip_batch,
+ next_hop_return, 5);
+ TEST_LPM_ASSERT(status == 0 && next_hop_return[0] == -1
+ && next_hop_return[1] == -1 && next_hop_return[2] == -1
+ && next_hop_return[3] == -1 && next_hop_return[4] == -1);
+
+ rte_lpm6_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Add an extended rule (i.e. depth greater than 24, lookup (hit), delete,
+ * lookup (miss) in a for loop of 30 times. This will check tbl8 extension
+ * and contraction.
+ */
+int32_t
+test23(void)
+{
+ struct rte_lpm6 *lpm = NULL;
+ struct rte_lpm6_config config;
+ uint32_t i;
+ uint8_t ip[16];
+ uint8_t depth, next_hop_add, next_hop_return;
+ int32_t status = 0;
+
+ config.max_rules = MAX_RULES;
+ config.number_tbl8s = NUMBER_TBL8S;
+ config.flags = 0;
+
+ lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ depth = 128;
+ next_hop_add = 100;
+
+ for (i = 0; i < 30; i++) {
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) &&
+ (next_hop_return == next_hop_add));
+
+ status = rte_lpm6_delete(lpm, ip, depth);
+ TEST_LPM_ASSERT(status == 0);
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT(status == -ENOENT);
+ }
+
+ rte_lpm6_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Sequence of operations for find existing lpm table
+ *
+ * - create table
+ * - find existing table: hit
+ * - find non-existing table: miss
+ */
+int32_t
+test24(void)
+{
+ struct rte_lpm6 *lpm = NULL, *result = NULL;
+ struct rte_lpm6_config config;
+
+ config.max_rules = 256 * 32;
+ config.number_tbl8s = NUMBER_TBL8S;
+ config.flags = 0;
+
+ /* Create lpm */
+ lpm = rte_lpm6_create("lpm_find_existing", SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ /* Try to find existing lpm */
+ result = rte_lpm6_find_existing("lpm_find_existing");
+ TEST_LPM_ASSERT(result == lpm);
+
+ /* Try to find non-existing lpm */
+ result = rte_lpm6_find_existing("lpm_find_non_existing");
+ TEST_LPM_ASSERT(result == NULL);
+
+ /* Cleanup. */
+ rte_lpm6_delete_all(lpm);
+ rte_lpm6_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Add a set of random routes with random depths.
+ * Lookup different IP addresses that match the routes previously added.
+ * Checks that the next hop is the expected one.
+ * The routes, IP addresses and expected result for every case have been
+ * precalculated by using a python script and stored in a .h file.
+ */
+int32_t
+test25(void)
+{
+ struct rte_lpm6 *lpm = NULL;
+ struct rte_lpm6_config config;
+ uint8_t ip[16];
+ uint32_t i;
+ uint8_t depth, next_hop_add, next_hop_return, next_hop_expected;
+ int32_t status = 0;
+
+ config.max_rules = MAX_RULES;
+ config.number_tbl8s = NUMBER_TBL8S;
+ config.flags = 0;
+
+ lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ for (i = 0; i < 1000; i++) {
+ memcpy(ip, large_route_table[i].ip, 16);
+ depth = large_route_table[i].depth;
+ next_hop_add = large_route_table[i].next_hop;
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+ }
+
+ /* generate large IPS table and expected next_hops */
+ generate_large_ips_table(1);
+
+ for (i = 0; i < 100000; i++) {
+ memcpy(ip, large_ips_table[i].ip, 16);
+ next_hop_expected = large_ips_table[i].next_hop;
+
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ TEST_LPM_ASSERT((status == 0) &&
+ (next_hop_return == next_hop_expected));
+ }
+
+ rte_lpm6_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Test for overwriting of tbl8:
+ * - add rule /32 and lookup
+ * - add new rule /24 and lookup
+ * - add third rule /25 and lookup
+ * - lookup /32 and /24 rule to ensure the table has not been overwritten.
+ */
+int32_t
+test26(void)
+{
+ struct rte_lpm6 *lpm = NULL;
+ struct rte_lpm6_config config;
+ uint8_t ip_10_32[] = {10, 10, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+ uint8_t ip_10_24[] = {10, 10, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+ uint8_t ip_20_25[] = {10, 10, 20, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+ uint8_t d_ip_10_32 = 32;
+ uint8_t d_ip_10_24 = 24;
+ uint8_t d_ip_20_25 = 25;
+ uint8_t next_hop_ip_10_32 = 100;
+ uint8_t next_hop_ip_10_24 = 105;
+ uint8_t next_hop_ip_20_25 = 111;
+ uint8_t next_hop_return = 0;
+ int32_t status = 0;
+
+ config.max_rules = MAX_RULES;
+ config.number_tbl8s = NUMBER_TBL8S;
+ config.flags = 0;
+
+ lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ if ((status = rte_lpm6_add(lpm, ip_10_32, d_ip_10_32,
+ next_hop_ip_10_32)) < 0)
+ return -1;
+
+ status = rte_lpm6_lookup(lpm, ip_10_32, &next_hop_return);
+ uint8_t test_hop_10_32 = next_hop_return;
+ TEST_LPM_ASSERT(status == 0);
+ TEST_LPM_ASSERT(next_hop_return == next_hop_ip_10_32);
+
+ if ((status = rte_lpm6_add(lpm, ip_10_24, d_ip_10_24,
+ next_hop_ip_10_24)) < 0)
+ return -1;
+
+ status = rte_lpm6_lookup(lpm, ip_10_24, &next_hop_return);
+ uint8_t test_hop_10_24 = next_hop_return;
+ TEST_LPM_ASSERT(status == 0);
+ TEST_LPM_ASSERT(next_hop_return == next_hop_ip_10_24);
+
+ if ((status = rte_lpm6_add(lpm, ip_20_25, d_ip_20_25,
+ next_hop_ip_20_25)) < 0)
+ return -1;
+
+ status = rte_lpm6_lookup(lpm, ip_20_25, &next_hop_return);
+ uint8_t test_hop_20_25 = next_hop_return;
+ TEST_LPM_ASSERT(status == 0);
+ TEST_LPM_ASSERT(next_hop_return == next_hop_ip_20_25);
+
+ if (test_hop_10_32 == test_hop_10_24) {
+ printf("Next hop return equal\n");
+ return -1;
+ }
+
+ if (test_hop_10_24 == test_hop_20_25){
+ printf("Next hop return equal\n");
+ return -1;
+ }
+
+ status = rte_lpm6_lookup(lpm, ip_10_32, &next_hop_return);
+ TEST_LPM_ASSERT(status == 0);
+ TEST_LPM_ASSERT(next_hop_return == next_hop_ip_10_32);
+
+ status = rte_lpm6_lookup(lpm, ip_10_24, &next_hop_return);
+ TEST_LPM_ASSERT(status == 0);
+ TEST_LPM_ASSERT(next_hop_return == next_hop_ip_10_24);
+
+ rte_lpm6_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Add a rule that reaches the end of the tree.
+ * Add a rule that is more generic than the first one.
+ * Check every possible combination that produces a match for the second rule.
+ * This tests tbl expansion.
+ */
+int32_t
+test27(void)
+{
+ struct rte_lpm6 *lpm = NULL;
+ struct rte_lpm6_config config;
+ uint8_t ip[] = {128,128,128,128,128,128,128,128,128,128,128,128,128,128,0,0};
+ uint8_t depth = 128, next_hop_add = 100, next_hop_return;
+ int32_t status = 0;
+ int i, j;
+
+ config.max_rules = MAX_RULES;
+ config.number_tbl8s = NUMBER_TBL8S;
+ config.flags = 0;
+
+ lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ depth = 128;
+ next_hop_add = 128;
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ depth = 112;
+ next_hop_add = 112;
+ status = rte_lpm6_add(lpm, ip, depth, next_hop_add);
+ TEST_LPM_ASSERT(status == 0);
+
+ for (i = 0; i < 256; i++) {
+ ip[14] = (uint8_t)i;
+ for (j = 0; j < 256; j++) {
+ ip[15] = (uint8_t)j;
+ status = rte_lpm6_lookup(lpm, ip, &next_hop_return);
+ if (i == 0 && j == 0)
+ TEST_LPM_ASSERT(status == 0 && next_hop_return == 128);
+ else
+ TEST_LPM_ASSERT(status == 0 && next_hop_return == 112);
+ }
+ }
+
+ rte_lpm6_free(lpm);
+
+ return PASS;
+}
+
+/*
+ * Do all unit and performance tests.
+ */
+static int
+test_lpm6(void)
+{
+ unsigned i;
+ int status = -1, global_status = 0;
+
+ for (i = 0; i < NUM_LPM6_TESTS; i++) {
+ printf("# test %02d\n", i);
+ status = tests6[i]();
+
+ if (status < 0) {
+ printf("ERROR: LPM Test %s: FAIL\n", RTE_STR(tests6[i]));
+ global_status = status;
+ }
+ }
+
+ return global_status;
+}
+
+REGISTER_TEST_COMMAND_VERSION(lpm6_autotest,
+ test_lpm6, TEST_DPDK_ABI_VERSION_V20);
diff --git a/app/test/v2.0/test_lpm6_perf.c b/app/test/v2.0/test_lpm6_perf.c
new file mode 100644
index 000000000..abbd0a30e
--- /dev/null
+++ b/app/test/v2.0/test_lpm6_perf.c
@@ -0,0 +1,179 @@
+
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2019 Intel Corporation
+ *
+ * LPM6 Autotests from DPDK v17.02 for v2.0 abi compatibility testing.
+ *
+ */
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <rte_cycles.h>
+#include <rte_random.h>
+#include <rte_memory.h>
+
+/* remapping of DPDK v2.0 symbols */
+#include "dcompat.h"
+/* backported header from DPDK v2.0 */
+#include "rte_lpm6.h"
+
+#include "../test.h"
+#include "../test_lpm6_data.h"
+
+#define TEST_LPM_ASSERT(cond) do { \
+ if (!(cond)) { \
+ printf("Error at line %d: \n", __LINE__); \
+ return -1; \
+ } \
+ } while(0)
+
+static int32_t test_lpm6_perf(void);
+
+#define NUMBER_TBL8S (1 << 16)
+#define PASS 0
+
+/*
+ * Lookup performance test
+ */
+
+#define ITERATIONS (1 << 10)
+#define BATCH_SIZE 100000
+
+static void
+print_route_distribution(const struct rules_tbl_entry *table, uint32_t n)
+{
+ unsigned i, j;
+
+ printf("Route distribution per prefix width: \n");
+ printf("DEPTH QUANTITY (PERCENT)\n");
+ printf("--------------------------- \n");
+
+ /* Count depths. */
+ for(i = 1; i <= 128; i++) {
+ unsigned depth_counter = 0;
+ double percent_hits;
+
+ for (j = 0; j < n; j++)
+ if (table[j].depth == (uint8_t) i)
+ depth_counter++;
+
+ percent_hits = ((double)depth_counter)/((double)n) * 100;
+ printf("%.2u%15u (%.2f)\n", i, depth_counter, percent_hits);
+ }
+ printf("\n");
+}
+
+static int
+test_lpm6_perf(void)
+{
+ struct rte_lpm6 *lpm = NULL;
+ struct rte_lpm6_config config;
+ uint64_t begin, total_time;
+ unsigned i, j;
+ uint8_t next_hop_add = 0xAA, next_hop_return = 0;
+ int status = 0;
+ int64_t count = 0;
+
+ config.max_rules = 1000000;
+ config.number_tbl8s = NUMBER_TBL8S;
+ config.flags = 0;
+
+ rte_srand(rte_rdtsc());
+
+ printf("No. routes = %u\n", (unsigned) NUM_ROUTE_ENTRIES);
+
+ print_route_distribution(large_route_table, (uint32_t) NUM_ROUTE_ENTRIES);
+
+ /* Only generate IPv6 address of each item in large IPS table,
+ * here next_hop is not needed.
+ */
+ generate_large_ips_table(0);
+
+ lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ /* Measure add. */
+ begin = rte_rdtsc();
+
+ for (i = 0; i < NUM_ROUTE_ENTRIES; i++) {
+ if (rte_lpm6_add(lpm, large_route_table[i].ip,
+ large_route_table[i].depth, next_hop_add) == 0)
+ status++;
+ }
+ /* End Timer. */
+ total_time = rte_rdtsc() - begin;
+
+ printf("Unique added entries = %d\n", status);
+ printf("Average LPM Add: %g cycles\n",
+ (double)total_time / NUM_ROUTE_ENTRIES);
+
+ /* Measure single Lookup */
+ total_time = 0;
+ count = 0;
+
+ for (i = 0; i < ITERATIONS; i ++) {
+ begin = rte_rdtsc();
+
+ for (j = 0; j < NUM_IPS_ENTRIES; j ++) {
+ if (rte_lpm6_lookup(lpm, large_ips_table[j].ip,
+ &next_hop_return) != 0)
+ count++;
+ }
+
+ total_time += rte_rdtsc() - begin;
+
+ }
+ printf("Average LPM Lookup: %.1f cycles (fails = %.1f%%)\n",
+ (double)total_time / ((double)ITERATIONS * BATCH_SIZE),
+ (count * 100.0) / (double)(ITERATIONS * BATCH_SIZE));
+
+ /* Measure bulk Lookup */
+ total_time = 0;
+ count = 0;
+
+ uint8_t ip_batch[NUM_IPS_ENTRIES][16];
+ int16_t next_hops[NUM_IPS_ENTRIES];
+
+ for (i = 0; i < NUM_IPS_ENTRIES; i++)
+ memcpy(ip_batch[i], large_ips_table[i].ip, 16);
+
+ for (i = 0; i < ITERATIONS; i ++) {
+
+ /* Lookup per batch */
+ begin = rte_rdtsc();
+ rte_lpm6_lookup_bulk_func(lpm, ip_batch, next_hops, NUM_IPS_ENTRIES);
+ total_time += rte_rdtsc() - begin;
+
+ for (j = 0; j < NUM_IPS_ENTRIES; j++)
+ if (next_hops[j] < 0)
+ count++;
+ }
+ printf("BULK LPM Lookup: %.1f cycles (fails = %.1f%%)\n",
+ (double)total_time / ((double)ITERATIONS * BATCH_SIZE),
+ (count * 100.0) / (double)(ITERATIONS * BATCH_SIZE));
+
+ /* Delete */
+ status = 0;
+ begin = rte_rdtsc();
+
+ for (i = 0; i < NUM_ROUTE_ENTRIES; i++) {
+ /* rte_lpm_delete(lpm, ip, depth) */
+ status += rte_lpm6_delete(lpm, large_route_table[i].ip,
+ large_route_table[i].depth);
+ }
+
+ total_time += rte_rdtsc() - begin;
+
+ printf("Average LPM Delete: %g cycles\n",
+ (double)total_time / NUM_ROUTE_ENTRIES);
+
+ rte_lpm6_delete_all(lpm);
+ rte_lpm6_free(lpm);
+
+ return PASS;
+}
+
+REGISTER_TEST_COMMAND_VERSION(lpm6_perf_autotest,
+ test_lpm6_perf, TEST_DPDK_ABI_VERSION_V20);
diff --git a/app/test/v2.0/test_lpm_perf.c b/app/test/v2.0/test_lpm_perf.c
new file mode 100644
index 000000000..3c16213fe
--- /dev/null
+++ b/app/test/v2.0/test_lpm_perf.c
@@ -0,0 +1,212 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2019 Intel Corporation
+ *
+ * LPM Autotests from DPDK v2.2.0 for v2.0 abi compability testing.
+ *
+ */
+
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <sys/queue.h>
+
+#include <rte_common.h>
+#include <rte_cycles.h>
+#include <rte_memory.h>
+#include <rte_random.h>
+#include <rte_branch_prediction.h>
+#include <rte_ip.h>
+#include <time.h>
+
+#include "../test.h"
+
+/* remapping of DPDK v2.0 symbols */
+#include "dcompat.h"
+/* backported header from DPDK v2.0 */
+#include "rte_lpm.h"
+#include "../test_lpm_routes.h"
+
+#define TEST_LPM_ASSERT(cond) do { \
+ if (!(cond)) { \
+ printf("Error at line %d:\n", __LINE__); \
+ return -1; \
+ } \
+ } while (0)
+
+
+#define PASS 0
+
+/*
+ * Lookup performance test
+ */
+
+#define ITERATIONS (1 << 10)
+#define BATCH_SIZE (1 << 12)
+#define BULK_SIZE 32
+
+static int32_t test_lpm_perf(void);
+
+int32_t
+test_lpm_perf(void)
+{
+ struct rte_lpm *lpm = NULL;
+ uint64_t begin, total_time, lpm_used_entries = 0;
+ unsigned i, j;
+ uint8_t next_hop_add = 0xAA, next_hop_return = 0;
+ int status = 0;
+ uint64_t cache_line_counter = 0;
+ int64_t count = 0;
+
+ rte_srand(rte_rdtsc());
+
+ /* (re) generate the routing table */
+ generate_large_route_rule_table();
+
+ printf("No. routes = %u\n", (unsigned) NUM_ROUTE_ENTRIES);
+
+ print_route_distribution(large_route_table,
+ (uint32_t) NUM_ROUTE_ENTRIES);
+
+ lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, 1000000, 0);
+ TEST_LPM_ASSERT(lpm != NULL);
+
+ /* Measue add. */
+ begin = rte_rdtsc();
+
+ for (i = 0; i < NUM_ROUTE_ENTRIES; i++) {
+ if (rte_lpm_add(lpm, large_route_table[i].ip,
+ large_route_table[i].depth, next_hop_add) == 0)
+ status++;
+ }
+ /* End Timer. */
+ total_time = rte_rdtsc() - begin;
+
+ printf("Unique added entries = %d\n", status);
+ /* Obtain add statistics. */
+ for (i = 0; i < RTE_LPM_TBL24_NUM_ENTRIES; i++) {
+ if (lpm->tbl24[i].valid)
+ lpm_used_entries++;
+
+ if (i % 32 == 0){
+ if ((uint64_t)count < lpm_used_entries) {
+ cache_line_counter++;
+ count = lpm_used_entries;
+ }
+ }
+ }
+
+ printf("Used table 24 entries = %u (%g%%)\n",
+ (unsigned) lpm_used_entries,
+ (lpm_used_entries * 100.0) / RTE_LPM_TBL24_NUM_ENTRIES);
+ printf("64 byte Cache entries used = %u (%u bytes)\n",
+ (unsigned) cache_line_counter, (unsigned) cache_line_counter * 64);
+
+ printf("Average LPM Add: %g cycles\n", (double)total_time / NUM_ROUTE_ENTRIES);
+
+ /* Measure single Lookup */
+ total_time = 0;
+ count = 0;
+
+ for (i = 0; i < ITERATIONS; i++) {
+ static uint32_t ip_batch[BATCH_SIZE];
+
+ for (j = 0; j < BATCH_SIZE; j++)
+ ip_batch[j] = rte_rand();
+
+ /* Lookup per batch */
+ begin = rte_rdtsc();
+
+ for (j = 0; j < BATCH_SIZE; j++) {
+ if (rte_lpm_lookup(lpm, ip_batch[j], &next_hop_return) != 0)
+ count++;
+ }
+
+ total_time += rte_rdtsc() - begin;
+
+ }
+ printf("Average LPM Lookup: %.1f cycles (fails = %.1f%%)\n",
+ (double)total_time / ((double)ITERATIONS * BATCH_SIZE),
+ (count * 100.0) / (double)(ITERATIONS * BATCH_SIZE));
+
+ /* Measure bulk Lookup */
+ total_time = 0;
+ count = 0;
+ for (i = 0; i < ITERATIONS; i++) {
+ static uint32_t ip_batch[BATCH_SIZE];
+ uint16_t next_hops[BULK_SIZE];
+
+ /* Create array of random IP addresses */
+ for (j = 0; j < BATCH_SIZE; j++)
+ ip_batch[j] = rte_rand();
+
+ /* Lookup per batch */
+ begin = rte_rdtsc();
+ for (j = 0; j < BATCH_SIZE; j += BULK_SIZE) {
+ unsigned k;
+ rte_lpm_lookup_bulk(lpm, &ip_batch[j], next_hops, BULK_SIZE);
+ for (k = 0; k < BULK_SIZE; k++)
+ if (unlikely(!(next_hops[k] & RTE_LPM_LOOKUP_SUCCESS)))
+ count++;
+ }
+
+ total_time += rte_rdtsc() - begin;
+ }
+ printf("BULK LPM Lookup: %.1f cycles (fails = %.1f%%)\n",
+ (double)total_time / ((double)ITERATIONS * BATCH_SIZE),
+ (count * 100.0) / (double)(ITERATIONS * BATCH_SIZE));
+
+ /* Measure LookupX4 */
+ total_time = 0;
+ count = 0;
+ for (i = 0; i < ITERATIONS; i++) {
+ static uint32_t ip_batch[BATCH_SIZE];
+ uint16_t next_hops[4];
+
+ /* Create array of random IP addresses */
+ for (j = 0; j < BATCH_SIZE; j++)
+ ip_batch[j] = rte_rand();
+
+ /* Lookup per batch */
+ begin = rte_rdtsc();
+ for (j = 0; j < BATCH_SIZE; j += RTE_DIM(next_hops)) {
+ unsigned k;
+ __m128i ipx4;
+
+ ipx4 = _mm_loadu_si128((__m128i *)(ip_batch + j));
+ ipx4 = *(__m128i *)(ip_batch + j);
+ rte_lpm_lookupx4(lpm, ipx4, next_hops, UINT16_MAX);
+ for (k = 0; k < RTE_DIM(next_hops); k++)
+ if (unlikely(next_hops[k] == UINT16_MAX))
+ count++;
+ }
+
+ total_time += rte_rdtsc() - begin;
+ }
+ printf("LPM LookupX4: %.1f cycles (fails = %.1f%%)\n",
+ (double)total_time / ((double)ITERATIONS * BATCH_SIZE),
+ (count * 100.0) / (double)(ITERATIONS * BATCH_SIZE));
+
+ /* Delete */
+ status = 0;
+ begin = rte_rdtsc();
+
+ for (i = 0; i < NUM_ROUTE_ENTRIES; i++) {
+ /* rte_lpm_delete(lpm, ip, depth) */
+ status += rte_lpm_delete(lpm, large_route_table[i].ip,
+ large_route_table[i].depth);
+ }
+
+ total_time += rte_rdtsc() - begin;
+
+ printf("Average LPM Delete: %g cycles\n",
+ (double)total_time / NUM_ROUTE_ENTRIES);
+
+ rte_lpm_delete_all(lpm);
+ rte_lpm_free(lpm);
+
+ return PASS;
+}
+
+REGISTER_TEST_COMMAND_VERSION(lpm_perf_autotest,
+ test_lpm_perf, TEST_DPDK_ABI_VERSION_V20);
diff --git a/app/test/v2.0/test_v20.c b/app/test/v2.0/test_v20.c
new file mode 100644
index 000000000..6285e2882
--- /dev/null
+++ b/app/test/v2.0/test_v20.c
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2014 Intel Corporation
+ */
+
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#include <rte_ip.h>
+#include <rte_lpm.h>
+
+#include "../test.h"
+
+REGISTER_TEST_ABI_VERSION(v20, TEST_DPDK_ABI_VERSION_V20);
--
2.17.1
^ permalink raw reply [relevance 4%]
* [dpdk-dev] [PATCH v2 1/2] app/test: add abi version testing functionality
2019-08-22 16:07 9% [dpdk-dev] [PATCH v2 0/2] add abi version testing to app/test Ray Kinsella
@ 2019-08-22 16:07 14% ` Ray Kinsella
2019-08-22 16:07 4% ` [dpdk-dev] [PATCH v2 2/2] app/test: lpm abi version testing Ray Kinsella
2019-08-23 15:49 4% ` [dpdk-dev] [PATCH v2 0/2] add abi version testing to app/test Aaron Conole
2 siblings, 0 replies; 200+ results
From: Ray Kinsella @ 2019-08-22 16:07 UTC (permalink / raw)
To: dev
Cc: mdr, bruce.richardson, vladimir.medvedkin, john.mcnamara,
marko.kovacevic
This patchset adds ABI Version Testing functionality to the app/test
unit test framework, comprised of
1. The TEST_DPDK_ABI_VERSION_* and REGISTER_TEST_ABI_VERSION macros to
register abi versions with infrastructure.
2. The MAP_ABI_SYMBOL_VERSION macro to remap symbols based on their ABI
version.
3. The set_abi_version command to switch between ABI versions.
4. The BIND_VERSION_SYMBOL macro to bind against specific symbol
versions.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
app/test/commands.c | 131 ++++++++++++++++++---
app/test/test.c | 2 +
app/test/test.h | 52 ++++++--
doc/guides/contributing/versioning.rst | 4 +
lib/librte_eal/common/include/rte_compat.h | 7 ++
5 files changed, 170 insertions(+), 26 deletions(-)
diff --git a/app/test/commands.c b/app/test/commands.c
index 8d5a03a95..06fc33ee5 100644
--- a/app/test/commands.c
+++ b/app/test/commands.c
@@ -50,12 +50,22 @@
/****************/
+static uint8_t test_abi_version = TEST_DPDK_ABI_VERSION_DEFAULT;
+
+static struct test_abi_version_list abi_version_list =
+ TAILQ_HEAD_INITIALIZER(abi_version_list);
+
static struct test_commands_list commands_list =
TAILQ_HEAD_INITIALIZER(commands_list);
-void
-add_test_command(struct test_command *t)
+void add_abi_version(struct test_abi_version *av)
+{
+ TAILQ_INSERT_TAIL(&abi_version_list, av, next);
+}
+
+void add_test_command(struct test_command *t, uint8_t abi_version)
{
+ t->abi_version = abi_version;
TAILQ_INSERT_TAIL(&commands_list, t, next);
}
@@ -63,6 +73,12 @@ struct cmd_autotest_result {
cmdline_fixed_string_t autotest;
};
+cmdline_parse_token_string_t
+cmd_autotest_autotest[TEST_DPDK_ABI_VERSION_MAX] = {
+ [0 ... TEST_DPDK_ABI_VERSION_MAX-1] =
+ TOKEN_STRING_INITIALIZER(struct cmd_autotest_result, autotest, "")
+};
+
static void cmd_autotest_parsed(void *parsed_result,
__attribute__((unused)) struct cmdline *cl,
__attribute__((unused)) void *data)
@@ -72,7 +88,8 @@ static void cmd_autotest_parsed(void *parsed_result,
int ret = 0;
TAILQ_FOREACH(t, &commands_list, next) {
- if (!strcmp(res->autotest, t->command))
+ if (!strcmp(res->autotest, t->command)
+ && t->abi_version == test_abi_version)
ret = t->callback();
}
@@ -86,10 +103,6 @@ static void cmd_autotest_parsed(void *parsed_result,
fflush(stdout);
}
-cmdline_parse_token_string_t cmd_autotest_autotest =
- TOKEN_STRING_INITIALIZER(struct cmd_autotest_result, autotest,
- "");
-
cmdline_parse_inst_t cmd_autotest = {
.f = cmd_autotest_parsed, /* function to call */
.data = NULL, /* 2nd arg of func */
@@ -244,6 +257,53 @@ cmdline_parse_inst_t cmd_quit = {
/****************/
+struct cmd_set_abi_version_result {
+ cmdline_fixed_string_t set;
+ cmdline_fixed_string_t abi_version_name;
+};
+
+static void cmd_set_abi_version_parsed(
+ void *parsed_result,
+ __attribute__((unused)) struct cmdline *cl,
+ __attribute__((unused)) void *data)
+{
+ struct test_abi_version *av;
+ struct cmd_set_abi_version_result *res = parsed_result;
+
+ TAILQ_FOREACH(av, &abi_version_list, next) {
+ if (!strcmp(res->abi_version_name, av->version_name)) {
+
+ printf("abi version set to %s\n", av->version_name);
+ test_abi_version = av->version_id;
+ cmd_autotest.tokens[0] =
+ (void *)&cmd_autotest_autotest[av->version_id];
+ }
+ }
+
+ fflush(stdout);
+}
+
+cmdline_parse_token_string_t cmd_set_abi_version_set =
+ TOKEN_STRING_INITIALIZER(struct cmd_set_abi_version_result, set,
+ "set_abi_version");
+
+cmdline_parse_token_string_t cmd_set_abi_version_abi_version =
+ TOKEN_STRING_INITIALIZER(struct cmd_set_abi_version_result,
+ abi_version_name, NULL);
+
+cmdline_parse_inst_t cmd_set_abi_version = {
+ .f = cmd_set_abi_version_parsed, /* function to call */
+ .data = NULL, /* 2nd arg of func */
+ .help_str = "set abi version: ",
+ .tokens = { /* token list, NULL terminated */
+ (void *)&cmd_set_abi_version_set,
+ (void *)&cmd_set_abi_version_abi_version,
+ NULL,
+ },
+};
+
+/****************/
+
struct cmd_set_rxtx_result {
cmdline_fixed_string_t set;
cmdline_fixed_string_t mode;
@@ -259,7 +319,7 @@ static void cmd_set_rxtx_parsed(void *parsed_result, struct cmdline *cl,
cmdline_parse_token_string_t cmd_set_rxtx_set =
TOKEN_STRING_INITIALIZER(struct cmd_set_rxtx_result, set,
- "set_rxtx_mode");
+ "set_rxtx_mode");
cmdline_parse_token_string_t cmd_set_rxtx_mode =
TOKEN_STRING_INITIALIZER(struct cmd_set_rxtx_result, mode, NULL);
@@ -360,29 +420,66 @@ cmdline_parse_ctx_t main_ctx[] = {
(cmdline_parse_inst_t *)&cmd_set_rxtx,
(cmdline_parse_inst_t *)&cmd_set_rxtx_anchor,
(cmdline_parse_inst_t *)&cmd_set_rxtx_sc,
+ (cmdline_parse_inst_t *)&cmd_set_abi_version,
NULL,
};
int commands_init(void)
{
+ struct test_abi_version *av;
struct test_command *t;
- char *commands;
- int commands_len = 0;
+ char *commands[TEST_DPDK_ABI_VERSION_MAX];
+ char *help;
+
+ int commands_len[TEST_DPDK_ABI_VERSION_MAX] = {
+ [0 ... TEST_DPDK_ABI_VERSION_MAX-1] = 0
+ };
+ int help_len = strlen(cmd_set_abi_version.help_str);
+ int abi_version;
+
+ /* set the set_abi_version command help string */
+ TAILQ_FOREACH(av, &abi_version_list, next) {
+ help_len += strlen(av->version_name) + 1;
+ }
+
+ help = (char *)calloc(help_len, sizeof(char));
+ if (!help)
+ return -1;
+
+ strlcat(help, cmd_set_abi_version.help_str, help_len);
+ TAILQ_FOREACH(av, &abi_version_list, next) {
+ strlcat(help, av->version_name, help_len);
+ if (TAILQ_NEXT(av, next) != NULL)
+ strlcat(help, "|", help_len);
+ }
+
+ cmd_set_abi_version.help_str = help;
+ /* set the parse strings for the command lists */
TAILQ_FOREACH(t, &commands_list, next) {
- commands_len += strlen(t->command) + 1;
+ commands_len[t->abi_version] += strlen(t->command) + 1;
}
- commands = (char *)calloc(commands_len, sizeof(char));
- if (!commands)
- return -1;
+ for (abi_version = 0; abi_version < TEST_DPDK_ABI_VERSION_MAX;
+ abi_version++) {
+ commands[abi_version] =
+ (char *)calloc(commands_len[abi_version], sizeof(char));
+ if (!commands[abi_version])
+ return -1;
+ }
TAILQ_FOREACH(t, &commands_list, next) {
- strlcat(commands, t->command, commands_len);
+ strlcat(commands[t->abi_version],
+ t->command, commands_len[t->abi_version]);
if (TAILQ_NEXT(t, next) != NULL)
- strlcat(commands, "#", commands_len);
+ strlcat(commands[t->abi_version],
+ "#", commands_len[t->abi_version]);
}
- cmd_autotest_autotest.string_data.str = commands;
+ for (abi_version = 0; abi_version < TEST_DPDK_ABI_VERSION_MAX;
+ abi_version++)
+ cmd_autotest_autotest[abi_version].string_data.str =
+ commands[abi_version];
+
return 0;
}
diff --git a/app/test/test.c b/app/test/test.c
index cd7aaf645..67179d4af 100644
--- a/app/test/test.c
+++ b/app/test/test.c
@@ -307,3 +307,5 @@ unit_test_suite_runner(struct unit_test_suite *suite)
return TEST_SKIPPED;
return TEST_SUCCESS;
}
+
+REGISTER_TEST_ABI_VERSION(default, TEST_DPDK_ABI_VERSION_DEFAULT)
diff --git a/app/test/test.h b/app/test/test.h
index ac0c50616..5ec3728d0 100644
--- a/app/test/test.h
+++ b/app/test/test.h
@@ -162,25 +162,59 @@ int test_set_rxtx_conf(cmdline_fixed_string_t mode);
int test_set_rxtx_anchor(cmdline_fixed_string_t type);
int test_set_rxtx_sc(cmdline_fixed_string_t type);
+#define MAP_ABI_SYMBOL_VERSION(name, abi_version) \
+ __asm(".symver "RTE_STR(name)","RTE_STR(name)"@"RTE_STR(abi_version))
+
+#define TEST_DPDK_ABI_VERSION_DEFAULT 0
+#define TEST_DPDK_ABI_VERSION_V1604 1
+#define TEST_DPDK_ABI_VERSION_V20 2
+#define TEST_DPDK_ABI_VERSION_MAX 3
+
+TAILQ_HEAD(test_abi_version_list, test_abi_version);
+struct test_abi_version {
+ TAILQ_ENTRY(test_abi_version) next;
+ const char *version_name;
+ uint8_t version_id;
+};
+
+void add_abi_version(struct test_abi_version *av);
+
+/* Register a test function with its command string */
+#define REGISTER_TEST_ABI_VERSION(name, id) \
+ static struct test_abi_version test_struct_##name = { \
+ .version_name = RTE_STR(name), \
+ .version_id = id, \
+ }; \
+ RTE_INIT(test_register_##name) \
+ { \
+ add_abi_version(&test_struct_##name); \
+ }
+
typedef int (test_callback)(void);
TAILQ_HEAD(test_commands_list, test_command);
struct test_command {
TAILQ_ENTRY(test_command) next;
const char *command;
test_callback *callback;
+ uint8_t abi_version;
};
-void add_test_command(struct test_command *t);
+void add_test_command(struct test_command *t, uint8_t abi_version);
+
+/* Register a test function with its command string and abi version */
+#define REGISTER_TEST_COMMAND_VERSION(cmd, func, abi_version) \
+ static struct test_command test_struct_##cmd = { \
+ .command = RTE_STR(cmd), \
+ .callback = func, \
+ }; \
+ RTE_INIT(test_register_##cmd) \
+ { \
+ add_test_command(&test_struct_##cmd, abi_version); \
+ }
/* Register a test function with its command string */
+
#define REGISTER_TEST_COMMAND(cmd, func) \
- static struct test_command test_struct_##cmd = { \
- .command = RTE_STR(cmd), \
- .callback = func, \
- }; \
- RTE_INIT(test_register_##cmd) \
- { \
- add_test_command(&test_struct_##cmd); \
- }
+ REGISTER_TEST_COMMAND_VERSION(cmd, func, TEST_DPDK_ABI_VERSION_DEFAULT)
#endif
diff --git a/doc/guides/contributing/versioning.rst b/doc/guides/contributing/versioning.rst
index 3ab2c4346..63ef53ea3 100644
--- a/doc/guides/contributing/versioning.rst
+++ b/doc/guides/contributing/versioning.rst
@@ -221,6 +221,10 @@ The macros exported are:
the linker to bind references to symbol ``b`` to the internal symbol
``b_e``.
+* ``BIND_VERSION_SYMBOL(b, n)``: Creates a symbol version entry instructing
+ the linker to bind references to symbol ``b`` to the external symbol
+ ``b@DPDK_n``
+
* ``MAP_STATIC_SYMBOL(f, p)``: Declare the prototype ``f``, and map it to the
fully qualified function ``p``, so that if a symbol becomes versioned, it
can still be mapped back to the public symbol name.
diff --git a/lib/librte_eal/common/include/rte_compat.h b/lib/librte_eal/common/include/rte_compat.h
index 92ff28faf..be9724f4c 100644
--- a/lib/librte_eal/common/include/rte_compat.h
+++ b/lib/librte_eal/common/include/rte_compat.h
@@ -50,6 +50,13 @@
#define BIND_DEFAULT_SYMBOL(b, e, n) __asm__(".symver " RTE_STR(b) RTE_STR(e) ", " RTE_STR(b) "@@DPDK_" RTE_STR(n))
#define __vsym __attribute__((used))
+/*
+ * BIND_VERSION_SYMBOL
+ * Creates a symbol version entry instructing the linker to bind references to
+ * symbol <b> to the symbol version <b>@DPDK_<n>
+ */
+#define BIND_VERSION_SYMBOL(b, n) __asm__(".symver " RTE_STR(b) ", " RTE_STR(b) "@DPDK_" RTE_STR(n))
+
/*
* MAP_STATIC_SYMBOL
* If a function has been bifurcated into multiple versions, none of which
--
2.17.1
^ permalink raw reply [relevance 14%]
* [dpdk-dev] [PATCH v2 0/2] add abi version testing to app/test
@ 2019-08-22 16:07 9% Ray Kinsella
2019-08-22 16:07 14% ` [dpdk-dev] [PATCH v2 1/2] app/test: add abi version testing functionality Ray Kinsella
` (2 more replies)
0 siblings, 3 replies; 200+ results
From: Ray Kinsella @ 2019-08-22 16:07 UTC (permalink / raw)
To: dev
Cc: mdr, bruce.richardson, vladimir.medvedkin, john.mcnamara,
marko.kovacevic
This patchset adds ABI version testing to the app/test unit test framework,
addressing two issues previously raised during ML conversations on ABI
stability;
1. How do we unit test still supported previous ABI versions?
2. How to we unit test inline functions from still supported previous ABI
versions?
Starting with rte_lpm, I did the following:-
* I reproduced mostly unmodified unit tests for the v2.0 ABI, taken from DPDK
2.2 and 17.02.
* I reproduced the rte_lpm interface header from v2.0, including the inline
functions and remapping symbols to their appropriate versions.
* I added support for multiple abi versions to the app/test unit test framework
to allow users to switch between abi versions (set_abi_version), without
further polluting the already long list of unit tests available in app/test.
The intention here is that in future as developers need to deprecate APIs, the
associated unit tests may move into the ABI version testing mechanism of the
app/test instead of being replaced by the latest set of unit tests as would be
the case today.
v2:
* Added LPM IPv6 test cases for the v2.0 ABI.
* Fixed a number of checkpatch errors, stop short of substantially reworking
the test code from the v2.0 ABI.
* Removed duplicating test cases published in the original v1 patch.
Ray Kinsella (2):
app/test: add abi version testing functionality
app/test: lpm abi version testing
app/test/Makefile | 12 +-
app/test/commands.c | 131 +-
app/test/meson.build | 6 +
app/test/test.c | 2 +
app/test/test.h | 48 +-
app/test/test_lpm.c | 3 +-
app/test/test_lpm6.c | 2 +-
app/test/test_lpm_perf.c | 293 +---
app/test/test_lpm_routes.c | 287 ++++
app/test/test_lpm_routes.h | 25 +
app/test/v2.0/dcompat.h | 30 +
app/test/v2.0/rte_lpm.h | 451 +++++
app/test/v2.0/rte_lpm6.h | 198 +++
app/test/v2.0/test_lpm.c | 1139 +++++++++++++
app/test/v2.0/test_lpm6.c | 1748 ++++++++++++++++++++
app/test/v2.0/test_lpm6_perf.c | 179 ++
app/test/v2.0/test_lpm_perf.c | 212 +++
app/test/v2.0/test_v20.c | 14 +
doc/guides/contributing/versioning.rst | 4 +
lib/librte_eal/common/include/rte_compat.h | 7 +
20 files changed, 4471 insertions(+), 320 deletions(-)
create mode 100644 app/test/test_lpm_routes.c
create mode 100644 app/test/test_lpm_routes.h
create mode 100644 app/test/v2.0/dcompat.h
create mode 100644 app/test/v2.0/rte_lpm.h
create mode 100644 app/test/v2.0/rte_lpm6.h
create mode 100644 app/test/v2.0/test_lpm.c
create mode 100644 app/test/v2.0/test_lpm6.c
create mode 100644 app/test/v2.0/test_lpm6_perf.c
create mode 100644 app/test/v2.0/test_lpm_perf.c
create mode 100644 app/test/v2.0/test_v20.c
--
2.17.1
^ permalink raw reply [relevance 9%]
* Re: [dpdk-dev] [PATCH v2] ethdev: add more protocol support in flow API
2019-08-14 9:08 4% ` Adrien Mazarguil
@ 2019-08-19 11:53 0% ` Zhang, Qi Z
0 siblings, 0 replies; 200+ results
From: Zhang, Qi Z @ 2019-08-19 11:53 UTC (permalink / raw)
To: Adrien Mazarguil, Wang, Ying A; +Cc: Ye, Xiaolong, Yang, Qiming, dev
> -----Original Message-----
> From: Adrien Mazarguil [mailto:adrien.mazarguil@6wind.com]
> Sent: Wednesday, August 14, 2019 5:08 PM
> To: Wang, Ying A <ying.a.wang@intel.com>
> Cc: Zhang, Qi Z <qi.z.zhang@intel.com>; Ye, Xiaolong <xiaolong.ye@intel.com>;
> Yang, Qiming <qiming.yang@intel.com>; dev@dpdk.org
> Subject: Re: [dpdk-dev] [PATCH v2] ethdev: add more protocol support in flow
> API
>
> Hi Wang Ying,
>
> On Wed, Aug 14, 2019 at 11:24:30AM +0800, Wang Ying A wrote:
> > Add new protocol header match support as below
> >
> > RTE_FLOW_ITEM_TYPE_GTP_PSC
> > - matches a GTP PDU extension header (type is 0x85:
> > PDU Session Container)
> > RTE_FLOW_ITEM_TYPE_PPPOES
> > - matches a PPPoE Session header.
> > RTE_FLOW_ITEM_TYPE_PPPOED
> > - matches a PPPoE Discovery stage header.
> >
> > Signed-off-by: Wang Ying A <ying.a.wang@intel.com>
>
> OK, please split this patch, one for GTP and the other for PPPoE to make title
> less vague than "add more protocol support".
>
> For PPPoE, the distinction between session and discovery is not a bad idea but
> since discovery packets typically have a different format (tags in place of
> protocol), I'm not sure they should share a common structure.
>
> How about a single "PPPOE" item without proto_id to cover both session and
> discovery, then later add separate tag items on a needed basis for each
> possible/optional tag (e.g. PPPOE_TAG_SERVICE_NAME)? Likewise proto_id
> would be provided through a separate optional item if relevant
> (PPPOE_PROTO_ID).
PPPoE Discovery had PPPoE Session has different ethertype 0x8863 , 0x8864, so I think they should belong to separate
match item,( image the case we only need to match a PPPoE Session flow but don't care what kind of protocol payload it have )
but I agree they should only share the common part of the header,
then for PPPoED, it can append optional PPPoE Tag item(s) and for PPPoES , it can append optional PPPoE Protocol item(s)
> Such an approach is already used for IPV6 and IPV6_EXT.
>
> Another benefit is that applications could match PPPoE regardless of session or
> discovery when they do not care, while PPPOED/PPPOES make that distinction
> mandatory.
>
> Also by inserting new entries in the middle of the pattern items list, this patch
> breaks ABI. I think it's not on purpose, so please move them to the end (not
> grouping them with existing GTP stuff is fine, ABI compat is more
> important.) This must be reflected in rte_flow.h, rte_flow.c, testpmd and
> documentation.
>
> More nits below.
>
> > ---
> > ---
> > v2: Remove Gerrit Change-Id's.
> > ---
> > app/test-pmd/cmdline_flow.c | 80
> +++++++++++++++++++++++++++++
> > doc/guides/prog_guide/rte_flow.rst | 25 +++++++++
> > doc/guides/testpmd_app_ug/testpmd_funcs.rst | 10 ++++
> > lib/librte_ethdev/rte_flow.c | 3 ++
> > lib/librte_ethdev/rte_flow.h | 71
> +++++++++++++++++++++++++
> > 5 files changed, 189 insertions(+)
> >
> > diff --git a/app/test-pmd/cmdline_flow.c b/app/test-pmd/cmdline_flow.c
> [...]
> > + [ITEM_PPPOES] = {
> > + .name = "pppoes",
> > + .help = "match PPPoE Session header",
>
> Session => session
>
> > + .priv = PRIV_ITEM(PPPOES, sizeof(struct rte_flow_item_pppoe)),
> > + .next = NEXT(item_pppoe),
> > + .call = parse_vc,
> > + },
> > + [ITEM_PPPOED] = {
> > + .name = "pppoed",
> > + .help = "match PPPoE Discovery stage header",
>
> Discovery => discovery
>
> > + .priv = PRIV_ITEM(PPPOED, sizeof(struct rte_flow_item_pppoe)),
> > + .next = NEXT(item_pppoe),
> > + .call = parse_vc,
> > + },
> > + [ITEM_PPPOE_SEID] = {
> > + .name = "seid",
> > + .help = "Session identifier",
>
> Session => session
>
> [...]
> > diff --git a/doc/guides/prog_guide/rte_flow.rst
> > b/doc/guides/prog_guide/rte_flow.rst
> > index 821b524b3..d09c42071 100644
> > --- a/doc/guides/prog_guide/rte_flow.rst
> > +++ b/doc/guides/prog_guide/rte_flow.rst
> > @@ -1055,6 +1055,31 @@ flow rules.
> > - ``teid``: tunnel endpoint identifier.
> > - Default ``mask`` matches teid only.
> >
> > +Item: ``GTP_PSC``
> > +^^^^^^^^^^^^^^^^^^^^^^^
>
> Too many "^^^"'s.
>
> [...]
> > +Item: ``PPPOES``, ``PPPOED``
> > +^^^^^^^^^^^^^^^^^^^^^^^^^^^^
> > +
> > +Matches a PPPOE header.
>
> PPPOE => PPPoE
>
> > +
> > +Note: PPPOES and PPPOED use the same structure. PPPOES and PPPOED
> > +item
>
> item => items (or better, "pattern items")
>
> > +are defined for a user-friendly API when creating PPPOE-Session and
> > +PPPOE-Discovery flow rules.
>
> Super nit: use "PPPoE" when mentioning the protocol itself and "PPPOE" when
> mentioning the pattern item.
>
> > +
> > +- ``v_t_flags``: version (4b), type (4b).
>
> Why "flags"? I don't see any so you could name it "version_type" (same in
> documentation).
>
> > +- ``code``: Message type.
>
> Message => message
>
> > +- ``session_id``: Session identifier.
>
> Session => session
>
> > +- ``length``: Payload length.
>
> Payload => payload
>
> > +- ``proto_id``: PPP Protocol identifier.
>
> Protocol => protocol
>
> > +- Default ``mask`` matches session_id,proto_id.
>
> Missing space between session_id and proto_id.
>
> > +
> > Item: ``ESP``
> > ^^^^^^^^^^^^^
> >
> > diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> > b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> > index 313e0707e..0da36d5f1 100644
> > --- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> > +++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> > @@ -3904,6 +3904,16 @@ This section lists supported pattern items and
> their attributes, if any.
> >
> > - ``teid {unsigned}``: tunnel endpoint identifier.
> >
> > +- ``gtp_psc``: match GTPv1 entension header (type is 0x85).
> > +
> > + - ``pdu_type {unsigned}``: PDU type (0 or 1).
> > + - ``qfi {unsigned}``: QoS flow identifier.
> > +
> > +- ``pppoes``, ``pppoed``: match PPPOE header.
>
> PPPOE => PPPoE
>
> [...]
> > diff --git a/lib/librte_ethdev/rte_flow.h
> > b/lib/librte_ethdev/rte_flow.h index b66bf1495..ad5e46190 100644
> > --- a/lib/librte_ethdev/rte_flow.h
> > +++ b/lib/librte_ethdev/rte_flow.h
> > @@ -328,6 +328,34 @@ enum rte_flow_item_type {
> > */
> > RTE_FLOW_ITEM_TYPE_GTPU,
> >
> > + /**
> > + * Matches a GTP PDU extension header (type is 0x85:
> > + * PDU Session Container).
>
> Session Container => session container
>
> > + *
> > + * Configure flow for GTP packets with extension header type 0x85.
> > + *
> > + * See struct rte_flow_item_gtp_psc.
> > + */
> > + RTE_FLOW_ITEM_TYPE_GTP_PSC,
> > +
> > + /**
> > + * Matches a PPPOE header.
> > + *
> > + * Configure flow for PPPoE Session packets.
>
> Session => session
>
> > + *
> > + * See struct rte_flow_item_pppoe.
> > + */
> > + RTE_FLOW_ITEM_TYPE_PPPOES,
> > +
> > + /**
> > + * Matches a PPPOE header.
> > + *
> > + * Configure flow for PPPoE Discovery stage packets.
>
> Discovery => discovery
>
> > + *
> > + * See struct rte_flow_item_pppoe.
> > + */
> > + RTE_FLOW_ITEM_TYPE_PPPOED,
> > +
> > /**
> > * Matches a ESP header.
> > *
> > @@ -922,6 +950,49 @@ static const struct rte_flow_item_gtp
> > rte_flow_item_gtp_mask = { }; #endif
> >
> > +/**
> > + * RTE_FLOW_ITEM_TYPE_GTP_PSC.
> > + *
> > + * Matches a GTP-extension header
> > + * (type is 0x85: PDU Session Container).
>
> Session Container => session container
>
> (crusade against superfluous caps!)
>
> [...]
> > +/**
> > + * RTE_FLOW_ITEM_TYPE_PPPOE.
> > + *
> > + * Matches a PPPOE header.
> > + */
> > +struct rte_flow_item_pppoe {
> > + /**
> > + * Version (4b), type (4b).
> > + */
> > + uint8_t v_t_flags;
>
> v_t_flags => version_type
>
> > + uint8_t code; /**< Message type. */
> > + rte_be16_t session_id; /**< Session identifier. */
> > + rte_be16_t length; /**< Payload length. */
> > + rte_be16_t proto_id; /**< PPP Protocol identifier. */
>
> As discussed, I suggest dropping proto_id to make this a generic match for
> PPPoE.
>
> > +};
> > +
> > +/** Default mask for RTE_FLOW_ITEM_TYPE_PPPOE. */ #ifndef __cplusplus
> > +static const struct rte_flow_item_pppoe rte_flow_item_pppoe_mask = {
> > + .session_id = RTE_BE16(0xffff),
> > + .proto_id = RTE_BE16(0xffff),
> > +};
>
> I think the default for PPPoE should be an empty mask that simply says "match
> PPPoE" since session_id only becomes known after negotiation and proto_id
> shouldn't be part of this object.
>
> --
> Adrien Mazarguil
> 6WIND
^ permalink raw reply [relevance 0%]
* [dpdk-dev] [PATCH v10 3/5] kni: add app specific mempool create and free routines
@ 2019-08-16 6:12 3% ` vattunuru
0 siblings, 0 replies; 200+ results
From: vattunuru @ 2019-08-16 6:12 UTC (permalink / raw)
To: dev
Cc: thomas, jerinj, olivier.matz, ferruh.yigit, anatoly.burakov,
arybchenko, kirankumark, Vamsi Attunuru
From: Vamsi Attunuru <vattunuru@marvell.com>
When KNI operates in IOVA = VA mode, it requires mbuf memory
to be physically contiguous to ensure KNI kernel module could
translate IOVA addresses properly. Patch adds a KNI specific
mempool create routine to populate the KNI packet mbuf pool
with memory objects that are being on a page.
KNI applications need to use this mempool create & free routines
so that mbuf related requirements in IOVA = VA mode are handled
inside those routines based on the enabled mode.
Updated the release notes with these new routine details.
Signed-off-by: Vamsi Attunuru <vattunuru@marvell.com>
---
doc/guides/rel_notes/release_19_11.rst | 5 +++
examples/kni/main.c | 5 ++-
lib/librte_kni/Makefile | 1 +
lib/librte_kni/meson.build | 1 +
lib/librte_kni/rte_kni.c | 60 ++++++++++++++++++++++++++++++++++
lib/librte_kni/rte_kni.h | 48 +++++++++++++++++++++++++++
lib/librte_kni/rte_kni_version.map | 2 ++
7 files changed, 121 insertions(+), 1 deletion(-)
diff --git a/doc/guides/rel_notes/release_19_11.rst b/doc/guides/rel_notes/release_19_11.rst
index 8490d89..8813a10 100644
--- a/doc/guides/rel_notes/release_19_11.rst
+++ b/doc/guides/rel_notes/release_19_11.rst
@@ -85,6 +85,11 @@ API Changes
Also, make sure to start the actual text at the margin.
=========================================================
+* kni: ``rte_kni_pktmbuf_pool_create`` ``rte_kni_pktmbuf_pool_free`` functions
+ were introduced for KNI applications for creating & freeing packet pool.
+ Since IOVA=VA mode was added in KNI, packet pool's mbuf memory should be
+ physically contiguous for the KNI kernel module to work in IOVA=VA mode,
+ this requirement was taken care in the kni packet pool creation fucntions.
ABI Changes
-----------
diff --git a/examples/kni/main.c b/examples/kni/main.c
index 4710d71..fdfeed2 100644
--- a/examples/kni/main.c
+++ b/examples/kni/main.c
@@ -975,7 +975,7 @@ main(int argc, char** argv)
rte_exit(EXIT_FAILURE, "Could not parse input parameters\n");
/* Create the mbuf pool */
- pktmbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", NB_MBUF,
+ pktmbuf_pool = rte_kni_pktmbuf_pool_create("mbuf_pool", NB_MBUF,
MEMPOOL_CACHE_SZ, 0, MBUF_DATA_SZ, rte_socket_id());
if (pktmbuf_pool == NULL) {
rte_exit(EXIT_FAILURE, "Could not initialise mbuf pool\n");
@@ -1043,6 +1043,9 @@ main(int argc, char** argv)
continue;
kni_free_kni(port);
}
+
+ rte_kni_pktmbuf_pool_free(pktmbuf_pool);
+
for (i = 0; i < RTE_MAX_ETHPORTS; i++)
if (kni_port_params_array[i]) {
rte_free(kni_port_params_array[i]);
diff --git a/lib/librte_kni/Makefile b/lib/librte_kni/Makefile
index ab15d10..5e3dd01 100644
--- a/lib/librte_kni/Makefile
+++ b/lib/librte_kni/Makefile
@@ -6,6 +6,7 @@ include $(RTE_SDK)/mk/rte.vars.mk
# library name
LIB = librte_kni.a
+CFLAGS += -DALLOW_EXPERIMENTAL_API
CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3 -fno-strict-aliasing
CFLAGS += -I$(RTE_SDK)/drivers/bus/pci
LDLIBS += -lrte_eal -lrte_mempool -lrte_mbuf -lrte_ethdev
diff --git a/lib/librte_kni/meson.build b/lib/librte_kni/meson.build
index fd46f87..e357445 100644
--- a/lib/librte_kni/meson.build
+++ b/lib/librte_kni/meson.build
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: BSD-3-Clause
# Copyright(c) 2017 Intel Corporation
+allow_experimental_apis = true
if not is_linux or not dpdk_conf.get('RTE_ARCH_64')
build = false
reason = 'only supported on 64-bit linux'
diff --git a/lib/librte_kni/rte_kni.c b/lib/librte_kni/rte_kni.c
index 2aaaeaa..15dda45 100644
--- a/lib/librte_kni/rte_kni.c
+++ b/lib/librte_kni/rte_kni.c
@@ -22,6 +22,7 @@
#include <rte_tailq.h>
#include <rte_rwlock.h>
#include <rte_eal_memconfig.h>
+#include <rte_mbuf_pool_ops.h>
#include <rte_kni_common.h>
#include "rte_kni_fifo.h"
@@ -681,6 +682,65 @@ kni_allocate_mbufs(struct rte_kni *kni)
}
}
+struct rte_mempool *
+rte_kni_pktmbuf_pool_create(const char *name, unsigned int n,
+ unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size,
+ int socket_id)
+{
+ struct rte_pktmbuf_pool_private mbp_priv;
+ const char *mp_ops_name;
+ struct rte_mempool *mp;
+ unsigned int elt_size;
+ int ret;
+
+ if (RTE_ALIGN(priv_size, RTE_MBUF_PRIV_ALIGN) != priv_size) {
+ RTE_LOG(ERR, MBUF, "mbuf priv_size=%u is not aligned\n",
+ priv_size);
+ rte_errno = EINVAL;
+ return NULL;
+ }
+ elt_size = sizeof(struct rte_mbuf) + (unsigned int)priv_size +
+ (unsigned int)data_room_size;
+ mbp_priv.mbuf_data_room_size = data_room_size;
+ mbp_priv.mbuf_priv_size = priv_size;
+
+ mp = rte_mempool_create_empty(name, n, elt_size, cache_size,
+ sizeof(struct rte_pktmbuf_pool_private), socket_id, 0);
+ if (mp == NULL)
+ return NULL;
+
+ mp_ops_name = rte_mbuf_best_mempool_ops();
+ ret = rte_mempool_set_ops_byname(mp, mp_ops_name, NULL);
+ if (ret != 0) {
+ RTE_LOG(ERR, MBUF, "error setting mempool handler\n");
+ rte_mempool_free(mp);
+ rte_errno = -ret;
+ return NULL;
+ }
+ rte_pktmbuf_pool_init(mp, &mbp_priv);
+
+ if (rte_eal_iova_mode() == RTE_IOVA_VA)
+ ret = rte_mempool_populate_from_pg_sz_chunks(mp);
+ else
+ ret = rte_mempool_populate_default(mp);
+
+ if (ret < 0) {
+ rte_mempool_free(mp);
+ rte_errno = -ret;
+ return NULL;
+ }
+
+ rte_mempool_obj_iter(mp, rte_pktmbuf_init, NULL);
+
+ return mp;
+}
+
+void
+rte_kni_pktmbuf_pool_free(struct rte_mempool *mp)
+{
+ rte_mempool_free(mp);
+}
+
struct rte_kni *
rte_kni_get(const char *name)
{
diff --git a/lib/librte_kni/rte_kni.h b/lib/librte_kni/rte_kni.h
index 5699a64..99d263d 100644
--- a/lib/librte_kni/rte_kni.h
+++ b/lib/librte_kni/rte_kni.h
@@ -184,6 +184,54 @@ unsigned rte_kni_tx_burst(struct rte_kni *kni, struct rte_mbuf **mbufs,
unsigned num);
/**
+ * Create a kni packet mbuf pool.
+ *
+ * This function creates and initializes a packet mbuf pool for KNI applications
+ * It calls the required mempool populate routine based on the IOVA mode.
+ *
+ * @param name
+ * The name of the mbuf pool.
+ * @param n
+ * The number of elements in the mbuf pool. The optimum size (in terms
+ * of memory usage) for a mempool is when n is a power of two minus one:
+ * n = (2^q - 1).
+ * @param cache_size
+ * Size of the per-core object cache. See rte_mempool_create() for
+ * details.
+ * @param priv_size
+ * Size of application private are between the rte_mbuf structure
+ * and the data buffer. This value must be aligned to RTE_MBUF_PRIV_ALIGN.
+ * @param data_room_size
+ * Size of data buffer in each mbuf, including RTE_PKTMBUF_HEADROOM.
+ * @param socket_id
+ * The socket identifier where the memory should be allocated. The
+ * value can be *SOCKET_ID_ANY* if there is no NUMA constraint for the
+ * reserved zone.
+ * @return
+ * The pointer to the new allocated mempool, on success. NULL on error
+ * with rte_errno set appropriately. Possible rte_errno values include:
+ * - E_RTE_NO_CONFIG - function could not get pointer to rte_config structure
+ * - E_RTE_SECONDARY - function was called from a secondary process instance
+ * - EINVAL - cache size provided is too large, or priv_size is not aligned.
+ * - ENOSPC - the maximum number of memzones has already been allocated
+ * - EEXIST - a memzone with the same name already exists
+ * - ENOMEM - no appropriate memory area found in which to create memzone
+ */
+__rte_experimental
+struct rte_mempool *rte_kni_pktmbuf_pool_create(const char *name,
+ unsigned int n, unsigned int cache_size, uint16_t priv_size,
+ uint16_t data_room_size, int socket_id);
+
+/**
+ * Free the given packet mempool.
+ *
+ * @param mp
+ * The mempool pointer.
+ */
+__rte_experimental
+void rte_kni_pktmbuf_pool_free(struct rte_mempool *mp);
+
+/**
* Get the KNI context of its name.
*
* @param name
diff --git a/lib/librte_kni/rte_kni_version.map b/lib/librte_kni/rte_kni_version.map
index c877dc6..aba9728 100644
--- a/lib/librte_kni/rte_kni_version.map
+++ b/lib/librte_kni/rte_kni_version.map
@@ -20,4 +20,6 @@ EXPERIMENTAL {
global:
rte_kni_update_link;
+ rte_kni_pktmbuf_pool_create;
+ rte_kni_pktmbuf_pool_free;
};
--
2.8.4
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] [RFC] ethdev: configure SR-IOV VF from host
@ 2019-08-16 5:16 0% Jerin Jacob Kollanukkaran
0 siblings, 0 replies; 200+ results
From: Jerin Jacob Kollanukkaran @ 2019-08-16 5:16 UTC (permalink / raw)
To: Thomas Monjalon
Cc: Ferruh Yigit, Andrew Rybchenko, dev, Bernard Iremonger,
Shahaf Shuler, E. Scott Daniels, Wenzhuo Lu, Alex Zelezniak,
Ajit Khaparde, Declan Doherty
> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Thursday, August 15, 2019 11:29 PM
> To: Jerin Jacob Kollanukkaran <jerinj@marvell.com>
> Cc: Ferruh Yigit <ferruh.yigit@intel.com>; Andrew Rybchenko
> <arybchenko@solarflare.com>; dev@dpdk.org; Bernard Iremonger
> <bernard.iremonger@intel.com>; Shahaf Shuler <shahafs@mellanox.com>;
> E. Scott Daniels <daniels@research.att.com>; Wenzhuo Lu
> <wenzhuo.lu@intel.com>; Alex Zelezniak <alexz@att.com>; Ajit Khaparde
> <ajit.khaparde@broadcom.com>; Declan Doherty
> <declan.doherty@intel.com>
> Subject: [EXT] Re: [dpdk-dev] [RFC] ethdev: configure SR-IOV VF from host
>
> External Email
>
> ----------------------------------------------------------------------
> 15/08/2019 17:34, Jerin Jacob Kollanukkaran:
> > From: Thomas Monjalon
> > >
> > > In a virtual environment, the network controller may have to
> > > configure some SR-IOV VF parameters for security reasons.
> > >
> > > When the PF (host port) is drived by DPDK (OVS-DPDK case), we face
> > > two different cases:
> > > - driver is bifurcated (Mellanox case),
> > > so the VF can be configured via the kernel.
> > > - driver is on top of UIO or VFIO, so DPDK API is required.
> > >
> > > This RFC proposes to use generic DPDK API for VF configuration.
> > > The impacted functions are (can be extended):
> > >
> > > - rte_eth_dev_is_valid_port
> > > - rte_eth_promiscuous_enable
> > > - rte_eth_promiscuous_disable
> > > - rte_eth_promiscuous_get
> > > - rte_eth_allmulticast_enable
> > > - rte_eth_allmulticast_disable
> > > - rte_eth_allmulticast_get
> > > - rte_eth_dev_set_mc_addr_list
> > > - rte_eth_dev_default_mac_addr_set
> > > - rte_eth_macaddr_get
> > > - rte_eth_dev_mac_addr_add
> > > - rte_eth_dev_mac_addr_remove
> > > - rte_eth_dev_vlan_filter
> > > - rte_eth_dev_get_mtu
> > > - rte_eth_dev_set_mtu
> > >
> > > In order to target these functions to a VF (which has no port id in
> > > the host), the higher bit of port id is reserved:
> > >
> > > #define RTE_ETH_VF_PORT_FLAG (1 << 15)
> >
> > Instead of changing the port number behavior, How about adding a bit
> > field/ I think, There is no ABI breakage as the parent type of bit
> > field is uint8_t and there is still more room.
> >
> > --- a/lib/librte_ethdev/rte_ethdev_core.h
> > +++ b/lib/librte_ethdev/rte_ethdev_core.h
> > @@ -621,6 +621,7 @@ struct rte_eth_dev_data {
> > all_multicast : 1, /**< RX all multicast mode ON(1) / OFF(0). */
> > dev_started : 1, /**< Device state: STARTED(1) / STOPPED(0). */
> > lro : 1; /**< RX LRO is ON(1) / OFF(0) */
> > + vf : 1; /**< SR-IOV VF device */
> > uint8_t rx_queue_state[RTE_MAX_QUEUES_PER_PORT];
> > /**< Queues state: STARTED(1) / STOPPED(0). */
> > uint8_t tx_queue_state[RTE_MAX_QUEUES_PER_PORT];
>
> Sorry I don't understand how it can help.
> We need to specify which VF we want to configure.
>
> My proposal is to use the representor port, which is connected to a VF.
I got confused with non-representor case. Yes, My comment is not valid for
port representor case.
> We distinguish the representor and the VF with a flag in the port id
> parameter passed to the functions.
>
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [RFC] ethdev: configure SR-IOV VF from host
2019-08-15 15:34 3% ` Jerin Jacob Kollanukkaran
@ 2019-08-15 17:59 0% ` Thomas Monjalon
0 siblings, 0 replies; 200+ results
From: Thomas Monjalon @ 2019-08-15 17:59 UTC (permalink / raw)
To: Jerin Jacob Kollanukkaran
Cc: Ferruh Yigit, Andrew Rybchenko, dev, Bernard Iremonger,
Shahaf Shuler, E. Scott Daniels, Wenzhuo Lu, Alex Zelezniak,
Ajit Khaparde, Declan Doherty
15/08/2019 17:34, Jerin Jacob Kollanukkaran:
> From: Thomas Monjalon
> >
> > In a virtual environment, the network controller may have to configure some
> > SR-IOV VF parameters for security reasons.
> >
> > When the PF (host port) is drived by DPDK (OVS-DPDK case), we face two
> > different cases:
> > - driver is bifurcated (Mellanox case),
> > so the VF can be configured via the kernel.
> > - driver is on top of UIO or VFIO, so DPDK API is required.
> >
> > This RFC proposes to use generic DPDK API for VF configuration.
> > The impacted functions are (can be extended):
> >
> > - rte_eth_dev_is_valid_port
> > - rte_eth_promiscuous_enable
> > - rte_eth_promiscuous_disable
> > - rte_eth_promiscuous_get
> > - rte_eth_allmulticast_enable
> > - rte_eth_allmulticast_disable
> > - rte_eth_allmulticast_get
> > - rte_eth_dev_set_mc_addr_list
> > - rte_eth_dev_default_mac_addr_set
> > - rte_eth_macaddr_get
> > - rte_eth_dev_mac_addr_add
> > - rte_eth_dev_mac_addr_remove
> > - rte_eth_dev_vlan_filter
> > - rte_eth_dev_get_mtu
> > - rte_eth_dev_set_mtu
> >
> > In order to target these functions to a VF (which has no port id in the host),
> > the higher bit of port id is reserved:
> >
> > #define RTE_ETH_VF_PORT_FLAG (1 << 15)
>
> Instead of changing the port number behavior, How about adding a bit field/
> I think, There is no ABI breakage as the parent type of bit field is uint8_t and there
> is still more room.
>
> --- a/lib/librte_ethdev/rte_ethdev_core.h
> +++ b/lib/librte_ethdev/rte_ethdev_core.h
> @@ -621,6 +621,7 @@ struct rte_eth_dev_data {
> all_multicast : 1, /**< RX all multicast mode ON(1) / OFF(0). */
> dev_started : 1, /**< Device state: STARTED(1) / STOPPED(0). */
> lro : 1; /**< RX LRO is ON(1) / OFF(0) */
> + vf : 1; /**< SR-IOV VF device */
> uint8_t rx_queue_state[RTE_MAX_QUEUES_PER_PORT];
> /**< Queues state: STARTED(1) / STOPPED(0). */
> uint8_t tx_queue_state[RTE_MAX_QUEUES_PER_PORT];
Sorry I don't understand how it can help.
We need to specify which VF we want to configure.
My proposal is to use the representor port,
which is connected to a VF.
We distinguish the representor and the VF with a flag in the port id
parameter passed to the functions.
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [RFC] ethdev: configure SR-IOV VF from host
@ 2019-08-15 15:34 3% ` Jerin Jacob Kollanukkaran
2019-08-15 17:59 0% ` Thomas Monjalon
0 siblings, 1 reply; 200+ results
From: Jerin Jacob Kollanukkaran @ 2019-08-15 15:34 UTC (permalink / raw)
To: Thomas Monjalon, Ferruh Yigit, Andrew Rybchenko
Cc: dev, Bernard Iremonger, Shahaf Shuler, E. Scott Daniels,
Wenzhuo Lu, Alex Zelezniak, Ajit Khaparde, Declan Doherty
> -----Original Message-----
> From: dev <dev-bounces@dpdk.org> On Behalf Of Thomas Monjalon
> Sent: Thursday, August 15, 2019 8:36 PM
> To: Ferruh Yigit <ferruh.yigit@intel.com>; Andrew Rybchenko
> <arybchenko@solarflare.com>
> Cc: dev@dpdk.org; Bernard Iremonger <bernard.iremonger@intel.com>;
> Shahaf Shuler <shahafs@mellanox.com>; E. Scott Daniels
> <daniels@research.att.com>; Wenzhuo Lu <wenzhuo.lu@intel.com>; Alex
> Zelezniak <alexz@att.com>; Ajit Khaparde <ajit.khaparde@broadcom.com>;
> Declan Doherty <declan.doherty@intel.com>
> Subject: [dpdk-dev] [RFC] ethdev: configure SR-IOV VF from host
>
> In a virtual environment, the network controller may have to configure some
> SR-IOV VF parameters for security reasons.
>
> When the PF (host port) is drived by DPDK (OVS-DPDK case), we face two
> different cases:
> - driver is bifurcated (Mellanox case),
> so the VF can be configured via the kernel.
> - driver is on top of UIO or VFIO, so DPDK API is required.
>
> This RFC proposes to use generic DPDK API for VF configuration.
> The impacted functions are (can be extended):
>
> - rte_eth_dev_is_valid_port
> - rte_eth_promiscuous_enable
> - rte_eth_promiscuous_disable
> - rte_eth_promiscuous_get
> - rte_eth_allmulticast_enable
> - rte_eth_allmulticast_disable
> - rte_eth_allmulticast_get
> - rte_eth_dev_set_mc_addr_list
> - rte_eth_dev_default_mac_addr_set
> - rte_eth_macaddr_get
> - rte_eth_dev_mac_addr_add
> - rte_eth_dev_mac_addr_remove
> - rte_eth_dev_vlan_filter
> - rte_eth_dev_get_mtu
> - rte_eth_dev_set_mtu
>
> In order to target these functions to a VF (which has no port id in the host),
> the higher bit of port id is reserved:
>
> #define RTE_ETH_VF_PORT_FLAG (1 << 15)
Instead of changing the port number behavior, How about adding a bit field/
I think, There is no ABI breakage as the parent type of bit field is uint8_t and there
is still more room.
diff --git a/lib/librte_ethdev/rte_ethdev_core.h b/lib/librte_ethdev/rte_ethdev_core.h
index 2922d5b7c..19a60cab2 100644
--- a/lib/librte_ethdev/rte_ethdev_core.h
+++ b/lib/librte_ethdev/rte_ethdev_core.h
@@ -621,6 +621,7 @@ struct rte_eth_dev_data {
all_multicast : 1, /**< RX all multicast mode ON(1) / OFF(0). */
dev_started : 1, /**< Device state: STARTED(1) / STOPPED(0). */
lro : 1; /**< RX LRO is ON(1) / OFF(0) */
+ vf : 1; /**< SR-IOV VF device */
uint8_t rx_queue_state[RTE_MAX_QUEUES_PER_PORT];
/**< Queues state: STARTED(1) / STOPPED(0). */
uint8_t tx_queue_state[RTE_MAX_QUEUES_PER_PORT];
>
> This bit can be combined only with the port id of a representor.
> The meaning is to target the VF connected with the representor port, instead
> of the representor port itself.
>
> If a function is not expected to support VF configuration, it will return -
> EINVAL, i.e. there is no code change.
> If an API function (listed above) can support VF configuration, but the PMD
> does not support it, then -ENOTSUP must be returned.
>
> As an example, this is the change required in rte_eth_dev_is_valid_port:
>
> int
> rte_eth_dev_is_valid_port(uint16_t port_id) {
> + uint32_t dev_flags;
> + uint16_t vf_flag;
> +
> + vf_flag = port_id & RTE_ETH_VF_PORT_FLAG;
> + port_id &= RTE_ETH_VF_PORT_FLAG - 1; /* remove VF flag */
> +
> if (port_id >= RTE_MAX_ETHPORTS ||
> (rte_eth_devices[port_id].state == RTE_ETH_DEV_UNUSED))
> return 0;
> - else
> - return 1;
> +
> + dev_flags = rte_eth_dev_shared_data->data[port_id].dev_flags;
> + if (vf_flag != 0 && (dev_flags & RTE_ETH_DEV_REPRESENTOR) == 0)
> + return 0; /* VF flag has no meaning if not a representor
> + */
> +
> + return 1;
> }
>
>
^ permalink raw reply [relevance 3%]
* [dpdk-dev] [PATCH v3 1/4] doc: separate versioning.rst into version and policy
2019-08-15 10:23 11% [dpdk-dev] [PATCH v3 0/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
@ 2019-08-15 10:23 13% ` Ray Kinsella
2019-08-15 10:23 31% ` [dpdk-dev] [PATCH v3 2/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
` (2 subsequent siblings)
3 siblings, 0 replies; 200+ results
From: Ray Kinsella @ 2019-08-15 10:23 UTC (permalink / raw)
To: dev
Cc: mdr, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, nhorman,
maxime.coquelin, john.mcnamara, marko.kovacevic, hemant.agrawal
Separate versioning.rst into abi versioning and abi policy guidance, in
preparation for adding more detail to the abi policy.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
doc/guides/contributing/abi_policy.rst | 169 +++++++++
doc/guides/contributing/abi_versioning.rst | 427 +++++++++++++++++++++
doc/guides/contributing/index.rst | 3 +-
doc/guides/contributing/versioning.rst | 591 -----------------------------
4 files changed, 598 insertions(+), 592 deletions(-)
create mode 100644 doc/guides/contributing/abi_policy.rst
create mode 100644 doc/guides/contributing/abi_versioning.rst
delete mode 100644 doc/guides/contributing/versioning.rst
diff --git a/doc/guides/contributing/abi_policy.rst b/doc/guides/contributing/abi_policy.rst
new file mode 100644
index 0000000..55bacb4
--- /dev/null
+++ b/doc/guides/contributing/abi_policy.rst
@@ -0,0 +1,169 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+ Copyright 2018 The DPDK contributors
+
+.. abi_api_policy:
+
+DPDK ABI/API policy
+===================
+
+Description
+-----------
+
+This document details some methods for handling ABI management in the DPDK.
+
+General Guidelines
+------------------
+
+#. Whenever possible, ABI should be preserved
+#. ABI/API may be changed with a deprecation process
+#. The modification of symbols can generally be managed with versioning
+#. Libraries or APIs marked in ``experimental`` state may change without constraint
+#. New APIs will be marked as ``experimental`` for at least one release to allow
+ any issues found by users of the new API to be fixed quickly
+#. The addition of symbols is generally not problematic
+#. The removal of symbols generally is an ABI break and requires bumping of the
+ LIBABIVER macro
+#. Updates to the minimum hardware requirements, which drop support for hardware which
+ was previously supported, should be treated as an ABI change.
+
+What is an ABI
+~~~~~~~~~~~~~~
+
+An ABI (Application Binary Interface) is the set of runtime interfaces exposed
+by a library. It is similar to an API (Application Programming Interface) but
+is the result of compilation. It is also effectively cloned when applications
+link to dynamic libraries. That is to say when an application is compiled to
+link against dynamic libraries, it is assumed that the ABI remains constant
+between the time the application is compiled/linked, and the time that it runs.
+Therefore, in the case of dynamic linking, it is critical that an ABI is
+preserved, or (when modified), done in such a way that the application is unable
+to behave improperly or in an unexpected fashion.
+
+
+ABI/API Deprecation
+-------------------
+
+The DPDK ABI policy
+~~~~~~~~~~~~~~~~~~~
+
+ABI versions are set at the time of major release labeling, and the ABI may
+change multiple times, without warning, between the last release label and the
+HEAD label of the git tree.
+
+ABI versions, once released, are available until such time as their
+deprecation has been noted in the Release Notes for at least one major release
+cycle. For example consider the case where the ABI for DPDK 2.0 has been
+shipped and then a decision is made to modify it during the development of
+DPDK 2.1. The decision will be recorded in the Release Notes for the DPDK 2.1
+release and the modification will be made available in the DPDK 2.2 release.
+
+ABI versions may be deprecated in whole or in part as needed by a given
+update.
+
+Some ABI changes may be too significant to reasonably maintain multiple
+versions. In those cases ABI's may be updated without backward compatibility
+being provided. The requirements for doing so are:
+
+#. At least 3 acknowledgments of the need to do so must be made on the
+ dpdk.org mailing list.
+
+ - The acknowledgment of the maintainer of the component is mandatory, or if
+ no maintainer is available for the component, the tree/sub-tree maintainer
+ for that component must acknowledge the ABI change instead.
+
+ - It is also recommended that acknowledgments from different "areas of
+ interest" be sought for each deprecation, for example: from NIC vendors,
+ CPU vendors, end-users, etc.
+
+#. The changes (including an alternative map file) can be included with
+ deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option,
+ to provide more details about oncoming changes.
+ ``RTE_NEXT_ABI`` wrapper will be removed when it become the default ABI.
+ More preferred way to provide this information is sending the feature
+ as a separate patch and reference it in deprecation notice.
+
+#. A full deprecation cycle, as explained above, must be made to offer
+ downstream consumers sufficient warning of the change.
+
+Note that the above process for ABI deprecation should not be undertaken
+lightly. ABI stability is extremely important for downstream consumers of the
+DPDK, especially when distributed in shared object form. Every effort should
+be made to preserve the ABI whenever possible. The ABI should only be changed
+for significant reasons, such as performance enhancements. ABI breakage due to
+changes such as reorganizing public structure fields for aesthetic or
+readability purposes should be avoided.
+
+.. note::
+
+ Updates to the minimum hardware requirements, which drop support for hardware
+ which was previously supported, should be treated as an ABI change, and
+ follow the relevant deprecation policy procedures as above: 3 acks and
+ announcement at least one release in advance.
+
+Examples of Deprecation Notices
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The following are some examples of ABI deprecation notices which would be
+added to the Release Notes:
+
+* The Macro ``#RTE_FOO`` is deprecated and will be removed with version 2.0,
+ to be replaced with the inline function ``rte_foo()``.
+
+* The function ``rte_mbuf_grok()`` has been updated to include a new parameter
+ in version 2.0. Backwards compatibility will be maintained for this function
+ until the release of version 2.1
+
+* The members of ``struct rte_foo`` have been reorganized in release 2.0 for
+ performance reasons. Existing binary applications will have backwards
+ compatibility in release 2.0, while newly built binaries will need to
+ reference the new structure variant ``struct rte_foo2``. Compatibility will
+ be removed in release 2.2, and all applications will require updating and
+ rebuilding to the new structure at that time, which will be renamed to the
+ original ``struct rte_foo``.
+
+* Significant ABI changes are planned for the ``librte_dostuff`` library. The
+ upcoming release 2.0 will not contain these changes, but release 2.1 will,
+ and no backwards compatibility is planned due to the extensive nature of
+ these changes. Binaries using this library built prior to version 2.1 will
+ require updating and recompilation.
+
+New API replacing previous one
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If a new API proposed functionally replaces an existing one, when the new API
+becomes non-experimental then the old one is marked with ``__rte_deprecated``.
+Deprecated APIs are removed completely just after the next LTS.
+
+Reminder that old API should follow deprecation process to be removed.
+
+
+Experimental APIs
+-----------------
+
+APIs marked as ``experimental`` are not considered part of the ABI and may
+change without warning at any time. Since changes to APIs are most likely
+immediately after their introduction, as users begin to take advantage of
+those new APIs and start finding issues with them, new DPDK APIs will be
+automatically marked as ``experimental`` to allow for a period of stabilization
+before they become part of a tracked ABI.
+
+Note that marking an API as experimental is a multi step process.
+To mark an API as experimental, the symbols which are desired to be exported
+must be placed in an EXPERIMENTAL version block in the corresponding libraries'
+version map script.
+Secondly, the corresponding prototypes of those exported functions (in the
+development header files), must be marked with the ``__rte_experimental`` tag
+(see ``rte_compat.h``).
+The DPDK build makefiles perform a check to ensure that the map file and the
+C code reflect the same list of symbols.
+This check can be circumvented by defining ``ALLOW_EXPERIMENTAL_API``
+during compilation in the corresponding library Makefile.
+
+In addition to tagging the code with ``__rte_experimental``,
+the doxygen markup must also contain the EXPERIMENTAL string,
+and the MAINTAINERS file should note the EXPERIMENTAL libraries.
+
+For removing the experimental tag associated with an API, deprecation notice
+is not required. Though, an API should remain in experimental state for at least
+one release. Thereafter, normal process of posting patch for review to mailing
+list can be followed.
diff --git a/doc/guides/contributing/abi_versioning.rst b/doc/guides/contributing/abi_versioning.rst
new file mode 100644
index 0000000..53e6ac0
--- /dev/null
+++ b/doc/guides/contributing/abi_versioning.rst
@@ -0,0 +1,427 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+ Copyright 2018 The DPDK contributors
+
+.. library_versioning:
+
+Library versioning
+------------------
+
+Downstreams might want to provide different DPDK releases at the same time to
+support multiple consumers of DPDK linked against older and newer sonames.
+
+Also due to the interdependencies that DPDK libraries can have applications
+might end up with an executable space in which multiple versions of a library
+are mapped by ld.so.
+
+Think of LibA that got an ABI bump and LibB that did not get an ABI bump but is
+depending on LibA.
+
+.. note::
+
+ Application
+ \-> LibA.old
+ \-> LibB.new -> LibA.new
+
+That is a conflict which can be avoided by setting ``CONFIG_RTE_MAJOR_ABI``.
+If set, the value of ``CONFIG_RTE_MAJOR_ABI`` overwrites all - otherwise per
+library - versions defined in the libraries ``LIBABIVER``.
+An example might be ``CONFIG_RTE_MAJOR_ABI=16.11`` which will make all libraries
+``librte<?>.so.16.11`` instead of ``librte<?>.so.<LIBABIVER>``.
+
+
+ABI versioning
+--------------
+
+Versioning Macros
+~~~~~~~~~~~~~~~~~
+
+When a symbol is exported from a library to provide an API, it also provides a
+calling convention (ABI) that is embodied in its name, return type and
+arguments. Occasionally that function may need to change to accommodate new
+functionality or behavior. When that occurs, it is desirable to allow for
+backward compatibility for a time with older binaries that are dynamically
+linked to the DPDK.
+
+To support backward compatibility the ``rte_compat.h``
+header file provides macros to use when updating exported functions. These
+macros are used in conjunction with the ``rte_<library>_version.map`` file for
+a given library to allow multiple versions of a symbol to exist in a shared
+library so that older binaries need not be immediately recompiled.
+
+The macros exported are:
+
+* ``VERSION_SYMBOL(b, e, n)``: Creates a symbol version table entry binding
+ versioned symbol ``b@DPDK_n`` to the internal function ``b_e``.
+
+* ``BIND_DEFAULT_SYMBOL(b, e, n)``: Creates a symbol version entry instructing
+ the linker to bind references to symbol ``b`` to the internal symbol
+ ``b_e``.
+
+* ``MAP_STATIC_SYMBOL(f, p)``: Declare the prototype ``f``, and map it to the
+ fully qualified function ``p``, so that if a symbol becomes versioned, it
+ can still be mapped back to the public symbol name.
+
+Examples of ABI Macro use
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Updating a public API
+_____________________
+
+Assume we have a function as follows
+
+.. code-block:: c
+
+ /*
+ * Create an acl context object for apps to
+ * manipulate
+ */
+ struct rte_acl_ctx *
+ rte_acl_create(const struct rte_acl_param *param)
+ {
+ ...
+ }
+
+
+Assume that struct rte_acl_ctx is a private structure, and that a developer
+wishes to enhance the acl api so that a debugging flag can be enabled on a
+per-context basis. This requires an addition to the structure (which, being
+private, is safe), but it also requires modifying the code as follows
+
+.. code-block:: c
+
+ /*
+ * Create an acl context object for apps to
+ * manipulate
+ */
+ struct rte_acl_ctx *
+ rte_acl_create(const struct rte_acl_param *param, int debug)
+ {
+ ...
+ }
+
+
+Note also that, being a public function, the header file prototype must also be
+changed, as must all the call sites, to reflect the new ABI footprint. We will
+maintain previous ABI versions that are accessible only to previously compiled
+binaries
+
+The addition of a parameter to the function is ABI breaking as the function is
+public, and existing application may use it in its current form. However, the
+compatibility macros in DPDK allow a developer to use symbol versioning so that
+multiple functions can be mapped to the same public symbol based on when an
+application was linked to it. To see how this is done, we start with the
+requisite libraries version map file. Initially the version map file for the
+acl library looks like this
+
+.. code-block:: none
+
+ DPDK_2.0 {
+ global:
+
+ rte_acl_add_rules;
+ rte_acl_build;
+ rte_acl_classify;
+ rte_acl_classify_alg;
+ rte_acl_classify_scalar;
+ rte_acl_create;
+ rte_acl_dump;
+ rte_acl_find_existing;
+ rte_acl_free;
+ rte_acl_ipv4vlan_add_rules;
+ rte_acl_ipv4vlan_build;
+ rte_acl_list_dump;
+ rte_acl_reset;
+ rte_acl_reset_rules;
+ rte_acl_set_ctx_classify;
+
+ local: *;
+ };
+
+This file needs to be modified as follows
+
+.. code-block:: none
+
+ DPDK_2.0 {
+ global:
+
+ rte_acl_add_rules;
+ rte_acl_build;
+ rte_acl_classify;
+ rte_acl_classify_alg;
+ rte_acl_classify_scalar;
+ rte_acl_create;
+ rte_acl_dump;
+ rte_acl_find_existing;
+ rte_acl_free;
+ rte_acl_ipv4vlan_add_rules;
+ rte_acl_ipv4vlan_build;
+ rte_acl_list_dump;
+ rte_acl_reset;
+ rte_acl_reset_rules;
+ rte_acl_set_ctx_classify;
+
+ local: *;
+ };
+
+ DPDK_2.1 {
+ global:
+ rte_acl_create;
+
+ } DPDK_2.0;
+
+The addition of the new block tells the linker that a new version node is
+available (DPDK_2.1), which contains the symbol rte_acl_create, and inherits the
+symbols from the DPDK_2.0 node. This list is directly translated into a list of
+exported symbols when DPDK is compiled as a shared library
+
+Next, we need to specify in the code which function map to the rte_acl_create
+symbol at which versions. First, at the site of the initial symbol definition,
+we need to update the function so that it is uniquely named, and not in conflict
+with the public symbol name
+
+.. code-block:: c
+
+ struct rte_acl_ctx *
+ -rte_acl_create(const struct rte_acl_param *param)
+ +rte_acl_create_v20(const struct rte_acl_param *param)
+ {
+ size_t sz;
+ struct rte_acl_ctx *ctx;
+ ...
+
+Note that the base name of the symbol was kept intact, as this is conducive to
+the macros used for versioning symbols. That is our next step, mapping this new
+symbol name to the initial symbol name at version node 2.0. Immediately after
+the function, we add this line of code
+
+.. code-block:: c
+
+ VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
+
+Remembering to also add the rte_compat.h header to the requisite c file where
+these changes are being made. The above macro instructs the linker to create a
+new symbol ``rte_acl_create@DPDK_2.0``, which matches the symbol created in older
+builds, but now points to the above newly named function. We have now mapped
+the original rte_acl_create symbol to the original function (but with a new
+name)
+
+Next, we need to create the 2.1 version of the symbol. We create a new function
+name, with a different suffix, and implement it appropriately
+
+.. code-block:: c
+
+ struct rte_acl_ctx *
+ rte_acl_create_v21(const struct rte_acl_param *param, int debug);
+ {
+ struct rte_acl_ctx *ctx = rte_acl_create_v20(param);
+
+ ctx->debug = debug;
+
+ return ctx;
+ }
+
+This code serves as our new API call. Its the same as our old call, but adds
+the new parameter in place. Next we need to map this function to the symbol
+``rte_acl_create@DPDK_2.1``. To do this, we modify the public prototype of the call
+in the header file, adding the macro there to inform all including applications,
+that on re-link, the default rte_acl_create symbol should point to this
+function. Note that we could do this by simply naming the function above
+rte_acl_create, and the linker would chose the most recent version tag to apply
+in the version script, but we can also do this in the header file
+
+.. code-block:: c
+
+ struct rte_acl_ctx *
+ -rte_acl_create(const struct rte_acl_param *param);
+ +rte_acl_create(const struct rte_acl_param *param, int debug);
+ +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
+
+The BIND_DEFAULT_SYMBOL macro explicitly tells applications that include this
+header, to link to the rte_acl_create_v21 function and apply the DPDK_2.1
+version node to it. This method is more explicit and flexible than just
+re-implementing the exact symbol name, and allows for other features (such as
+linking to the old symbol version by default, when the new ABI is to be opt-in
+for a period.
+
+One last thing we need to do. Note that we've taken what was a public symbol,
+and duplicated it into two uniquely and differently named symbols. We've then
+mapped each of those back to the public symbol ``rte_acl_create`` with different
+version tags. This only applies to dynamic linking, as static linking has no
+notion of versioning. That leaves this code in a position of no longer having a
+symbol simply named ``rte_acl_create`` and a static build will fail on that
+missing symbol.
+
+To correct this, we can simply map a function of our choosing back to the public
+symbol in the static build with the ``MAP_STATIC_SYMBOL`` macro. Generally the
+assumption is that the most recent version of the symbol is the one you want to
+map. So, back in the C file where, immediately after ``rte_acl_create_v21`` is
+defined, we add this
+
+.. code-block:: c
+
+ struct rte_acl_ctx *
+ rte_acl_create_v21(const struct rte_acl_param *param, int debug)
+ {
+ ...
+ }
+ MAP_STATIC_SYMBOL(struct rte_acl_ctx *rte_acl_create(const struct rte_acl_param *param, int debug), rte_acl_create_v21);
+
+That tells the compiler that, when building a static library, any calls to the
+symbol ``rte_acl_create`` should be linked to ``rte_acl_create_v21``
+
+That's it, on the next shared library rebuild, there will be two versions of
+rte_acl_create, an old DPDK_2.0 version, used by previously built applications,
+and a new DPDK_2.1 version, used by future built applications.
+
+
+Deprecating part of a public API
+________________________________
+
+Lets assume that you've done the above update, and after a few releases have
+passed you decide you would like to retire the old version of the function.
+After having gone through the ABI deprecation announcement process, removal is
+easy. Start by removing the symbol from the requisite version map file:
+
+.. code-block:: none
+
+ DPDK_2.0 {
+ global:
+
+ rte_acl_add_rules;
+ rte_acl_build;
+ rte_acl_classify;
+ rte_acl_classify_alg;
+ rte_acl_classify_scalar;
+ rte_acl_dump;
+ - rte_acl_create
+ rte_acl_find_existing;
+ rte_acl_free;
+ rte_acl_ipv4vlan_add_rules;
+ rte_acl_ipv4vlan_build;
+ rte_acl_list_dump;
+ rte_acl_reset;
+ rte_acl_reset_rules;
+ rte_acl_set_ctx_classify;
+
+ local: *;
+ };
+
+ DPDK_2.1 {
+ global:
+ rte_acl_create;
+ } DPDK_2.0;
+
+
+Next remove the corresponding versioned export.
+
+.. code-block:: c
+
+ -VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
+
+
+Note that the internal function definition could also be removed, but its used
+in our example by the newer version _v21, so we leave it in place. This is a
+coding style choice.
+
+Lastly, we need to bump the LIBABIVER number for this library in the Makefile to
+indicate to applications doing dynamic linking that this is a later, and
+possibly incompatible library version:
+
+.. code-block:: c
+
+ -LIBABIVER := 1
+ +LIBABIVER := 2
+
+Deprecating an entire ABI version
+_________________________________
+
+While removing a symbol from and ABI may be useful, it is often more practical
+to remove an entire version node at once. If a version node completely
+specifies an API, then removing part of it, typically makes it incomplete. In
+those cases it is better to remove the entire node
+
+To do this, start by modifying the version map file, such that all symbols from
+the node to be removed are merged into the next node in the map
+
+In the case of our map above, it would transform to look as follows
+
+.. code-block:: none
+
+ DPDK_2.1 {
+ global:
+
+ rte_acl_add_rules;
+ rte_acl_build;
+ rte_acl_classify;
+ rte_acl_classify_alg;
+ rte_acl_classify_scalar;
+ rte_acl_dump;
+ rte_acl_create
+ rte_acl_find_existing;
+ rte_acl_free;
+ rte_acl_ipv4vlan_add_rules;
+ rte_acl_ipv4vlan_build;
+ rte_acl_list_dump;
+ rte_acl_reset;
+ rte_acl_reset_rules;
+ rte_acl_set_ctx_classify;
+
+ local: *;
+ };
+
+Then any uses of BIND_DEFAULT_SYMBOL that pointed to the old node should be
+updated to point to the new version node in any header files for all affected
+symbols.
+
+.. code-block:: c
+
+ -BIND_DEFAULT_SYMBOL(rte_acl_create, _v20, 2.0);
+ +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
+
+Lastly, any VERSION_SYMBOL macros that point to the old version node should be
+removed, taking care to keep, where need old code in place to support newer
+versions of the symbol.
+
+
+Running the ABI Validator
+-------------------------
+
+The ``devtools`` directory in the DPDK source tree contains a utility program,
+``validate-abi.sh``, for validating the DPDK ABI based on the Linux `ABI
+Compliance Checker
+<http://ispras.linuxbase.org/index.php/ABI_compliance_checker>`_.
+
+This has a dependency on the ``abi-compliance-checker`` and ``and abi-dumper``
+utilities which can be installed via a package manager. For example::
+
+ sudo yum install abi-compliance-checker
+ sudo yum install abi-dumper
+
+The syntax of the ``validate-abi.sh`` utility is::
+
+ ./devtools/validate-abi.sh <REV1> <REV2>
+
+Where ``REV1`` and ``REV2`` are valid gitrevisions(7)
+https://www.kernel.org/pub/software/scm/git/docs/gitrevisions.html
+on the local repo.
+
+For example::
+
+ # Check between the previous and latest commit:
+ ./devtools/validate-abi.sh HEAD~1 HEAD
+
+ # Check on a specific compilation target:
+ ./devtools/validate-abi.sh -t x86_64-native-linux-gcc HEAD~1 HEAD
+
+ # Check between two tags:
+ ./devtools/validate-abi.sh v2.0.0 v2.1.0
+
+ # Check between git master and local topic-branch "vhost-hacking":
+ ./devtools/validate-abi.sh master vhost-hacking
+
+After the validation script completes (it can take a while since it need to
+compile both tags) it will create compatibility reports in the
+``./abi-check/compat_report`` directory. Listed incompatibilities can be found
+as follows::
+
+ grep -lr Incompatible abi-check/compat_reports/
diff --git a/doc/guides/contributing/index.rst b/doc/guides/contributing/index.rst
index e2608d3..2fefd91 100644
--- a/doc/guides/contributing/index.rst
+++ b/doc/guides/contributing/index.rst
@@ -10,7 +10,8 @@ Contributor's Guidelines
coding_style
design
- versioning
+ abi_policy
+ abi_versioning
documentation
patches
vulnerability
diff --git a/doc/guides/contributing/versioning.rst b/doc/guides/contributing/versioning.rst
deleted file mode 100644
index 3ab2c43..0000000
--- a/doc/guides/contributing/versioning.rst
+++ /dev/null
@@ -1,591 +0,0 @@
-.. SPDX-License-Identifier: BSD-3-Clause
- Copyright 2018 The DPDK contributors
-
-DPDK ABI/API policy
-===================
-
-Description
------------
-
-This document details some methods for handling ABI management in the DPDK.
-
-General Guidelines
-------------------
-
-#. Whenever possible, ABI should be preserved
-#. ABI/API may be changed with a deprecation process
-#. The modification of symbols can generally be managed with versioning
-#. Libraries or APIs marked in ``experimental`` state may change without constraint
-#. New APIs will be marked as ``experimental`` for at least one release to allow
- any issues found by users of the new API to be fixed quickly
-#. The addition of symbols is generally not problematic
-#. The removal of symbols generally is an ABI break and requires bumping of the
- LIBABIVER macro
-#. Updates to the minimum hardware requirements, which drop support for hardware which
- was previously supported, should be treated as an ABI change.
-
-What is an ABI
-~~~~~~~~~~~~~~
-
-An ABI (Application Binary Interface) is the set of runtime interfaces exposed
-by a library. It is similar to an API (Application Programming Interface) but
-is the result of compilation. It is also effectively cloned when applications
-link to dynamic libraries. That is to say when an application is compiled to
-link against dynamic libraries, it is assumed that the ABI remains constant
-between the time the application is compiled/linked, and the time that it runs.
-Therefore, in the case of dynamic linking, it is critical that an ABI is
-preserved, or (when modified), done in such a way that the application is unable
-to behave improperly or in an unexpected fashion.
-
-
-ABI/API Deprecation
--------------------
-
-The DPDK ABI policy
-~~~~~~~~~~~~~~~~~~~
-
-ABI versions are set at the time of major release labeling, and the ABI may
-change multiple times, without warning, between the last release label and the
-HEAD label of the git tree.
-
-ABI versions, once released, are available until such time as their
-deprecation has been noted in the Release Notes for at least one major release
-cycle. For example consider the case where the ABI for DPDK 2.0 has been
-shipped and then a decision is made to modify it during the development of
-DPDK 2.1. The decision will be recorded in the Release Notes for the DPDK 2.1
-release and the modification will be made available in the DPDK 2.2 release.
-
-ABI versions may be deprecated in whole or in part as needed by a given
-update.
-
-Some ABI changes may be too significant to reasonably maintain multiple
-versions. In those cases ABI's may be updated without backward compatibility
-being provided. The requirements for doing so are:
-
-#. At least 3 acknowledgments of the need to do so must be made on the
- dpdk.org mailing list.
-
- - The acknowledgment of the maintainer of the component is mandatory, or if
- no maintainer is available for the component, the tree/sub-tree maintainer
- for that component must acknowledge the ABI change instead.
-
- - It is also recommended that acknowledgments from different "areas of
- interest" be sought for each deprecation, for example: from NIC vendors,
- CPU vendors, end-users, etc.
-
-#. The changes (including an alternative map file) can be included with
- deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option,
- to provide more details about oncoming changes.
- ``RTE_NEXT_ABI`` wrapper will be removed when it become the default ABI.
- More preferred way to provide this information is sending the feature
- as a separate patch and reference it in deprecation notice.
-
-#. A full deprecation cycle, as explained above, must be made to offer
- downstream consumers sufficient warning of the change.
-
-Note that the above process for ABI deprecation should not be undertaken
-lightly. ABI stability is extremely important for downstream consumers of the
-DPDK, especially when distributed in shared object form. Every effort should
-be made to preserve the ABI whenever possible. The ABI should only be changed
-for significant reasons, such as performance enhancements. ABI breakage due to
-changes such as reorganizing public structure fields for aesthetic or
-readability purposes should be avoided.
-
-.. note::
-
- Updates to the minimum hardware requirements, which drop support for hardware
- which was previously supported, should be treated as an ABI change, and
- follow the relevant deprecation policy procedures as above: 3 acks and
- announcement at least one release in advance.
-
-Examples of Deprecation Notices
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The following are some examples of ABI deprecation notices which would be
-added to the Release Notes:
-
-* The Macro ``#RTE_FOO`` is deprecated and will be removed with version 2.0,
- to be replaced with the inline function ``rte_foo()``.
-
-* The function ``rte_mbuf_grok()`` has been updated to include a new parameter
- in version 2.0. Backwards compatibility will be maintained for this function
- until the release of version 2.1
-
-* The members of ``struct rte_foo`` have been reorganized in release 2.0 for
- performance reasons. Existing binary applications will have backwards
- compatibility in release 2.0, while newly built binaries will need to
- reference the new structure variant ``struct rte_foo2``. Compatibility will
- be removed in release 2.2, and all applications will require updating and
- rebuilding to the new structure at that time, which will be renamed to the
- original ``struct rte_foo``.
-
-* Significant ABI changes are planned for the ``librte_dostuff`` library. The
- upcoming release 2.0 will not contain these changes, but release 2.1 will,
- and no backwards compatibility is planned due to the extensive nature of
- these changes. Binaries using this library built prior to version 2.1 will
- require updating and recompilation.
-
-New API replacing previous one
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-If a new API proposed functionally replaces an existing one, when the new API
-becomes non-experimental then the old one is marked with ``__rte_deprecated``.
-Deprecated APIs are removed completely just after the next LTS.
-
-Reminder that old API should follow deprecation process to be removed.
-
-
-Experimental APIs
------------------
-
-APIs marked as ``experimental`` are not considered part of the ABI and may
-change without warning at any time. Since changes to APIs are most likely
-immediately after their introduction, as users begin to take advantage of
-those new APIs and start finding issues with them, new DPDK APIs will be
-automatically marked as ``experimental`` to allow for a period of stabilization
-before they become part of a tracked ABI.
-
-Note that marking an API as experimental is a multi step process.
-To mark an API as experimental, the symbols which are desired to be exported
-must be placed in an EXPERIMENTAL version block in the corresponding libraries'
-version map script.
-Secondly, the corresponding prototypes of those exported functions (in the
-development header files), must be marked with the ``__rte_experimental`` tag
-(see ``rte_compat.h``).
-The DPDK build makefiles perform a check to ensure that the map file and the
-C code reflect the same list of symbols.
-This check can be circumvented by defining ``ALLOW_EXPERIMENTAL_API``
-during compilation in the corresponding library Makefile.
-
-In addition to tagging the code with ``__rte_experimental``,
-the doxygen markup must also contain the EXPERIMENTAL string,
-and the MAINTAINERS file should note the EXPERIMENTAL libraries.
-
-For removing the experimental tag associated with an API, deprecation notice
-is not required. Though, an API should remain in experimental state for at least
-one release. Thereafter, normal process of posting patch for review to mailing
-list can be followed.
-
-
-Library versioning
-------------------
-
-Downstreams might want to provide different DPDK releases at the same time to
-support multiple consumers of DPDK linked against older and newer sonames.
-
-Also due to the interdependencies that DPDK libraries can have applications
-might end up with an executable space in which multiple versions of a library
-are mapped by ld.so.
-
-Think of LibA that got an ABI bump and LibB that did not get an ABI bump but is
-depending on LibA.
-
-.. note::
-
- Application
- \-> LibA.old
- \-> LibB.new -> LibA.new
-
-That is a conflict which can be avoided by setting ``CONFIG_RTE_MAJOR_ABI``.
-If set, the value of ``CONFIG_RTE_MAJOR_ABI`` overwrites all - otherwise per
-library - versions defined in the libraries ``LIBABIVER``.
-An example might be ``CONFIG_RTE_MAJOR_ABI=16.11`` which will make all libraries
-``librte<?>.so.16.11`` instead of ``librte<?>.so.<LIBABIVER>``.
-
-
-ABI versioning
---------------
-
-Versioning Macros
-~~~~~~~~~~~~~~~~~
-
-When a symbol is exported from a library to provide an API, it also provides a
-calling convention (ABI) that is embodied in its name, return type and
-arguments. Occasionally that function may need to change to accommodate new
-functionality or behavior. When that occurs, it is desirable to allow for
-backward compatibility for a time with older binaries that are dynamically
-linked to the DPDK.
-
-To support backward compatibility the ``rte_compat.h``
-header file provides macros to use when updating exported functions. These
-macros are used in conjunction with the ``rte_<library>_version.map`` file for
-a given library to allow multiple versions of a symbol to exist in a shared
-library so that older binaries need not be immediately recompiled.
-
-The macros exported are:
-
-* ``VERSION_SYMBOL(b, e, n)``: Creates a symbol version table entry binding
- versioned symbol ``b@DPDK_n`` to the internal function ``b_e``.
-
-* ``BIND_DEFAULT_SYMBOL(b, e, n)``: Creates a symbol version entry instructing
- the linker to bind references to symbol ``b`` to the internal symbol
- ``b_e``.
-
-* ``MAP_STATIC_SYMBOL(f, p)``: Declare the prototype ``f``, and map it to the
- fully qualified function ``p``, so that if a symbol becomes versioned, it
- can still be mapped back to the public symbol name.
-
-Examples of ABI Macro use
-^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Updating a public API
-_____________________
-
-Assume we have a function as follows
-
-.. code-block:: c
-
- /*
- * Create an acl context object for apps to
- * manipulate
- */
- struct rte_acl_ctx *
- rte_acl_create(const struct rte_acl_param *param)
- {
- ...
- }
-
-
-Assume that struct rte_acl_ctx is a private structure, and that a developer
-wishes to enhance the acl api so that a debugging flag can be enabled on a
-per-context basis. This requires an addition to the structure (which, being
-private, is safe), but it also requires modifying the code as follows
-
-.. code-block:: c
-
- /*
- * Create an acl context object for apps to
- * manipulate
- */
- struct rte_acl_ctx *
- rte_acl_create(const struct rte_acl_param *param, int debug)
- {
- ...
- }
-
-
-Note also that, being a public function, the header file prototype must also be
-changed, as must all the call sites, to reflect the new ABI footprint. We will
-maintain previous ABI versions that are accessible only to previously compiled
-binaries
-
-The addition of a parameter to the function is ABI breaking as the function is
-public, and existing application may use it in its current form. However, the
-compatibility macros in DPDK allow a developer to use symbol versioning so that
-multiple functions can be mapped to the same public symbol based on when an
-application was linked to it. To see how this is done, we start with the
-requisite libraries version map file. Initially the version map file for the
-acl library looks like this
-
-.. code-block:: none
-
- DPDK_2.0 {
- global:
-
- rte_acl_add_rules;
- rte_acl_build;
- rte_acl_classify;
- rte_acl_classify_alg;
- rte_acl_classify_scalar;
- rte_acl_create;
- rte_acl_dump;
- rte_acl_find_existing;
- rte_acl_free;
- rte_acl_ipv4vlan_add_rules;
- rte_acl_ipv4vlan_build;
- rte_acl_list_dump;
- rte_acl_reset;
- rte_acl_reset_rules;
- rte_acl_set_ctx_classify;
-
- local: *;
- };
-
-This file needs to be modified as follows
-
-.. code-block:: none
-
- DPDK_2.0 {
- global:
-
- rte_acl_add_rules;
- rte_acl_build;
- rte_acl_classify;
- rte_acl_classify_alg;
- rte_acl_classify_scalar;
- rte_acl_create;
- rte_acl_dump;
- rte_acl_find_existing;
- rte_acl_free;
- rte_acl_ipv4vlan_add_rules;
- rte_acl_ipv4vlan_build;
- rte_acl_list_dump;
- rte_acl_reset;
- rte_acl_reset_rules;
- rte_acl_set_ctx_classify;
-
- local: *;
- };
-
- DPDK_2.1 {
- global:
- rte_acl_create;
-
- } DPDK_2.0;
-
-The addition of the new block tells the linker that a new version node is
-available (DPDK_2.1), which contains the symbol rte_acl_create, and inherits the
-symbols from the DPDK_2.0 node. This list is directly translated into a list of
-exported symbols when DPDK is compiled as a shared library
-
-Next, we need to specify in the code which function map to the rte_acl_create
-symbol at which versions. First, at the site of the initial symbol definition,
-we need to update the function so that it is uniquely named, and not in conflict
-with the public symbol name
-
-.. code-block:: c
-
- struct rte_acl_ctx *
- -rte_acl_create(const struct rte_acl_param *param)
- +rte_acl_create_v20(const struct rte_acl_param *param)
- {
- size_t sz;
- struct rte_acl_ctx *ctx;
- ...
-
-Note that the base name of the symbol was kept intact, as this is conducive to
-the macros used for versioning symbols. That is our next step, mapping this new
-symbol name to the initial symbol name at version node 2.0. Immediately after
-the function, we add this line of code
-
-.. code-block:: c
-
- VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
-
-Remembering to also add the rte_compat.h header to the requisite c file where
-these changes are being made. The above macro instructs the linker to create a
-new symbol ``rte_acl_create@DPDK_2.0``, which matches the symbol created in older
-builds, but now points to the above newly named function. We have now mapped
-the original rte_acl_create symbol to the original function (but with a new
-name)
-
-Next, we need to create the 2.1 version of the symbol. We create a new function
-name, with a different suffix, and implement it appropriately
-
-.. code-block:: c
-
- struct rte_acl_ctx *
- rte_acl_create_v21(const struct rte_acl_param *param, int debug);
- {
- struct rte_acl_ctx *ctx = rte_acl_create_v20(param);
-
- ctx->debug = debug;
-
- return ctx;
- }
-
-This code serves as our new API call. Its the same as our old call, but adds
-the new parameter in place. Next we need to map this function to the symbol
-``rte_acl_create@DPDK_2.1``. To do this, we modify the public prototype of the call
-in the header file, adding the macro there to inform all including applications,
-that on re-link, the default rte_acl_create symbol should point to this
-function. Note that we could do this by simply naming the function above
-rte_acl_create, and the linker would chose the most recent version tag to apply
-in the version script, but we can also do this in the header file
-
-.. code-block:: c
-
- struct rte_acl_ctx *
- -rte_acl_create(const struct rte_acl_param *param);
- +rte_acl_create(const struct rte_acl_param *param, int debug);
- +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
-
-The BIND_DEFAULT_SYMBOL macro explicitly tells applications that include this
-header, to link to the rte_acl_create_v21 function and apply the DPDK_2.1
-version node to it. This method is more explicit and flexible than just
-re-implementing the exact symbol name, and allows for other features (such as
-linking to the old symbol version by default, when the new ABI is to be opt-in
-for a period.
-
-One last thing we need to do. Note that we've taken what was a public symbol,
-and duplicated it into two uniquely and differently named symbols. We've then
-mapped each of those back to the public symbol ``rte_acl_create`` with different
-version tags. This only applies to dynamic linking, as static linking has no
-notion of versioning. That leaves this code in a position of no longer having a
-symbol simply named ``rte_acl_create`` and a static build will fail on that
-missing symbol.
-
-To correct this, we can simply map a function of our choosing back to the public
-symbol in the static build with the ``MAP_STATIC_SYMBOL`` macro. Generally the
-assumption is that the most recent version of the symbol is the one you want to
-map. So, back in the C file where, immediately after ``rte_acl_create_v21`` is
-defined, we add this
-
-.. code-block:: c
-
- struct rte_acl_ctx *
- rte_acl_create_v21(const struct rte_acl_param *param, int debug)
- {
- ...
- }
- MAP_STATIC_SYMBOL(struct rte_acl_ctx *rte_acl_create(const struct rte_acl_param *param, int debug), rte_acl_create_v21);
-
-That tells the compiler that, when building a static library, any calls to the
-symbol ``rte_acl_create`` should be linked to ``rte_acl_create_v21``
-
-That's it, on the next shared library rebuild, there will be two versions of
-rte_acl_create, an old DPDK_2.0 version, used by previously built applications,
-and a new DPDK_2.1 version, used by future built applications.
-
-
-Deprecating part of a public API
-________________________________
-
-Lets assume that you've done the above update, and after a few releases have
-passed you decide you would like to retire the old version of the function.
-After having gone through the ABI deprecation announcement process, removal is
-easy. Start by removing the symbol from the requisite version map file:
-
-.. code-block:: none
-
- DPDK_2.0 {
- global:
-
- rte_acl_add_rules;
- rte_acl_build;
- rte_acl_classify;
- rte_acl_classify_alg;
- rte_acl_classify_scalar;
- rte_acl_dump;
- - rte_acl_create
- rte_acl_find_existing;
- rte_acl_free;
- rte_acl_ipv4vlan_add_rules;
- rte_acl_ipv4vlan_build;
- rte_acl_list_dump;
- rte_acl_reset;
- rte_acl_reset_rules;
- rte_acl_set_ctx_classify;
-
- local: *;
- };
-
- DPDK_2.1 {
- global:
- rte_acl_create;
- } DPDK_2.0;
-
-
-Next remove the corresponding versioned export.
-
-.. code-block:: c
-
- -VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
-
-
-Note that the internal function definition could also be removed, but its used
-in our example by the newer version _v21, so we leave it in place. This is a
-coding style choice.
-
-Lastly, we need to bump the LIBABIVER number for this library in the Makefile to
-indicate to applications doing dynamic linking that this is a later, and
-possibly incompatible library version:
-
-.. code-block:: c
-
- -LIBABIVER := 1
- +LIBABIVER := 2
-
-Deprecating an entire ABI version
-_________________________________
-
-While removing a symbol from and ABI may be useful, it is often more practical
-to remove an entire version node at once. If a version node completely
-specifies an API, then removing part of it, typically makes it incomplete. In
-those cases it is better to remove the entire node
-
-To do this, start by modifying the version map file, such that all symbols from
-the node to be removed are merged into the next node in the map
-
-In the case of our map above, it would transform to look as follows
-
-.. code-block:: none
-
- DPDK_2.1 {
- global:
-
- rte_acl_add_rules;
- rte_acl_build;
- rte_acl_classify;
- rte_acl_classify_alg;
- rte_acl_classify_scalar;
- rte_acl_dump;
- rte_acl_create
- rte_acl_find_existing;
- rte_acl_free;
- rte_acl_ipv4vlan_add_rules;
- rte_acl_ipv4vlan_build;
- rte_acl_list_dump;
- rte_acl_reset;
- rte_acl_reset_rules;
- rte_acl_set_ctx_classify;
-
- local: *;
- };
-
-Then any uses of BIND_DEFAULT_SYMBOL that pointed to the old node should be
-updated to point to the new version node in any header files for all affected
-symbols.
-
-.. code-block:: c
-
- -BIND_DEFAULT_SYMBOL(rte_acl_create, _v20, 2.0);
- +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
-
-Lastly, any VERSION_SYMBOL macros that point to the old version node should be
-removed, taking care to keep, where need old code in place to support newer
-versions of the symbol.
-
-
-Running the ABI Validator
--------------------------
-
-The ``devtools`` directory in the DPDK source tree contains a utility program,
-``validate-abi.sh``, for validating the DPDK ABI based on the Linux `ABI
-Compliance Checker
-<http://ispras.linuxbase.org/index.php/ABI_compliance_checker>`_.
-
-This has a dependency on the ``abi-compliance-checker`` and ``and abi-dumper``
-utilities which can be installed via a package manager. For example::
-
- sudo yum install abi-compliance-checker
- sudo yum install abi-dumper
-
-The syntax of the ``validate-abi.sh`` utility is::
-
- ./devtools/validate-abi.sh <REV1> <REV2>
-
-Where ``REV1`` and ``REV2`` are valid gitrevisions(7)
-https://www.kernel.org/pub/software/scm/git/docs/gitrevisions.html
-on the local repo.
-
-For example::
-
- # Check between the previous and latest commit:
- ./devtools/validate-abi.sh HEAD~1 HEAD
-
- # Check on a specific compilation target:
- ./devtools/validate-abi.sh -t x86_64-native-linux-gcc HEAD~1 HEAD
-
- # Check between two tags:
- ./devtools/validate-abi.sh v2.0.0 v2.1.0
-
- # Check between git master and local topic-branch "vhost-hacking":
- ./devtools/validate-abi.sh master vhost-hacking
-
-After the validation script completes (it can take a while since it need to
-compile both tags) it will create compatibility reports in the
-``./abi-check/compat_report`` directory. Listed incompatibilities can be found
-as follows::
-
- grep -lr Incompatible abi-check/compat_reports/
--
2.7.4
^ permalink raw reply [relevance 13%]
* [dpdk-dev] [PATCH v3 3/4] doc: updates to versioning guide for abi versions
2019-08-15 10:23 11% [dpdk-dev] [PATCH v3 0/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
2019-08-15 10:23 13% ` [dpdk-dev] [PATCH v3 1/4] doc: separate versioning.rst into version and policy Ray Kinsella
2019-08-15 10:23 31% ` [dpdk-dev] [PATCH v3 2/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
@ 2019-08-15 10:23 30% ` Ray Kinsella
2019-08-15 10:23 13% ` [dpdk-dev] [PATCH v3 4/4] doc: add maintainer for abi policy Ray Kinsella
3 siblings, 0 replies; 200+ results
From: Ray Kinsella @ 2019-08-15 10:23 UTC (permalink / raw)
To: dev
Cc: mdr, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, nhorman,
maxime.coquelin, john.mcnamara, marko.kovacevic, hemant.agrawal
Updates to the ABI versioning guide, to account for the changes to the DPDK
ABI/API policy. Fixes for references to abi versioning and policy guides.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
doc/guides/contributing/abi_versioning.rst | 248 +++++++++++++++++++----------
doc/guides/contributing/patches.rst | 6 +-
doc/guides/rel_notes/deprecation.rst | 2 +-
3 files changed, 172 insertions(+), 84 deletions(-)
diff --git a/doc/guides/contributing/abi_versioning.rst b/doc/guides/contributing/abi_versioning.rst
index 53e6ac0..76c63c8 100644
--- a/doc/guides/contributing/abi_versioning.rst
+++ b/doc/guides/contributing/abi_versioning.rst
@@ -1,44 +1,134 @@
.. SPDX-License-Identifier: BSD-3-Clause
Copyright 2018 The DPDK contributors
-.. library_versioning:
+.. _abi_versioning:
-Library versioning
+ABI Versioning
+==============
+
+This document details the mechanics of ABI version management in DPDK.
+
+.. _what_is_soname:
+
+What is a library's soname?
+---------------------------
+
+System libraries usually adopt the familiar major and minor version naming
+convention, where major versions (e.g. ``librte_eal 20.x, 21.x``) are presumed
+to be ABI incompatible with each other and minor versions (e.g. ``librte_eal
+20.1, 20.2``) are presumed to be ABI compatible. A library's `soname
+<https://en.wikipedia.org/wiki/Soname>`_. is typically used to provide backward
+compatibility information about a given library, describing the lowest common
+denominator ABI supported by the library. The soname or logical name for the
+library, is typically comprised of the library's name and major version e.g.
+``librte_eal.so.20``.
+
+During an application's build process, a library's soname is noted as a runtime
+dependency of the application. This information is then used by the `dynamic
+linker <https://en.wikipedia.org/wiki/Dynamic_linker>`_ when resolving the
+applications dependencies at runtime, to load a library supporting the correct
+ABI version. The library loaded at runtime therefore, may be a minor revision
+supporting the same major ABI version (e.g. ``librte_eal.20.2``), as the library
+used to link the application (e.g ``librte_eal.20.0``).
+
+.. _major_abi_versions:
+
+Major ABI versions
------------------
-Downstreams might want to provide different DPDK releases at the same time to
-support multiple consumers of DPDK linked against older and newer sonames.
+An ABI version change to a given library, especially in core libraries such as
+``librte_mbuf``, may cause an implicit ripple effect on the ABI of it's
+consuming libraries, causing ABI breakages. There may however be no explicit
+reason to bump a dependent library's ABI version, as there may have been no
+obvious change to the dependent library's API, even though the library's ABI
+compatibility will have been broken.
+
+This interdependence of DPDK libraries, means that ABI versioning of libraries
+is more manageable at a project level, with all project libraries sharing a
+**single ABI version**. In addition, the need to maintain a stable ABI for some
+number of releases as described in the section :ref:`abi_policy`, means
+that ABI version increments need to carefully planned and managed at a project
+level.
+
+Major ABI versions are therefore declared typically aligned with an LTS release
+and is then supported some number of subsequent releases, shared across all
+libraries. This means that a single project level ABI version, reflected in all
+individual library's soname, library filenames and associated version maps
+persists over multiple releases.
+
+.. code-block:: none
+
+ $ head ./lib/librte_acl/rte_acl_version.map
+ DPDK_20 {
+ global:
+ ...
-Also due to the interdependencies that DPDK libraries can have applications
-might end up with an executable space in which multiple versions of a library
-are mapped by ld.so.
+ $ head ./lib/librte_eal/rte_eal_version.map
+ DPDK_20 {
+ global:
+ ...
-Think of LibA that got an ABI bump and LibB that did not get an ABI bump but is
-depending on LibA.
+When an ABI change is made between major ABI versions to a given library, a new
+section is added to that library's version map describing the impending new ABI
+version, as described in the section :ref:`example_abi_macro_usage`. The
+library's soname and filename however do not change, e.g. ``libacl.so.20``, as
+ABI compatibility with the last major ABI version continues to be preserved for
+that library.
-.. note::
+.. code-block:: none
- Application
- \-> LibA.old
- \-> LibB.new -> LibA.new
+ $ head ./lib/librte_acl/rte_acl_version.map
+ DPDK_20 {
+ global:
+ ...
-That is a conflict which can be avoided by setting ``CONFIG_RTE_MAJOR_ABI``.
-If set, the value of ``CONFIG_RTE_MAJOR_ABI`` overwrites all - otherwise per
-library - versions defined in the libraries ``LIBABIVER``.
-An example might be ``CONFIG_RTE_MAJOR_ABI=16.11`` which will make all libraries
-``librte<?>.so.16.11`` instead of ``librte<?>.so.<LIBABIVER>``.
+ DPDK_21 {
+ global:
+
+ } DPDK_20;
+ ...
+ $ head ./lib/librte_eal/rte_eal_version.map
+ DPDK_20 {
+ global:
+ ...
+
+However when a new ABI version is declared, for example DPDK ``21``, old
+depreciated functions may be safely removed at this point and the entire old
+major ABI version removed, see the section :ref:`deprecating_entire_abi` on
+how this may be done.
+
+.. code-block:: none
+
+ $ head ./lib/librte_acl/rte_acl_version.map
+ DPDK_21 {
+ global:
+ ...
+
+ $ head ./lib/librte_eal/rte_eal_version.map
+ DPDK_21 {
+ global:
+ ...
+
+At the same time, the major ABI version is changed atomically across all
+libraries by incrementing the major version in individual library's soname, e.g.
+``libacl.so.21``. This is done by bumping the LIBABIVER number in the libraries
+Makefile to indicate to dynamic linking applications that this is a later, and
+possibly incompatible library version:
+
+.. code-block:: c
+
+ -LIBABIVER := 20
+ +LIBABIVER := 21
-ABI versioning
---------------
Versioning Macros
-~~~~~~~~~~~~~~~~~
+-----------------
When a symbol is exported from a library to provide an API, it also provides a
calling convention (ABI) that is embodied in its name, return type and
arguments. Occasionally that function may need to change to accommodate new
-functionality or behavior. When that occurs, it is desirable to allow for
+functionality or behavior. When that occurs, it is may be required to allow for
backward compatibility for a time with older binaries that are dynamically
linked to the DPDK.
@@ -61,8 +151,10 @@ The macros exported are:
fully qualified function ``p``, so that if a symbol becomes versioned, it
can still be mapped back to the public symbol name.
+.. _example_abi_macro_usage:
+
Examples of ABI Macro use
-^^^^^^^^^^^^^^^^^^^^^^^^^
+~~~~~~~~~~~~~~~~~~~~~~~~~
Updating a public API
_____________________
@@ -106,16 +198,16 @@ maintain previous ABI versions that are accessible only to previously compiled
binaries
The addition of a parameter to the function is ABI breaking as the function is
-public, and existing application may use it in its current form. However, the
+public, and existing application may use it in its current form. However, the
compatibility macros in DPDK allow a developer to use symbol versioning so that
multiple functions can be mapped to the same public symbol based on when an
-application was linked to it. To see how this is done, we start with the
-requisite libraries version map file. Initially the version map file for the
-acl library looks like this
+application was linked to it. To see how this is done, we start with the
+requisite libraries version map file. Initially the version map file for the acl
+library looks like this
.. code-block:: none
- DPDK_2.0 {
+ DPDK_20 {
global:
rte_acl_add_rules;
@@ -141,7 +233,7 @@ This file needs to be modified as follows
.. code-block:: none
- DPDK_2.0 {
+ DPDK_20 {
global:
rte_acl_add_rules;
@@ -163,16 +255,16 @@ This file needs to be modified as follows
local: *;
};
- DPDK_2.1 {
+ DPDK_21 {
global:
rte_acl_create;
- } DPDK_2.0;
+ } DPDK_20;
The addition of the new block tells the linker that a new version node is
-available (DPDK_2.1), which contains the symbol rte_acl_create, and inherits the
-symbols from the DPDK_2.0 node. This list is directly translated into a list of
-exported symbols when DPDK is compiled as a shared library
+available (DPDK_21), which contains the symbol rte_acl_create, and inherits
+the symbols from the DPDK_20 node. This list is directly translated into a
+list of exported symbols when DPDK is compiled as a shared library
Next, we need to specify in the code which function map to the rte_acl_create
symbol at which versions. First, at the site of the initial symbol definition,
@@ -191,22 +283,22 @@ with the public symbol name
Note that the base name of the symbol was kept intact, as this is conducive to
the macros used for versioning symbols. That is our next step, mapping this new
-symbol name to the initial symbol name at version node 2.0. Immediately after
+symbol name to the initial symbol name at version node 20. Immediately after
the function, we add this line of code
.. code-block:: c
- VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
+ VERSION_SYMBOL(rte_acl_create, _v20, 20);
Remembering to also add the rte_compat.h header to the requisite c file where
-these changes are being made. The above macro instructs the linker to create a
-new symbol ``rte_acl_create@DPDK_2.0``, which matches the symbol created in older
-builds, but now points to the above newly named function. We have now mapped
-the original rte_acl_create symbol to the original function (but with a new
-name)
+these changes are being made. The above macro instructs the linker to create a
+new symbol ``rte_acl_create@DPDK_20``, which matches the symbol created in
+older builds, but now points to the above newly named function. We have now
+mapped the original rte_acl_create symbol to the original function (but with a
+new name)
-Next, we need to create the 2.1 version of the symbol. We create a new function
-name, with a different suffix, and implement it appropriately
+Next, we need to create the 21 version of the symbol. We create a new function
+name, with a different suffix, and implement it appropriately
.. code-block:: c
@@ -220,12 +312,12 @@ name, with a different suffix, and implement it appropriately
return ctx;
}
-This code serves as our new API call. Its the same as our old call, but adds
-the new parameter in place. Next we need to map this function to the symbol
-``rte_acl_create@DPDK_2.1``. To do this, we modify the public prototype of the call
-in the header file, adding the macro there to inform all including applications,
-that on re-link, the default rte_acl_create symbol should point to this
-function. Note that we could do this by simply naming the function above
+This code serves as our new API call. Its the same as our old call, but adds the
+new parameter in place. Next we need to map this function to the symbol
+``rte_acl_create@DPDK_21``. To do this, we modify the public prototype of the
+call in the header file, adding the macro there to inform all including
+applications, that on re-link, the default rte_acl_create symbol should point to
+this function. Note that we could do this by simply naming the function above
rte_acl_create, and the linker would chose the most recent version tag to apply
in the version script, but we can also do this in the header file
@@ -233,11 +325,11 @@ in the version script, but we can also do this in the header file
struct rte_acl_ctx *
-rte_acl_create(const struct rte_acl_param *param);
- +rte_acl_create(const struct rte_acl_param *param, int debug);
- +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
+ +rte_acl_create_v21(const struct rte_acl_param *param, int debug);
+ +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 21);
The BIND_DEFAULT_SYMBOL macro explicitly tells applications that include this
-header, to link to the rte_acl_create_v21 function and apply the DPDK_2.1
+header, to link to the rte_acl_create_v21 function and apply the DPDK_21
version node to it. This method is more explicit and flexible than just
re-implementing the exact symbol name, and allows for other features (such as
linking to the old symbol version by default, when the new ABI is to be opt-in
@@ -257,6 +349,7 @@ assumption is that the most recent version of the symbol is the one you want to
map. So, back in the C file where, immediately after ``rte_acl_create_v21`` is
defined, we add this
+
.. code-block:: c
struct rte_acl_ctx *
@@ -270,21 +363,22 @@ That tells the compiler that, when building a static library, any calls to the
symbol ``rte_acl_create`` should be linked to ``rte_acl_create_v21``
That's it, on the next shared library rebuild, there will be two versions of
-rte_acl_create, an old DPDK_2.0 version, used by previously built applications,
-and a new DPDK_2.1 version, used by future built applications.
+rte_acl_create, an old DPDK_20 version, used by previously built applications,
+and a new DPDK_21 version, used by future built applications.
Deprecating part of a public API
________________________________
-Lets assume that you've done the above update, and after a few releases have
-passed you decide you would like to retire the old version of the function.
-After having gone through the ABI deprecation announcement process, removal is
-easy. Start by removing the symbol from the requisite version map file:
+Lets assume that you've done the above update, and in preparation for the next
+major ABI version you decide you would like to retire the old version of the
+function. After having gone through the ABI deprecation announcement process,
+removal is easy. Start by removing the symbol from the requisite version map
+file:
.. code-block:: none
- DPDK_2.0 {
+ DPDK_20 {
global:
rte_acl_add_rules;
@@ -306,48 +400,42 @@ easy. Start by removing the symbol from the requisite version map file:
local: *;
};
- DPDK_2.1 {
+ DPDK_21 {
global:
rte_acl_create;
- } DPDK_2.0;
+ } DPDK_20;
Next remove the corresponding versioned export.
.. code-block:: c
- -VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
+ -VERSION_SYMBOL(rte_acl_create, _v20, 20);
Note that the internal function definition could also be removed, but its used
-in our example by the newer version _v21, so we leave it in place. This is a
-coding style choice.
-
-Lastly, we need to bump the LIBABIVER number for this library in the Makefile to
-indicate to applications doing dynamic linking that this is a later, and
-possibly incompatible library version:
-
-.. code-block:: c
+in our example by the newer version v21, so we leave it in place and declare it
+as static. This is a coding style choice.
- -LIBABIVER := 1
- +LIBABIVER := 2
+.. _deprecating_entire_abi:
Deprecating an entire ABI version
_________________________________
-While removing a symbol from and ABI may be useful, it is often more practical
-to remove an entire version node at once. If a version node completely
-specifies an API, then removing part of it, typically makes it incomplete. In
-those cases it is better to remove the entire node
+While removing a symbol from an ABI may be useful, it is more practical to
+remove an entire version node at once, as is typically done at the declaration
+of a major ABI version. If a version node completely specifies an API, then
+removing part of it, typically makes it incomplete. In those cases it is better
+to remove the entire node.
To do this, start by modifying the version map file, such that all symbols from
-the node to be removed are merged into the next node in the map
+the node to be removed are merged into the next node in the map.
In the case of our map above, it would transform to look as follows
.. code-block:: none
- DPDK_2.1 {
+ DPDK_21 {
global:
rte_acl_add_rules;
@@ -375,8 +463,8 @@ symbols.
.. code-block:: c
- -BIND_DEFAULT_SYMBOL(rte_acl_create, _v20, 2.0);
- +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
+ -BIND_DEFAULT_SYMBOL(rte_acl_create, _v20, 20);
+ +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 21);
Lastly, any VERSION_SYMBOL macros that point to the old version node should be
removed, taking care to keep, where need old code in place to support newer
diff --git a/doc/guides/contributing/patches.rst b/doc/guides/contributing/patches.rst
index 9e1013b..d63c9f5 100644
--- a/doc/guides/contributing/patches.rst
+++ b/doc/guides/contributing/patches.rst
@@ -156,9 +156,9 @@ Make your planned changes in the cloned ``dpdk`` repo. Here are some guidelines
* For other PMDs and more info, refer to the ``MAINTAINERS`` file.
-* New external functions should be added to the local ``version.map`` file.
- See the :doc:`Guidelines for ABI policy and versioning </contributing/versioning>`.
- New external functions should also be added in alphabetical order.
+* New external functions should be added to the local ``version.map`` file. See
+ the :ref:`ABI policy <abi_policy>` and :ref:`ABI versioning <abi_versioning>`
+ guides. New external functions should also be added in alphabetical order.
* Important changes will require an addition to the release notes in ``doc/guides/rel_notes/``.
See the :ref:`Release Notes section of the Documentation Guidelines <doc_guidelines>` for details.
diff --git a/doc/guides/rel_notes/deprecation.rst b/doc/guides/rel_notes/deprecation.rst
index 0ee8533..2e163a5 100644
--- a/doc/guides/rel_notes/deprecation.rst
+++ b/doc/guides/rel_notes/deprecation.rst
@@ -4,7 +4,7 @@
ABI and API Deprecation
=======================
-See the :doc:`guidelines document for details of the ABI policy </contributing/versioning>`.
+See the :ref:`guidelines document for details of the ABI policy <abi_policy>`.
API and ABI deprecation notices are to be posted here.
--
2.7.4
^ permalink raw reply [relevance 30%]
* [dpdk-dev] [PATCH v3 4/4] doc: add maintainer for abi policy
2019-08-15 10:23 11% [dpdk-dev] [PATCH v3 0/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
` (2 preceding siblings ...)
2019-08-15 10:23 30% ` [dpdk-dev] [PATCH v3 3/4] doc: updates to versioning guide for " Ray Kinsella
@ 2019-08-15 10:23 13% ` Ray Kinsella
3 siblings, 0 replies; 200+ results
From: Ray Kinsella @ 2019-08-15 10:23 UTC (permalink / raw)
To: dev
Cc: mdr, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, nhorman,
maxime.coquelin, john.mcnamara, marko.kovacevic, hemant.agrawal
Add an entry to the maintainer file for the abi policy.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
MAINTAINERS | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/MAINTAINERS b/MAINTAINERS
index 4100260..a36afc1 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -81,6 +81,10 @@ M: Marko Kovacevic <marko.kovacevic@intel.com>
F: README
F: doc/
+ABI Policy
+M: Ray Kinsella <mdr@ashroe.eu>
+F: doc/guides/contributing/abi_*.rst
+
Developers and Maintainers Tools
M: Thomas Monjalon <thomas@monjalon.net>
F: MAINTAINERS
--
2.7.4
^ permalink raw reply [relevance 13%]
* [dpdk-dev] [PATCH v3 2/4] doc: changes to abi policy introducing major abi versions
2019-08-15 10:23 11% [dpdk-dev] [PATCH v3 0/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
2019-08-15 10:23 13% ` [dpdk-dev] [PATCH v3 1/4] doc: separate versioning.rst into version and policy Ray Kinsella
@ 2019-08-15 10:23 31% ` Ray Kinsella
2019-08-30 16:20 10% ` Kevin Traynor
2019-08-15 10:23 30% ` [dpdk-dev] [PATCH v3 3/4] doc: updates to versioning guide for " Ray Kinsella
2019-08-15 10:23 13% ` [dpdk-dev] [PATCH v3 4/4] doc: add maintainer for abi policy Ray Kinsella
3 siblings, 1 reply; 200+ results
From: Ray Kinsella @ 2019-08-15 10:23 UTC (permalink / raw)
To: dev
Cc: mdr, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, nhorman,
maxime.coquelin, john.mcnamara, marko.kovacevic, hemant.agrawal
This policy change introduces major ABI versions, these are
declared every year, typically aligned with the LTS release
and are supported by subsequent releases in the following year.
This change is intended to improve ABI stabilty for those projects
consuming DPDK.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
doc/guides/contributing/abi_policy.rst | 308 ++++++++++++++++++++++++---------
doc/guides/contributing/stable.rst | 38 ++--
2 files changed, 245 insertions(+), 101 deletions(-)
diff --git a/doc/guides/contributing/abi_policy.rst b/doc/guides/contributing/abi_policy.rst
index 55bacb4..6190bdc 100644
--- a/doc/guides/contributing/abi_policy.rst
+++ b/doc/guides/contributing/abi_policy.rst
@@ -1,33 +1,46 @@
.. SPDX-License-Identifier: BSD-3-Clause
- Copyright 2018 The DPDK contributors
+ Copyright 2019 The DPDK contributors
-.. abi_api_policy:
+.. _abi_policy:
-DPDK ABI/API policy
-===================
+ABI Policy
+==========
Description
-----------
-This document details some methods for handling ABI management in the DPDK.
+This document details the management policy that ensures the long-term stability
+of the DPDK ABI and API.
General Guidelines
------------------
-#. Whenever possible, ABI should be preserved
-#. ABI/API may be changed with a deprecation process
-#. The modification of symbols can generally be managed with versioning
-#. Libraries or APIs marked in ``experimental`` state may change without constraint
-#. New APIs will be marked as ``experimental`` for at least one release to allow
- any issues found by users of the new API to be fixed quickly
-#. The addition of symbols is generally not problematic
-#. The removal of symbols generally is an ABI break and requires bumping of the
- LIBABIVER macro
-#. Updates to the minimum hardware requirements, which drop support for hardware which
- was previously supported, should be treated as an ABI change.
-
-What is an ABI
-~~~~~~~~~~~~~~
+#. Major ABI versions are declared every **year** and are then supported for one
+ year, typically aligned with the :ref:`LTS release <stable_lts_releases>`.
+#. The ABI version is managed at a project level in DPDK, with the ABI version
+ reflected in all :ref:`library's soname <what_is_soname>`.
+#. The ABI should be preserved and not changed lightly. ABI changes must follow
+ the outlined :ref:`deprecation process <abi_changes>`.
+#. The addition of symbols is generally not problematic. The modification of
+ symbols is managed with :ref:`ABI Versioning <abi_versioning>`.
+#. The removal of symbols is considered an :ref:`ABI breakage <abi_breakages>`,
+ once approved these will form part of the next ABI version.
+#. Libraries or APIs marked as :ref:`Experimental <experimental_apis>` are not
+ considered part of an ABI version and may change without constraint.
+#. Updates to the :ref:`minimum hardware requirements <hw_rqmts>`, which drop
+ support for hardware which was previously supported, should be treated as an
+ ABI change.
+
+.. note::
+
+ In 2019, the DPDK community stated it's intention to move to ABI stable
+ releases, over a number of release cycles. Beginning with maintaining ABI
+ stability through one year of DPDK releases starting from DPDK 19.11. This
+ policy will be reviewed in 2020, with intention of lengthening the stability
+ period.
+
+What is an ABI?
+~~~~~~~~~~~~~~~
An ABI (Application Binary Interface) is the set of runtime interfaces exposed
by a library. It is similar to an API (Application Programming Interface) but
@@ -39,30 +52,67 @@ Therefore, in the case of dynamic linking, it is critical that an ABI is
preserved, or (when modified), done in such a way that the application is unable
to behave improperly or in an unexpected fashion.
+What is an ABI version?
+~~~~~~~~~~~~~~~~~~~~~~~
-ABI/API Deprecation
--------------------
+An ABI version is an instance of a library's ABI at a specific release. Certain
+releases are considered by the community to be milestone releases, the yearly
+LTS for example. Supporting those milestone release's ABI for some number of
+subsequent releases is desirable to facilitate application upgrade. Those ABI
+version's aligned with milestones release are therefore called 'ABI major
+versions' and are supported for some number of releases.
+
+More details on major ABI version can be found in the :ref:`ABI versioning
+<major_abi_versions>` guide.
The DPDK ABI policy
-~~~~~~~~~~~~~~~~~~~
+-------------------
+
+A major ABI version is declared every year, aligned with that year's LTS
+release, e.g. v19.11. This ABI version is then supported for one year by all
+subsequent releases within that time period, until the next LTS release, e.g.
+v20.11.
+
+At the declaration of a major ABI version, major version numbers encoded in
+libraries soname's are bumped to indicate the new version, with the minor
+version reset to ``0``. An example would be ``librte_eal.so.20.3`` would become
+``librte_eal.so.21.0``.
+
+The ABI may then change multiple times, without warning, between the last major
+ABI version increment and the HEAD label of the git tree, with the condition
+that ABI compatibility with the major ABI version is preserved and therefore
+soname's do not change.
+
+Minor versions are incremented to indicate the release of a new ABI compatible
+DPDK release, typically the DPDK quarterly releases. An example of this, might
+be that ``librte_eal.so.20.1`` would indicate the first ABI compatible DPDK
+release, following the declaration of the new major ABI version ``20``.
+
+ABI versions, are supported by each release until such time as the next major
+ABI version is declared. At that time, the deprecation of the previous major ABI
+version will be noted in the Release Notes with guidance on individual symbol
+depreciation and upgrade notes provided.
-ABI versions are set at the time of major release labeling, and the ABI may
-change multiple times, without warning, between the last release label and the
-HEAD label of the git tree.
+.. _abi_changes:
-ABI versions, once released, are available until such time as their
-deprecation has been noted in the Release Notes for at least one major release
-cycle. For example consider the case where the ABI for DPDK 2.0 has been
-shipped and then a decision is made to modify it during the development of
-DPDK 2.1. The decision will be recorded in the Release Notes for the DPDK 2.1
-release and the modification will be made available in the DPDK 2.2 release.
+ABI Changes
+~~~~~~~~~~~
-ABI versions may be deprecated in whole or in part as needed by a given
-update.
+The ABI may still change after the declaration of a major ABI version, that is
+new APIs may be still added or existing APIs may be modified.
-Some ABI changes may be too significant to reasonably maintain multiple
-versions. In those cases ABI's may be updated without backward compatibility
-being provided. The requirements for doing so are:
+.. Warning::
+
+ Note that, the process for ABI deprecation should not be undertaken lightly.
+ ABI stability is extremely important for downstream consumers of the DPDK,
+ especially when distributed in shared object form. Every effort should be
+ made to preserve the ABI whenever possible. The ABI should only be changed
+ for significant reasons, such as performance enhancements. ABI breakage due
+ to changes such as reorganizing public structure fields for aesthetic or
+ readability purposes should be avoided.
+
+
+The requirements for changing the ABI are:
#. At least 3 acknowledgments of the need to do so must be made on the
dpdk.org mailing list.
@@ -71,34 +121,119 @@ being provided. The requirements for doing so are:
no maintainer is available for the component, the tree/sub-tree maintainer
for that component must acknowledge the ABI change instead.
+ - The acknowledgment of a member of the technical board, as a delegate of the
+ `technical board <https://core.dpdk.org/techboard/>`_ acknowledging the
+ need for the ABI change, is also mandatory.
+
- It is also recommended that acknowledgments from different "areas of
interest" be sought for each deprecation, for example: from NIC vendors,
CPU vendors, end-users, etc.
-#. The changes (including an alternative map file) can be included with
- deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option,
- to provide more details about oncoming changes.
- ``RTE_NEXT_ABI`` wrapper will be removed when it become the default ABI.
- More preferred way to provide this information is sending the feature
- as a separate patch and reference it in deprecation notice.
+#. Backward compatibly with the major ABI version must be maintained through
+ :ref:`abi_versioning`, with :ref:`forward-only <forward-only>` compatibility
+ offered for any ABI changes that are indicated to be part of the next ABI
+ version.
-#. A full deprecation cycle, as explained above, must be made to offer
- downstream consumers sufficient warning of the change.
+ - In situations were backward compatibility is not possible, read the
+ section on :ref:`abi_breakages`.
-Note that the above process for ABI deprecation should not be undertaken
-lightly. ABI stability is extremely important for downstream consumers of the
-DPDK, especially when distributed in shared object form. Every effort should
-be made to preserve the ABI whenever possible. The ABI should only be changed
-for significant reasons, such as performance enhancements. ABI breakage due to
-changes such as reorganizing public structure fields for aesthetic or
-readability purposes should be avoided.
+ - No backward or forward compatibility is offered for API changes marked as
+ ``experimental``, as described in the section on :ref:`Experimental APIs
+ and Libraries <experimental_apis>`.
-.. note::
+#. If a newly proposed API functionally replaces an existing one, when the new
+ API becomes non-experimental, then the old one is marked with
+ ``__rte_deprecated``.
+
+ - The depreciated API should follow the notification process to be removed,
+ see :ref:`deprecation_notices`.
+
+ - At the declaration of the next major ABI version, those ABI changes then
+ become a formal part of the new ABI and the requirement to preserve ABI
+ compatibility with the last major ABI version is then dropped.
+
+ - The responsibility for removing redundant ABI compatibility code rests
+ with the original contributor of the ABI changes, failing that, then with
+ the contributor's company and then finally with the maintainer.
+
+.. _forward-only:
+
+.. Note::
+
+ Note that forward-only compatibility is offered for those changes made
+ between major ABI versions. As a library's soname can only describe
+ compatibility with the last major ABI version, until the next major ABI
+ version is declared, these changes therefore cannot be resolved as a runtime
+ dependency through the soname. Therefore any application wishing to make use
+ of these ABI changes can only ensure that it's runtime dependencies are met
+ through Operating System package versioning.
+
+.. _hw_rqmts:
+
+.. Note::
Updates to the minimum hardware requirements, which drop support for hardware
which was previously supported, should be treated as an ABI change, and
- follow the relevant deprecation policy procedures as above: 3 acks and
- announcement at least one release in advance.
+ follow the relevant deprecation policy procedures as above: 3 acks, technical
+ board approval and announcement at least one release in advance.
+
+.. _abi_breakages:
+
+ABI Breakages
+~~~~~~~~~~~~~
+
+For those ABI changes that are too significant to reasonably maintain multiple
+symbol versions, there is an amended process. In these cases, ABIs may be
+updated without the requirement of backward compatibility being provided. These
+changes must follow the `same process :ref:`described above <abi_changes>` as non-breaking
+changes, however with the following additional requirements:
+
+#. ABI breaking changes (including an alternative map file) can be included with
+ deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option, to provide
+ more details about oncoming changes. ``RTE_NEXT_ABI`` wrapper will be removed
+ at the declaration of the next major ABI version.
+
+#. Once approved, and after the depreciation notice has been observed these
+ changes will form part of the next declared major ABI version.
+
+Examples of ABI Changes
+~~~~~~~~~~~~~~~~~~~~~~~
+
+The following are examples of allowable ABI changes occurring between
+declarations of major ABI versions.
+
+* DPDK 19.11 release, defines the function ``rte_foo()``, and ``rte_foo()``
+ as part of the major ABI version ``20``.
+
+* DPDK 20.02 release defines a new function ``rte_foo(uint8_t bar)``, and
+ this is not a problem as long as the symbol ``rte_foo@DPDK20`` is
+ preserved through :ref:`abi_versioning`.
+
+ - The new function may be marked with the ``__rte_experimental`` tag for a
+ number of releases, as described in the section :ref:`experimental_apis`.
+
+ - Once ``rte_foo(uint8_t bar)`` becomes non-experimental ``rte_foo()`` is then
+ declared as ``__rte_depreciated``, with an associated deprecation notice
+ provided.
+
+* DPDK 19.11 is not re-released to include ``rte_foo(uint8_t bar)``, the new
+ version of ``rte_foo`` only exists from DPDK 20.02 onwards as described in the
+ :ref:`note on forward-only compatibility<forward-only>`.
+
+* DPDK 20.02 release defines the experimental function ``__rte_experimental
+ rte_baz()``. This function may or may not exist in the DPDK 20.05 release.
+
+* An application ``dPacket`` wishes to use ``rte_foo(uint8_t bar)``, before the
+ declaration of the DPDK ``21`` major API version. The application can only
+ ensure it's runtime dependencies are met by specifying ``DPDK (>= 20.2)`` as
+ an explicit package dependency, as the soname only may only indicate the
+ supported major ABI version.
+
+* At the release of DPDK 20.11, the function ``rte_foo(uint8_t bar)`` becomes
+ formally part of then new major ABI version DPDK 21.0 and ``rte_foo()`` may be
+ removed.
+
+.. _deprecation_notices:
Examples of Deprecation Notices
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -106,46 +241,42 @@ Examples of Deprecation Notices
The following are some examples of ABI deprecation notices which would be
added to the Release Notes:
-* The Macro ``#RTE_FOO`` is deprecated and will be removed with version 2.0,
- to be replaced with the inline function ``rte_foo()``.
+* The Macro ``#RTE_FOO`` is deprecated and will be removed with ABI version
+ 21, to be replaced with the inline function ``rte_foo()``.
* The function ``rte_mbuf_grok()`` has been updated to include a new parameter
- in version 2.0. Backwards compatibility will be maintained for this function
- until the release of version 2.1
+ in version 20.2. Backwards compatibility will be maintained for this function
+ until the release of the new DPDK major ABI version 21, in DPDK version
+ 20.11.
-* The members of ``struct rte_foo`` have been reorganized in release 2.0 for
+* The members of ``struct rte_foo`` have been reorganized in DPDK 20.02 for
performance reasons. Existing binary applications will have backwards
- compatibility in release 2.0, while newly built binaries will need to
- reference the new structure variant ``struct rte_foo2``. Compatibility will
- be removed in release 2.2, and all applications will require updating and
+ compatibility in release 20.02, while newly built binaries will need to
+ reference the new structure variant ``struct rte_foo2``. Compatibility will be
+ removed in release 20.11, and all applications will require updating and
rebuilding to the new structure at that time, which will be renamed to the
original ``struct rte_foo``.
* Significant ABI changes are planned for the ``librte_dostuff`` library. The
- upcoming release 2.0 will not contain these changes, but release 2.1 will,
+ upcoming release 20.02 will not contain these changes, but release 20.11 will,
and no backwards compatibility is planned due to the extensive nature of
- these changes. Binaries using this library built prior to version 2.1 will
+ these changes. Binaries using this library built prior to ABI version 21 will
require updating and recompilation.
-New API replacing previous one
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-If a new API proposed functionally replaces an existing one, when the new API
-becomes non-experimental then the old one is marked with ``__rte_deprecated``.
-Deprecated APIs are removed completely just after the next LTS.
-
-Reminder that old API should follow deprecation process to be removed.
+.. _experimental_apis:
+Experimental
+------------
-Experimental APIs
------------------
+APIs
+~~~~
-APIs marked as ``experimental`` are not considered part of the ABI and may
-change without warning at any time. Since changes to APIs are most likely
-immediately after their introduction, as users begin to take advantage of
-those new APIs and start finding issues with them, new DPDK APIs will be
-automatically marked as ``experimental`` to allow for a period of stabilization
-before they become part of a tracked ABI.
+APIs marked as ``experimental`` are not considered part of an ABI version and
+may change without warning at any time. Since changes to APIs are most likely
+immediately after their introduction, as users begin to take advantage of those
+new APIs and start finding issues with them, new DPDK APIs will be automatically
+marked as ``experimental`` to allow for a period of stabilization before they
+become part of a tracked ABI version.
Note that marking an API as experimental is a multi step process.
To mark an API as experimental, the symbols which are desired to be exported
@@ -163,7 +294,16 @@ In addition to tagging the code with ``__rte_experimental``,
the doxygen markup must also contain the EXPERIMENTAL string,
and the MAINTAINERS file should note the EXPERIMENTAL libraries.
-For removing the experimental tag associated with an API, deprecation notice
-is not required. Though, an API should remain in experimental state for at least
-one release. Thereafter, normal process of posting patch for review to mailing
-list can be followed.
+For removing the experimental tag associated with an API, deprecation notice is
+not required. Though, an API should remain in experimental state for at least
+one release. Thereafter, the normal process of posting patch for review to
+mailing list can be followed.
+
+Libraries
+~~~~~~~~~
+
+Libraries marked as ``experimental`` are entirely not considered part of an ABI
+version, and may change without warning at any time. Experimental libraries
+always have a major version of ``0`` to indicate they exist outside of
+:ref:`abi_versioning` , with the minor version incremented with each ABI change
+to library.
diff --git a/doc/guides/contributing/stable.rst b/doc/guides/contributing/stable.rst
index 6a5eee9..d95c200 100644
--- a/doc/guides/contributing/stable.rst
+++ b/doc/guides/contributing/stable.rst
@@ -1,7 +1,7 @@
.. SPDX-License-Identifier: BSD-3-Clause
Copyright 2018 The DPDK contributors
-.. stable_lts_releases:
+.. _stable_lts_releases:
DPDK Stable Releases and Long Term Support
==========================================
@@ -53,6 +53,9 @@ year's November (X.11) release will be maintained as an LTS for 2 years.
After the X.11 release, an LTS branch will be created for it at
http://git.dpdk.org/dpdk-stable where bugfixes will be backported to.
+A LTS release may align with the declaration of a new major ABI version,
+please read the :ref:`abi_policy` for more information.
+
It is anticipated that there will be at least 4 releases per year of the LTS
or approximately 1 every 3 months. However, the cadence can be shorter or
longer depending on the number and criticality of the backported
@@ -68,10 +71,13 @@ point the LTS branch will no longer be maintained with no further releases.
What changes should be backported
---------------------------------
-Backporting should be limited to bug fixes. All patches accepted on the master
-branch with a Fixes: tag should be backported to the relevant stable/LTS
-branches, unless the submitter indicates otherwise. If there are exceptions,
-they will be discussed on the mailing lists.
+Backporting is a naturally conservative activity, and therefore should only
+include bug fixes and support for new hardware, were adding support does not
+necessitate DPDK ABI/API changes.
+
+All patches accepted on the master branch with a Fixes: tag should be backported
+to the relevant stable/LTS branches, unless the submitter indicates otherwise.
+If there are exceptions, they will be discussed on the mailing lists.
Fixes suitable for backport should have a ``Cc: stable@dpdk.org`` tag in the
commit message body as follows::
@@ -86,13 +92,18 @@ commit message body as follows::
Signed-off-by: Alex Smith <alex.smith@example.com>
-Fixes not suitable for backport should not include the ``Cc: stable@dpdk.org`` tag.
+Fixes not suitable for backport should not include the ``Cc: stable@dpdk.org``
+tag.
-Features should not be backported to stable releases. It may be acceptable, in
-limited cases, to back port features for the LTS release where:
+New features, with the exception of new hardware support, should not be
+backported to stable releases. In the case of new hardware support or any other
+exceptional circumstances limited backporting maybe permitted to the LTS release
+where:
-* There is a justifiable use case (for example a new PMD).
-* The change is non-invasive.
+* There is a justifiable use case, for example the change is required to support
+ a new platform or device (for example a new PMD).
+* The change is ABI/API preserving, it does not present an obvious "new feature"
+ to end consumer.
* The work of preparing the backport is done by the proposer.
* There is support within the community.
@@ -119,10 +130,3 @@ A Stable Release will be released by:
list.
Stable releases are available on the `dpdk.org download page <http://core.dpdk.org/download/>`_.
-
-
-ABI
----
-
-The Stable Release should not be seen as a way of breaking or circumventing
-the DPDK ABI policy.
--
2.7.4
^ permalink raw reply [relevance 31%]
* [dpdk-dev] [PATCH v3 0/4] doc: changes to abi policy introducing major abi versions
@ 2019-08-15 10:23 11% Ray Kinsella
2019-08-15 10:23 13% ` [dpdk-dev] [PATCH v3 1/4] doc: separate versioning.rst into version and policy Ray Kinsella
` (3 more replies)
0 siblings, 4 replies; 200+ results
From: Ray Kinsella @ 2019-08-15 10:23 UTC (permalink / raw)
To: dev
Cc: mdr, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, nhorman,
maxime.coquelin, john.mcnamara, marko.kovacevic, hemant.agrawal
TL;DR abbreviation:
A major ABI version that all DPDK releases during a one year period
support. ABI versioning is managed at a project-level, in place of library-level
management. ABI changes to add new features are permitted, as long as ABI
compatibility with the major ABI version is maintained.
Detail:
This patch introduces major ABI versions, supported for one year and released
aligned with the LTS release. This ABI version is then supported by all
subsequent releases within that one year period. The intention is that the one
year support period, will then be reviewed after the initial year with the
intention of lengthing the support period for the next ABI version.
ABI changes that preserve ABI compatibility with the major ABI version are
permitted in subsequent releases. ABI changes, follow similar approval rules as
before with the additional gate of now requiring technical board approval. The
merging and release of ABI breaking changes would now be pushed to the
declaration of the next major ABI version.
This change encourages developers to maintain ABI compatibility with the major
ABI version, by promoting a permissive culture around those changes that
preserve ABI compatibility. This approach begins to align DPDK with those
projects that declare major ABI versions (e.g. version 2.x, 3.x) and support
those versions for some period, typically two years or more.
To provide an example of how this might work in practice:
* DPDK v20 is declared as the supported ABI version for one year, aligned with
the DPDK v19.11 (LTS) release. All library sonames are updated to reflect the
new ABI version, e.g. librte_eal.so.20, librte_acl.so.20...
* DPDK v20.02 .. v20.08 releases are ABI compatible with the DPDK v20 ABI. ABI
changes are permitted from DPDK v20.02 onwards, with the condition that ABI
compatibility with DPDK v20 is preserved.
* DPDK v21 is declared as the new supported ABI version for two years, aligned
with the DPDK v20.11 (LTS) release. The DPDK v20 ABI is now depreciated,
library sonames are updated to v21 and ABI compatibility breaking changes may
be introduced.
Ray Kinsella (4):
doc: separate versioning.rst into version and policy
doc: changes to abi policy introducing major abi versions
doc: updates to versioning guide for abi versions
doc: add maintainer for abi policy
MAINTAINERS | 4 +
doc/guides/contributing/abi_policy.rst | 309 +++++++++++++++
doc/guides/contributing/abi_versioning.rst | 515 +++++++++++++++++++++++++
doc/guides/contributing/index.rst | 3 +-
doc/guides/contributing/patches.rst | 6 +-
doc/guides/contributing/stable.rst | 38 +-
doc/guides/contributing/versioning.rst | 591 -----------------------------
doc/guides/rel_notes/deprecation.rst | 2 +-
8 files changed, 855 insertions(+), 613 deletions(-)
create mode 100644 doc/guides/contributing/abi_policy.rst
create mode 100644 doc/guides/contributing/abi_versioning.rst
delete mode 100644 doc/guides/contributing/versioning.rst
--
2.7.4
^ permalink raw reply [relevance 11%]
* Re: [dpdk-dev] [PATCH v2] ethdev: add more protocol support in flow API
@ 2019-08-14 9:08 4% ` Adrien Mazarguil
2019-08-19 11:53 0% ` Zhang, Qi Z
0 siblings, 1 reply; 200+ results
From: Adrien Mazarguil @ 2019-08-14 9:08 UTC (permalink / raw)
To: Wang Ying A; +Cc: qi.z.zhang, xiaolong.ye, qiming.yang, dev
Hi Wang Ying,
On Wed, Aug 14, 2019 at 11:24:30AM +0800, Wang Ying A wrote:
> Add new protocol header match support as below
>
> RTE_FLOW_ITEM_TYPE_GTP_PSC
> - matches a GTP PDU extension header (type is 0x85:
> PDU Session Container)
> RTE_FLOW_ITEM_TYPE_PPPOES
> - matches a PPPoE Session header.
> RTE_FLOW_ITEM_TYPE_PPPOED
> - matches a PPPoE Discovery stage header.
>
> Signed-off-by: Wang Ying A <ying.a.wang@intel.com>
OK, please split this patch, one for GTP and the other for PPPoE to make
title less vague than "add more protocol support".
For PPPoE, the distinction between session and discovery is not a bad idea
but since discovery packets typically have a different format (tags in place
of protocol), I'm not sure they should share a common structure.
How about a single "PPPOE" item without proto_id to cover both session and
discovery, then later add separate tag items on a needed basis for each
possible/optional tag (e.g. PPPOE_TAG_SERVICE_NAME)? Likewise proto_id would
be provided through a separate optional item if relevant (PPPOE_PROTO_ID).
Such an approach is already used for IPV6 and IPV6_EXT.
Another benefit is that applications could match PPPoE regardless of session
or discovery when they do not care, while PPPOED/PPPOES make that
distinction mandatory.
Also by inserting new entries in the middle of the pattern items list, this
patch breaks ABI. I think it's not on purpose, so please move them to the
end (not grouping them with existing GTP stuff is fine, ABI compat is more
important.) This must be reflected in rte_flow.h, rte_flow.c, testpmd and
documentation.
More nits below.
> ---
> ---
> v2: Remove Gerrit Change-Id's.
> ---
> app/test-pmd/cmdline_flow.c | 80 +++++++++++++++++++++++++++++
> doc/guides/prog_guide/rte_flow.rst | 25 +++++++++
> doc/guides/testpmd_app_ug/testpmd_funcs.rst | 10 ++++
> lib/librte_ethdev/rte_flow.c | 3 ++
> lib/librte_ethdev/rte_flow.h | 71 +++++++++++++++++++++++++
> 5 files changed, 189 insertions(+)
>
> diff --git a/app/test-pmd/cmdline_flow.c b/app/test-pmd/cmdline_flow.c
[...]
> + [ITEM_PPPOES] = {
> + .name = "pppoes",
> + .help = "match PPPoE Session header",
Session => session
> + .priv = PRIV_ITEM(PPPOES, sizeof(struct rte_flow_item_pppoe)),
> + .next = NEXT(item_pppoe),
> + .call = parse_vc,
> + },
> + [ITEM_PPPOED] = {
> + .name = "pppoed",
> + .help = "match PPPoE Discovery stage header",
Discovery => discovery
> + .priv = PRIV_ITEM(PPPOED, sizeof(struct rte_flow_item_pppoe)),
> + .next = NEXT(item_pppoe),
> + .call = parse_vc,
> + },
> + [ITEM_PPPOE_SEID] = {
> + .name = "seid",
> + .help = "Session identifier",
Session => session
[...]
> diff --git a/doc/guides/prog_guide/rte_flow.rst b/doc/guides/prog_guide/rte_flow.rst
> index 821b524b3..d09c42071 100644
> --- a/doc/guides/prog_guide/rte_flow.rst
> +++ b/doc/guides/prog_guide/rte_flow.rst
> @@ -1055,6 +1055,31 @@ flow rules.
> - ``teid``: tunnel endpoint identifier.
> - Default ``mask`` matches teid only.
>
> +Item: ``GTP_PSC``
> +^^^^^^^^^^^^^^^^^^^^^^^
Too many "^^^"'s.
[...]
> +Item: ``PPPOES``, ``PPPOED``
> +^^^^^^^^^^^^^^^^^^^^^^^^^^^^
> +
> +Matches a PPPOE header.
PPPOE => PPPoE
> +
> +Note: PPPOES and PPPOED use the same structure. PPPOES and PPPOED item
item => items (or better, "pattern items")
> +are defined for a user-friendly API when creating PPPOE-Session and
> +PPPOE-Discovery flow rules.
Super nit: use "PPPoE" when mentioning the protocol itself and "PPPOE" when
mentioning the pattern item.
> +
> +- ``v_t_flags``: version (4b), type (4b).
Why "flags"? I don't see any so you could name it "version_type" (same in
documentation).
> +- ``code``: Message type.
Message => message
> +- ``session_id``: Session identifier.
Session => session
> +- ``length``: Payload length.
Payload => payload
> +- ``proto_id``: PPP Protocol identifier.
Protocol => protocol
> +- Default ``mask`` matches session_id,proto_id.
Missing space between session_id and proto_id.
> +
> Item: ``ESP``
> ^^^^^^^^^^^^^
>
> diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> index 313e0707e..0da36d5f1 100644
> --- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> +++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> @@ -3904,6 +3904,16 @@ This section lists supported pattern items and their attributes, if any.
>
> - ``teid {unsigned}``: tunnel endpoint identifier.
>
> +- ``gtp_psc``: match GTPv1 entension header (type is 0x85).
> +
> + - ``pdu_type {unsigned}``: PDU type (0 or 1).
> + - ``qfi {unsigned}``: QoS flow identifier.
> +
> +- ``pppoes``, ``pppoed``: match PPPOE header.
PPPOE => PPPoE
[...]
> diff --git a/lib/librte_ethdev/rte_flow.h b/lib/librte_ethdev/rte_flow.h
> index b66bf1495..ad5e46190 100644
> --- a/lib/librte_ethdev/rte_flow.h
> +++ b/lib/librte_ethdev/rte_flow.h
> @@ -328,6 +328,34 @@ enum rte_flow_item_type {
> */
> RTE_FLOW_ITEM_TYPE_GTPU,
>
> + /**
> + * Matches a GTP PDU extension header (type is 0x85:
> + * PDU Session Container).
Session Container => session container
> + *
> + * Configure flow for GTP packets with extension header type 0x85.
> + *
> + * See struct rte_flow_item_gtp_psc.
> + */
> + RTE_FLOW_ITEM_TYPE_GTP_PSC,
> +
> + /**
> + * Matches a PPPOE header.
> + *
> + * Configure flow for PPPoE Session packets.
Session => session
> + *
> + * See struct rte_flow_item_pppoe.
> + */
> + RTE_FLOW_ITEM_TYPE_PPPOES,
> +
> + /**
> + * Matches a PPPOE header.
> + *
> + * Configure flow for PPPoE Discovery stage packets.
Discovery => discovery
> + *
> + * See struct rte_flow_item_pppoe.
> + */
> + RTE_FLOW_ITEM_TYPE_PPPOED,
> +
> /**
> * Matches a ESP header.
> *
> @@ -922,6 +950,49 @@ static const struct rte_flow_item_gtp rte_flow_item_gtp_mask = {
> };
> #endif
>
> +/**
> + * RTE_FLOW_ITEM_TYPE_GTP_PSC.
> + *
> + * Matches a GTP-extension header
> + * (type is 0x85: PDU Session Container).
Session Container => session container
(crusade against superfluous caps!)
[...]
> +/**
> + * RTE_FLOW_ITEM_TYPE_PPPOE.
> + *
> + * Matches a PPPOE header.
> + */
> +struct rte_flow_item_pppoe {
> + /**
> + * Version (4b), type (4b).
> + */
> + uint8_t v_t_flags;
v_t_flags => version_type
> + uint8_t code; /**< Message type. */
> + rte_be16_t session_id; /**< Session identifier. */
> + rte_be16_t length; /**< Payload length. */
> + rte_be16_t proto_id; /**< PPP Protocol identifier. */
As discussed, I suggest dropping proto_id to make this a generic match for
PPPoE.
> +};
> +
> +/** Default mask for RTE_FLOW_ITEM_TYPE_PPPOE. */
> +#ifndef __cplusplus
> +static const struct rte_flow_item_pppoe rte_flow_item_pppoe_mask = {
> + .session_id = RTE_BE16(0xffff),
> + .proto_id = RTE_BE16(0xffff),
> +};
I think the default for PPPoE should be an empty mask that simply says
"match PPPoE" since session_id only becomes known after negotiation and
proto_id shouldn't be part of this object.
--
Adrien Mazarguil
6WIND
^ permalink raw reply [relevance 4%]
* [dpdk-dev] [PATCH v9 1/3] eal/arm64: add 128-bit atomic compare exchange
@ 2019-08-14 8:27 2% ` Phil Yang
0 siblings, 0 replies; 200+ results
From: Phil Yang @ 2019-08-14 8:27 UTC (permalink / raw)
To: thomas, jerinj, gage.eads, dev
Cc: hemant.agrawal, Honnappa.Nagarahalli, gavin.hu, nd
Add 128-bit atomic compare exchange on aarch64.
Suggested-by: Jerin Jacob <jerinj@marvell.com>
Signed-off-by: Phil Yang <phil.yang@arm.com>
Reviewed-by: Honnappa Nagarahalli <honnappa.nagarahalli@arm.com>
Tested-by: Honnappa Nagarahalli <honnappa.nagarahalli@arm.com>
Acked-by: Jerin Jacob <jerinj@marvell.com>
---
v9:
Updated 19.11 release note.
v8:
Fixed "WARNING:LONG_LINE: line over 80 characters" warnings with latest kernel
checkpatch.pl
v7:
1. Adjust code comment.
v6:
1. Put the RTE_ARM_FEATURE_ATOMICS flag into EAL group. (Jerin Jocob)
2. Keep rte_stack_lf_stubs.h doing nothing. (Gage Eads)
3. Fixed 32 bit build issue.
v5:
1. Enable RTE_ARM_FEATURE_ATOMICS on octeontx2 in default. (Jerin Jocob)
2. Record the reason of introducing "rte_stack_lf_stubs.h" in git
commit.
(Jerin, Jocob)
3. Fixed a conditional MACRO error in rte_atomic128_cmp_exchange. (Jerin
Jocob)
v4:
1. Add RTE_ARM_FEATURE_ATOMICS flag to support LSE CASP instructions.
(Jerin Jocob)
2. Fix possible arm64 ABI break by making casp_op_name noinline. (Jerin
Jocob)
3. Add rte_stack_lf_stubs.h to reduce the ifdef clutter. (Gage
Eads/Jerin Jocob)
v3:
1. Avoid duplication code with macro. (Jerin Jocob)
2. Make invalid memory order to strongest barrier. (Jerin Jocob)
3. Update doc/guides/prog_guide/env_abstraction_layer.rst. (Gage Eads)
4. Fix 32-bit x86 builds issue. (Gage Eads)
5. Correct documentation issues in UT. (Gage Eads)
v2:
Initial version.
config/arm/meson.build | 2 +
config/common_base | 3 +
config/defconfig_arm64-octeontx2-linuxapp-gcc | 1 +
config/defconfig_arm64-thunderx2-linuxapp-gcc | 1 +
.../common/include/arch/arm/rte_atomic_64.h | 163 +++++++++++++++++++++
.../common/include/arch/x86/rte_atomic_64.h | 12 --
lib/librte_eal/common/include/generic/rte_atomic.h | 17 ++-
7 files changed, 186 insertions(+), 13 deletions(-)
diff --git a/config/arm/meson.build b/config/arm/meson.build
index 979018e..9f28271 100644
--- a/config/arm/meson.build
+++ b/config/arm/meson.build
@@ -71,11 +71,13 @@ flags_thunderx2_extra = [
['RTE_CACHE_LINE_SIZE', 64],
['RTE_MAX_NUMA_NODES', 2],
['RTE_MAX_LCORE', 256],
+ ['RTE_ARM_FEATURE_ATOMICS', true],
['RTE_USE_C11_MEM_MODEL', true]]
flags_octeontx2_extra = [
['RTE_MACHINE', '"octeontx2"'],
['RTE_MAX_NUMA_NODES', 1],
['RTE_MAX_LCORE', 24],
+ ['RTE_ARM_FEATURE_ATOMICS', true],
['RTE_EAL_IGB_UIO', false],
['RTE_USE_C11_MEM_MODEL', true]]
diff --git a/config/common_base b/config/common_base
index 8ef75c2..2054480 100644
--- a/config/common_base
+++ b/config/common_base
@@ -82,6 +82,9 @@ CONFIG_RTE_MAX_LCORE=128
CONFIG_RTE_MAX_NUMA_NODES=8
CONFIG_RTE_MAX_HEAPS=32
CONFIG_RTE_MAX_MEMSEG_LISTS=64
+
+# Use ARM LSE ATOMIC instructions
+CONFIG_RTE_ARM_FEATURE_ATOMICS=n
# each memseg list will be limited to either RTE_MAX_MEMSEG_PER_LIST pages
# or RTE_MAX_MEM_MB_PER_LIST megabytes worth of memory, whichever is smaller
CONFIG_RTE_MAX_MEMSEG_PER_LIST=8192
diff --git a/config/defconfig_arm64-octeontx2-linuxapp-gcc b/config/defconfig_arm64-octeontx2-linuxapp-gcc
index f20da24..7687dbe 100644
--- a/config/defconfig_arm64-octeontx2-linuxapp-gcc
+++ b/config/defconfig_arm64-octeontx2-linuxapp-gcc
@@ -9,6 +9,7 @@ CONFIG_RTE_MACHINE="octeontx2"
CONFIG_RTE_CACHE_LINE_SIZE=128
CONFIG_RTE_MAX_NUMA_NODES=1
CONFIG_RTE_MAX_LCORE=24
+CONFIG_RTE_ARM_FEATURE_ATOMICS=y
# Doesn't support NUMA
CONFIG_RTE_EAL_NUMA_AWARE_HUGEPAGES=n
diff --git a/config/defconfig_arm64-thunderx2-linuxapp-gcc b/config/defconfig_arm64-thunderx2-linuxapp-gcc
index cc5c64b..af4a89c 100644
--- a/config/defconfig_arm64-thunderx2-linuxapp-gcc
+++ b/config/defconfig_arm64-thunderx2-linuxapp-gcc
@@ -9,3 +9,4 @@ CONFIG_RTE_MACHINE="thunderx2"
CONFIG_RTE_CACHE_LINE_SIZE=64
CONFIG_RTE_MAX_NUMA_NODES=2
CONFIG_RTE_MAX_LCORE=256
+CONFIG_RTE_ARM_FEATURE_ATOMICS=y
diff --git a/lib/librte_eal/common/include/arch/arm/rte_atomic_64.h b/lib/librte_eal/common/include/arch/arm/rte_atomic_64.h
index 97060e4..14d869b 100644
--- a/lib/librte_eal/common/include/arch/arm/rte_atomic_64.h
+++ b/lib/librte_eal/common/include/arch/arm/rte_atomic_64.h
@@ -1,5 +1,6 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2015 Cavium, Inc
+ * Copyright(c) 2019 Arm Limited
*/
#ifndef _RTE_ATOMIC_ARM64_H_
@@ -14,6 +15,9 @@ extern "C" {
#endif
#include "generic/rte_atomic.h"
+#include <rte_branch_prediction.h>
+#include <rte_compat.h>
+#include <rte_debug.h>
#define dsb(opt) asm volatile("dsb " #opt : : : "memory")
#define dmb(opt) asm volatile("dmb " #opt : : : "memory")
@@ -40,6 +44,165 @@ extern "C" {
#define rte_cio_rmb() dmb(oshld)
+/*------------------------ 128 bit atomic operations -------------------------*/
+
+#define __HAS_ACQ(mo) ((mo) != __ATOMIC_RELAXED && (mo) != __ATOMIC_RELEASE)
+#define __HAS_RLS(mo) ((mo) == __ATOMIC_RELEASE || (mo) == __ATOMIC_ACQ_REL || \
+ (mo) == __ATOMIC_SEQ_CST)
+
+#define __MO_LOAD(mo) (__HAS_ACQ((mo)) ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED)
+#define __MO_STORE(mo) (__HAS_RLS((mo)) ? __ATOMIC_RELEASE : __ATOMIC_RELAXED)
+
+#if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS)
+#define __ATOMIC128_CAS_OP(cas_op_name, op_string) \
+static __rte_noinline rte_int128_t \
+cas_op_name(rte_int128_t *dst, rte_int128_t old, \
+ rte_int128_t updated) \
+{ \
+ /* caspX instructions register pair must start from even-numbered
+ * register at operand 1.
+ * So, specify registers for local variables here.
+ */ \
+ register uint64_t x0 __asm("x0") = (uint64_t)old.val[0]; \
+ register uint64_t x1 __asm("x1") = (uint64_t)old.val[1]; \
+ register uint64_t x2 __asm("x2") = (uint64_t)updated.val[0]; \
+ register uint64_t x3 __asm("x3") = (uint64_t)updated.val[1]; \
+ asm volatile( \
+ op_string " %[old0], %[old1], %[upd0], %[upd1], [%[dst]]" \
+ : [old0] "+r" (x0), \
+ [old1] "+r" (x1) \
+ : [upd0] "r" (x2), \
+ [upd1] "r" (x3), \
+ [dst] "r" (dst) \
+ : "memory"); \
+ old.val[0] = x0; \
+ old.val[1] = x1; \
+ return old; \
+}
+
+__ATOMIC128_CAS_OP(__rte_cas_relaxed, "casp")
+__ATOMIC128_CAS_OP(__rte_cas_acquire, "caspa")
+__ATOMIC128_CAS_OP(__rte_cas_release, "caspl")
+__ATOMIC128_CAS_OP(__rte_cas_acq_rel, "caspal")
+#else
+#define __ATOMIC128_LDX_OP(ldx_op_name, op_string) \
+static inline rte_int128_t \
+ldx_op_name(const rte_int128_t *src) \
+{ \
+ rte_int128_t ret; \
+ asm volatile( \
+ op_string " %0, %1, %2" \
+ : "=&r" (ret.val[0]), \
+ "=&r" (ret.val[1]) \
+ : "Q" (src->val[0]) \
+ : "memory"); \
+ return ret; \
+}
+
+__ATOMIC128_LDX_OP(__rte_ldx_relaxed, "ldxp")
+__ATOMIC128_LDX_OP(__rte_ldx_acquire, "ldaxp")
+
+#define __ATOMIC128_STX_OP(stx_op_name, op_string) \
+static inline uint32_t \
+stx_op_name(rte_int128_t *dst, const rte_int128_t src) \
+{ \
+ uint32_t ret; \
+ asm volatile( \
+ op_string " %w0, %1, %2, %3" \
+ : "=&r" (ret) \
+ : "r" (src.val[0]), \
+ "r" (src.val[1]), \
+ "Q" (dst->val[0]) \
+ : "memory"); \
+ /* Return 0 on success, 1 on failure */ \
+ return ret; \
+}
+
+__ATOMIC128_STX_OP(__rte_stx_relaxed, "stxp")
+__ATOMIC128_STX_OP(__rte_stx_release, "stlxp")
+#endif
+
+static inline int __rte_experimental
+rte_atomic128_cmp_exchange(rte_int128_t *dst,
+ rte_int128_t *exp,
+ const rte_int128_t *src,
+ unsigned int weak,
+ int success,
+ int failure)
+{
+ /* Always do strong CAS */
+ RTE_SET_USED(weak);
+ /* Ignore memory ordering for failure, memory order for
+ * success must be stronger or equal
+ */
+ RTE_SET_USED(failure);
+ /* Find invalid memory order */
+ RTE_ASSERT(success == __ATOMIC_RELAXED
+ || success == __ATOMIC_ACQUIRE
+ || success == __ATOMIC_RELEASE
+ || success == __ATOMIC_ACQ_REL
+ || success == __ATOMIC_SEQ_CST);
+
+#if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS)
+ rte_int128_t expected = *exp;
+ rte_int128_t desired = *src;
+ rte_int128_t old;
+
+ if (success == __ATOMIC_RELAXED)
+ old = __rte_cas_relaxed(dst, expected, desired);
+ else if (success == __ATOMIC_ACQUIRE)
+ old = __rte_cas_acquire(dst, expected, desired);
+ else if (success == __ATOMIC_RELEASE)
+ old = __rte_cas_release(dst, expected, desired);
+ else
+ old = __rte_cas_acq_rel(dst, expected, desired);
+#else
+ int ldx_mo = __MO_LOAD(success);
+ int stx_mo = __MO_STORE(success);
+ uint32_t ret = 1;
+ register rte_int128_t expected = *exp;
+ register rte_int128_t desired = *src;
+ register rte_int128_t old;
+
+ /* ldx128 can not guarantee atomic,
+ * Must write back src or old to verify atomicity of ldx128;
+ */
+ do {
+ if (ldx_mo == __ATOMIC_RELAXED)
+ old = __rte_ldx_relaxed(dst);
+ else
+ old = __rte_ldx_acquire(dst);
+
+ if (likely(old.int128 == expected.int128)) {
+ if (stx_mo == __ATOMIC_RELAXED)
+ ret = __rte_stx_relaxed(dst, desired);
+ else
+ ret = __rte_stx_release(dst, desired);
+ } else {
+ /* In the failure case (since 'weak' is ignored and only
+ * weak == 0 is implemented), expected should contain
+ * the atomically read value of dst. This means, 'old'
+ * needs to be stored back to ensure it was read
+ * atomically.
+ */
+ if (stx_mo == __ATOMIC_RELAXED)
+ ret = __rte_stx_relaxed(dst, old);
+ else
+ ret = __rte_stx_release(dst, old);
+ }
+ } while (unlikely(ret));
+#endif
+
+ /* Unconditionally updating expected removes
+ * an 'if' statement.
+ * expected should already be in register if
+ * not in the cache.
+ */
+ *exp = old;
+
+ return (old.int128 == expected.int128);
+}
+
#ifdef __cplusplus
}
#endif
diff --git a/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h b/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h
index 1335d92..cfe7067 100644
--- a/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h
+++ b/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h
@@ -183,18 +183,6 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
/*------------------------ 128 bit atomic operations -------------------------*/
-/**
- * 128-bit integer structure.
- */
-RTE_STD_C11
-typedef struct {
- RTE_STD_C11
- union {
- uint64_t val[2];
- __extension__ __int128 int128;
- };
-} __rte_aligned(16) rte_int128_t;
-
__rte_experimental
static inline int
rte_atomic128_cmp_exchange(rte_int128_t *dst,
diff --git a/lib/librte_eal/common/include/generic/rte_atomic.h b/lib/librte_eal/common/include/generic/rte_atomic.h
index 24ff7dc..e6ab15a 100644
--- a/lib/librte_eal/common/include/generic/rte_atomic.h
+++ b/lib/librte_eal/common/include/generic/rte_atomic.h
@@ -1081,6 +1081,20 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
/*------------------------ 128 bit atomic operations -------------------------*/
+/**
+ * 128-bit integer structure.
+ */
+RTE_STD_C11
+typedef struct {
+ RTE_STD_C11
+ union {
+ uint64_t val[2];
+#ifdef RTE_ARCH_64
+ __extension__ __int128 int128;
+#endif
+ };
+} __rte_aligned(16) rte_int128_t;
+
#ifdef __DOXYGEN__
/**
@@ -1093,7 +1107,8 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
* *exp = *dst
* @endcode
*
- * @note This function is currently only available for the x86-64 platform.
+ * @note This function is currently available for the x86-64 and aarch64
+ * platforms.
*
* @note The success and failure arguments must be one of the __ATOMIC_* values
* defined in the C++11 standard. For details on their behavior, refer to the
--
2.7.4
^ permalink raw reply [relevance 2%]
* [dpdk-dev] [PATCH v2] version: 19.11-rc0
2019-08-12 11:43 6% [dpdk-dev] [PATCH] version: 19.11-rc0 David Marchand
@ 2019-08-13 12:18 6% ` David Marchand
0 siblings, 0 replies; 200+ results
From: David Marchand @ 2019-08-13 12:18 UTC (permalink / raw)
To: dev; +Cc: thomas
Start a new release cycle with empty release notes.
Signed-off-by: David Marchand <david.marchand@redhat.com>
---
Changelog since v1:
- added missing reference in index,
- bumped the libraries version to 19.08,
- included isonum,
---
VERSION | 2 +-
doc/guides/rel_notes/index.rst | 1 +
doc/guides/rel_notes/release_19_11.rst | 216 +++++++++++++++++++++++++++++++++
3 files changed, 218 insertions(+), 1 deletion(-)
create mode 100644 doc/guides/rel_notes/release_19_11.rst
diff --git a/VERSION b/VERSION
index 909cfd6..fff18fc 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-19.08.0
+19.11.0-rc0
diff --git a/doc/guides/rel_notes/index.rst b/doc/guides/rel_notes/index.rst
index adfaf12..26f4a97 100644
--- a/doc/guides/rel_notes/index.rst
+++ b/doc/guides/rel_notes/index.rst
@@ -8,6 +8,7 @@ Release Notes
:maxdepth: 1
:numbered:
+ release_19_11
release_19_08
release_19_05
release_19_02
diff --git a/doc/guides/rel_notes/release_19_11.rst b/doc/guides/rel_notes/release_19_11.rst
new file mode 100644
index 0000000..8490d89
--- /dev/null
+++ b/doc/guides/rel_notes/release_19_11.rst
@@ -0,0 +1,216 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+ Copyright 2019 The DPDK contributors
+
+.. include:: <isonum.txt>
+
+DPDK Release 19.11
+==================
+
+.. **Read this first.**
+
+ The text in the sections below explains how to update the release notes.
+
+ Use proper spelling, capitalization and punctuation in all sections.
+
+ Variable and config names should be quoted as fixed width text:
+ ``LIKE_THIS``.
+
+ Build the docs and view the output file to ensure the changes are correct::
+
+ make doc-guides-html
+
+ xdg-open build/doc/html/guides/rel_notes/release_19_11.html
+
+
+New Features
+------------
+
+.. This section should contain new features added in this release.
+ Sample format:
+
+ * **Add a title in the past tense with a full stop.**
+
+ Add a short 1-2 sentence description in the past tense.
+ The description should be enough to allow someone scanning
+ the release notes to understand the new feature.
+
+ If the feature adds a lot of sub-features you can use a bullet list
+ like this:
+
+ * Added feature foo to do something.
+ * Enhanced feature bar to do something else.
+
+ Refer to the previous release notes for examples.
+
+ Suggested order in release notes items:
+ * Core libs (EAL, mempool, ring, mbuf, buses)
+ * Device abstraction libs and PMDs
+ - ethdev (lib, PMDs)
+ - cryptodev (lib, PMDs)
+ - eventdev (lib, PMDs)
+ - etc
+ * Other libs
+ * Apps, Examples, Tools (if significant)
+
+ This section is a comment. Do not overwrite or remove it.
+ Also, make sure to start the actual text at the margin.
+ =========================================================
+
+
+Removed Items
+-------------
+
+.. This section should contain removed items in this release. Sample format:
+
+ * Add a short 1-2 sentence description of the removed item
+ in the past tense.
+
+ This section is a comment. Do not overwrite or remove it.
+ Also, make sure to start the actual text at the margin.
+ =========================================================
+
+
+API Changes
+-----------
+
+.. This section should contain API changes. Sample format:
+
+ * sample: Add a short 1-2 sentence description of the API change
+ which was announced in the previous releases and made in this release.
+ Start with a scope label like "ethdev:".
+ Use fixed width quotes for ``function_names`` or ``struct_names``.
+ Use the past tense.
+
+ This section is a comment. Do not overwrite or remove it.
+ Also, make sure to start the actual text at the margin.
+ =========================================================
+
+
+ABI Changes
+-----------
+
+.. This section should contain ABI changes. Sample format:
+
+ * sample: Add a short 1-2 sentence description of the ABI change
+ which was announced in the previous releases and made in this release.
+ Start with a scope label like "ethdev:".
+ Use fixed width quotes for ``function_names`` or ``struct_names``.
+ Use the past tense.
+
+ This section is a comment. Do not overwrite or remove it.
+ Also, make sure to start the actual text at the margin.
+ =========================================================
+
+
+Shared Library Versions
+-----------------------
+
+.. Update any library version updated in this release
+ and prepend with a ``+`` sign, like this:
+
+ libfoo.so.1
+ + libupdated.so.2
+ libbar.so.1
+
+ This section is a comment. Do not overwrite or remove it.
+ =========================================================
+
+The libraries prepended with a plus sign were incremented in this version.
+
+.. code-block:: diff
+
+ librte_acl.so.2
+ librte_bbdev.so.1
+ librte_bitratestats.so.2
+ librte_bpf.so.1
+ librte_bus_dpaa.so.2
+ librte_bus_fslmc.so.2
+ librte_bus_ifpga.so.2
+ librte_bus_pci.so.2
+ librte_bus_vdev.so.2
+ librte_bus_vmbus.so.2
+ librte_cfgfile.so.2
+ librte_cmdline.so.2
+ librte_compressdev.so.1
+ librte_cryptodev.so.8
+ librte_distributor.so.1
+ librte_eal.so.11
+ librte_efd.so.1
+ librte_ethdev.so.12
+ librte_eventdev.so.7
+ librte_flow_classify.so.1
+ librte_gro.so.1
+ librte_gso.so.1
+ librte_hash.so.2
+ librte_ip_frag.so.1
+ librte_ipsec.so.1
+ librte_jobstats.so.1
+ librte_kni.so.2
+ librte_kvargs.so.1
+ librte_latencystats.so.1
+ librte_lpm.so.2
+ librte_mbuf.so.5
+ librte_member.so.1
+ librte_mempool.so.5
+ librte_meter.so.3
+ librte_metrics.so.1
+ librte_net.so.1
+ librte_pci.so.1
+ librte_pdump.so.3
+ librte_pipeline.so.3
+ librte_pmd_bnxt.so.2
+ librte_pmd_bond.so.2
+ librte_pmd_i40e.so.2
+ librte_pmd_ixgbe.so.2
+ librte_pmd_dpaa2_qdma.so.1
+ librte_pmd_ring.so.2
+ librte_pmd_softnic.so.1
+ librte_pmd_vhost.so.2
+ librte_port.so.3
+ librte_power.so.1
+ librte_rawdev.so.1
+ librte_rcu.so.1
+ librte_reorder.so.1
+ librte_ring.so.2
+ librte_sched.so.3
+ librte_security.so.2
+ librte_stack.so.1
+ librte_table.so.3
+ librte_timer.so.1
+ librte_vhost.so.4
+
+
+Known Issues
+------------
+
+.. This section should contain new known issues in this release. Sample format:
+
+ * **Add title in present tense with full stop.**
+
+ Add a short 1-2 sentence description of the known issue
+ in the present tense. Add information on any known workarounds.
+
+ This section is a comment. Do not overwrite or remove it.
+ Also, make sure to start the actual text at the margin.
+ =========================================================
+
+
+Tested Platforms
+----------------
+
+.. This section should contain a list of platforms that were tested
+ with this release.
+
+ The format is:
+
+ * <vendor> platform with <vendor> <type of devices> combinations
+
+ * List of CPU
+ * List of OS
+ * List of devices
+ * Other relevant details...
+
+ This section is a comment. Do not overwrite or remove it.
+ Also, make sure to start the actual text at the margin.
+ =========================================================
+
--
1.8.3.1
^ permalink raw reply [relevance 6%]
* [dpdk-dev] [RFC v2 0/3] show the Rx/Tx burst description field
@ 2019-08-13 3:06 3% Haiyue Wang
0 siblings, 0 replies; 200+ results
From: Haiyue Wang @ 2019-08-13 3:06 UTC (permalink / raw)
To: dev; +Cc: Haiyue Wang
v1: ABI breaking
http://mails.dpdk.org/archives/dev/2019-August/140916.html
https://patchwork.dpdk.org/patch/57624/
https://patchwork.dpdk.org/patch/57625/
https://patchwork.dpdk.org/patch/57626/
v2: add a simple trace API to export string type information, if the
return value > 0, then valid information is generated.
testpmd> show rxq info 0 0
********************* Infos for port 0 , RX queue 0 *********************
Mempool: mbuf_pool_socket_0
RX prefetch threshold: 0
RX host threshold: 0
RX writeback threshold: 0
RX free threshold: 32
RX drop packets: off
RX deferred start: off
RX scattered packets: on
Number of RXDs: 1024
Burst description: AVX2 Vector Scattered Rx
testpmd> show txq info 0 0
********************* Infos for port 0 , TX queue 0 *********************
TX prefetch threshold: 32
TX host threshold: 0
TX writeback threshold: 0
TX RS threshold: 32
TX free threshold: 32
TX deferred start: off
Number of TXDs: 1024
Burst description: AVX2 Vector Tx
Haiyue Wang (3):
ethdev: add the API for getting trace information
testpmd: show the Rx/Tx burst description
net/ice: support the Rx/Tx burst description trace
app/test-pmd/config.c | 12 +++++++++
drivers/net/ice/ice_ethdev.c | 26 +++++++++++++++++++
drivers/net/ice/ice_rxtx.c | 52 +++++++++++++++++++++++++++++++++++++
drivers/net/ice/ice_rxtx.h | 4 +++
lib/librte_ethdev/rte_ethdev.c | 18 +++++++++++++
lib/librte_ethdev/rte_ethdev.h | 9 +++++++
lib/librte_ethdev/rte_ethdev_core.h | 4 +++
7 files changed, 125 insertions(+)
--
2.7.4
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] [RFC v1 0/3] show the Rx/Tx burst description field
2019-08-12 17:28 0% ` Stephen Hemminger
@ 2019-08-12 17:36 0% ` Wang, Haiyue
0 siblings, 0 replies; 200+ results
From: Wang, Haiyue @ 2019-08-12 17:36 UTC (permalink / raw)
To: Stephen Hemminger; +Cc: David Marchand, dev
> -----Original Message-----
> From: Stephen Hemminger [mailto:stephen@networkplumber.org]
> Sent: Tuesday, August 13, 2019 01:29
> To: Wang, Haiyue <haiyue.wang@intel.com>
> Cc: David Marchand <david.marchand@redhat.com>; dev <dev@dpdk.org>
> Subject: Re: [dpdk-dev] [RFC v1 0/3] show the Rx/Tx burst description field
>
> On Mon, 12 Aug 2019 16:00:27 +0000
> "Wang, Haiyue" <haiyue.wang@intel.com> wrote:
>
> > > -----Original Message-----
> > > From: Stephen Hemminger [mailto:stephen@networkplumber.org]
> > > Sent: Monday, August 12, 2019 23:54
> > > To: Wang, Haiyue <haiyue.wang@intel.com>
> > > Cc: David Marchand <david.marchand@redhat.com>; dev <dev@dpdk.org>
> > > Subject: Re: [dpdk-dev] [RFC v1 0/3] show the Rx/Tx burst description field
> > >
> > > On Mon, 12 Aug 2019 15:42:45 +0000
> > > "Wang, Haiyue" <haiyue.wang@intel.com> wrote:
> > >
> > > > > -----Original Message-----
> > > > > From: Stephen Hemminger [mailto:stephen@networkplumber.org]
> > > > > Sent: Monday, August 12, 2019 23:38
> > > > > To: David Marchand <david.marchand@redhat.com>
> > > > > Cc: Wang, Haiyue <haiyue.wang@intel.com>; dev <dev@dpdk.org>
> > > > > Subject: Re: [dpdk-dev] [RFC v1 0/3] show the Rx/Tx burst description field
> > > > >
> > > > > On Mon, 12 Aug 2019 16:27:11 +0200
> > > > > David Marchand <david.marchand@redhat.com> wrote:
> > > > >
> > > > > > On Mon, Aug 12, 2019 at 4:20 PM Haiyue Wang <haiyue.wang@intel.com> wrote:
> > > > > > >
> > > > > > > Since some PMDs have multi-path for Rx/Tx, FD.io VPP will tell you in
> > > > > > > the Debug CLI what rx/tx function is being used:
> > > > > > > #show hardware-interface
> > > > > > >
> > > > > > > tx burst function: ice_xmit_pkts
> > > > > > > rx burst function: ice_recv_scattered_pkts
> > > > > > >
> > > > > > > But if the tx/rx is static, then 'dladdr' will return nil:
> > > > > > >
> > > > > > > tx burst function: (nil) │······················
> > > > > > > rx burst function: (nil) │······················
> > > > > > >
> > > > > > > For making things consistent and gracefull, we introduce an new string
> > > > > > > field to describe the Rx/Tx burst information. This is vendor-neutral,
> > > > > > > it is used to identify the Rx/Tx burst selection if the PMD has more
> > > > > > > than one.
> > > > > > >
> > > > > > > If a PMD supports this, then rxqinfo/txqinfo->burst_info[0] != '\0'.
> > > > > >
> > > > > > The rx/tx handlers are the same for all queues of a ethdev port.
> > > > > > What is the added value to put this in a per queue api ?
> > > > >
> > > > > With some symbol table lookup tools it is possible to do introspection
> > > > > to find the symbol from the function pointer. Without breaking API/ABI.
> > > >
> > > > Sounds cool, any link can be reached ?
> > > >
> > > > VPP uses as below, but will fail for static function.
> > > >
> > > > static const char *
> > > > ptr2sname (void *p)
> > > > {
> > > > Dl_info info = { 0 };
> > > >
> > > > if (dladdr (p, &info) == 0)
> > > > return 0;
> > > >
> > > > return info.dli_sname;
> > > > }
> > >
> > > You need to link with -g and not strip the binary.
> >
> > You mean gdb debug mode, I guess they will not accept this also. ;-)
>
> There other ways to mark symbol as non-debug but -g is easiest.
> One way is to not mark the function as static.
> Other ways are to achieve the same think like __attribute((visibility())
> and linker scripts etc.
>
> Remember this is VPP you are talking about and it doesn't use standard
> DPDK make and flags.
>
Yes, this calling is a little geek style, so we are trying to scratch
some code to make thing graceful, at least, this makes sense if multiple
rx /tx paths are provided.
Thanks for your quick feedback, we will try to find other possible design.
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [RFC v1 0/3] show the Rx/Tx burst description field
2019-08-12 16:00 0% ` Wang, Haiyue
@ 2019-08-12 17:28 0% ` Stephen Hemminger
2019-08-12 17:36 0% ` Wang, Haiyue
0 siblings, 1 reply; 200+ results
From: Stephen Hemminger @ 2019-08-12 17:28 UTC (permalink / raw)
To: Wang, Haiyue; +Cc: David Marchand, dev
On Mon, 12 Aug 2019 16:00:27 +0000
"Wang, Haiyue" <haiyue.wang@intel.com> wrote:
> > -----Original Message-----
> > From: Stephen Hemminger [mailto:stephen@networkplumber.org]
> > Sent: Monday, August 12, 2019 23:54
> > To: Wang, Haiyue <haiyue.wang@intel.com>
> > Cc: David Marchand <david.marchand@redhat.com>; dev <dev@dpdk.org>
> > Subject: Re: [dpdk-dev] [RFC v1 0/3] show the Rx/Tx burst description field
> >
> > On Mon, 12 Aug 2019 15:42:45 +0000
> > "Wang, Haiyue" <haiyue.wang@intel.com> wrote:
> >
> > > > -----Original Message-----
> > > > From: Stephen Hemminger [mailto:stephen@networkplumber.org]
> > > > Sent: Monday, August 12, 2019 23:38
> > > > To: David Marchand <david.marchand@redhat.com>
> > > > Cc: Wang, Haiyue <haiyue.wang@intel.com>; dev <dev@dpdk.org>
> > > > Subject: Re: [dpdk-dev] [RFC v1 0/3] show the Rx/Tx burst description field
> > > >
> > > > On Mon, 12 Aug 2019 16:27:11 +0200
> > > > David Marchand <david.marchand@redhat.com> wrote:
> > > >
> > > > > On Mon, Aug 12, 2019 at 4:20 PM Haiyue Wang <haiyue.wang@intel.com> wrote:
> > > > > >
> > > > > > Since some PMDs have multi-path for Rx/Tx, FD.io VPP will tell you in
> > > > > > the Debug CLI what rx/tx function is being used:
> > > > > > #show hardware-interface
> > > > > >
> > > > > > tx burst function: ice_xmit_pkts
> > > > > > rx burst function: ice_recv_scattered_pkts
> > > > > >
> > > > > > But if the tx/rx is static, then 'dladdr' will return nil:
> > > > > >
> > > > > > tx burst function: (nil) │······················
> > > > > > rx burst function: (nil) │······················
> > > > > >
> > > > > > For making things consistent and gracefull, we introduce an new string
> > > > > > field to describe the Rx/Tx burst information. This is vendor-neutral,
> > > > > > it is used to identify the Rx/Tx burst selection if the PMD has more
> > > > > > than one.
> > > > > >
> > > > > > If a PMD supports this, then rxqinfo/txqinfo->burst_info[0] != '\0'.
> > > > >
> > > > > The rx/tx handlers are the same for all queues of a ethdev port.
> > > > > What is the added value to put this in a per queue api ?
> > > >
> > > > With some symbol table lookup tools it is possible to do introspection
> > > > to find the symbol from the function pointer. Without breaking API/ABI.
> > >
> > > Sounds cool, any link can be reached ?
> > >
> > > VPP uses as below, but will fail for static function.
> > >
> > > static const char *
> > > ptr2sname (void *p)
> > > {
> > > Dl_info info = { 0 };
> > >
> > > if (dladdr (p, &info) == 0)
> > > return 0;
> > >
> > > return info.dli_sname;
> > > }
> >
> > You need to link with -g and not strip the binary.
>
> You mean gdb debug mode, I guess they will not accept this also. ;-)
There other ways to mark symbol as non-debug but -g is easiest.
One way is to not mark the function as static.
Other ways are to achieve the same think like __attribute((visibility())
and linker scripts etc.
Remember this is VPP you are talking about and it doesn't use standard
DPDK make and flags.
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [RFC v1 0/3] show the Rx/Tx burst description field
2019-08-12 15:54 0% ` Stephen Hemminger
@ 2019-08-12 16:00 0% ` Wang, Haiyue
2019-08-12 17:28 0% ` Stephen Hemminger
0 siblings, 1 reply; 200+ results
From: Wang, Haiyue @ 2019-08-12 16:00 UTC (permalink / raw)
To: Stephen Hemminger; +Cc: David Marchand, dev
> -----Original Message-----
> From: Stephen Hemminger [mailto:stephen@networkplumber.org]
> Sent: Monday, August 12, 2019 23:54
> To: Wang, Haiyue <haiyue.wang@intel.com>
> Cc: David Marchand <david.marchand@redhat.com>; dev <dev@dpdk.org>
> Subject: Re: [dpdk-dev] [RFC v1 0/3] show the Rx/Tx burst description field
>
> On Mon, 12 Aug 2019 15:42:45 +0000
> "Wang, Haiyue" <haiyue.wang@intel.com> wrote:
>
> > > -----Original Message-----
> > > From: Stephen Hemminger [mailto:stephen@networkplumber.org]
> > > Sent: Monday, August 12, 2019 23:38
> > > To: David Marchand <david.marchand@redhat.com>
> > > Cc: Wang, Haiyue <haiyue.wang@intel.com>; dev <dev@dpdk.org>
> > > Subject: Re: [dpdk-dev] [RFC v1 0/3] show the Rx/Tx burst description field
> > >
> > > On Mon, 12 Aug 2019 16:27:11 +0200
> > > David Marchand <david.marchand@redhat.com> wrote:
> > >
> > > > On Mon, Aug 12, 2019 at 4:20 PM Haiyue Wang <haiyue.wang@intel.com> wrote:
> > > > >
> > > > > Since some PMDs have multi-path for Rx/Tx, FD.io VPP will tell you in
> > > > > the Debug CLI what rx/tx function is being used:
> > > > > #show hardware-interface
> > > > >
> > > > > tx burst function: ice_xmit_pkts
> > > > > rx burst function: ice_recv_scattered_pkts
> > > > >
> > > > > But if the tx/rx is static, then 'dladdr' will return nil:
> > > > >
> > > > > tx burst function: (nil) │······················
> > > > > rx burst function: (nil) │······················
> > > > >
> > > > > For making things consistent and gracefull, we introduce an new string
> > > > > field to describe the Rx/Tx burst information. This is vendor-neutral,
> > > > > it is used to identify the Rx/Tx burst selection if the PMD has more
> > > > > than one.
> > > > >
> > > > > If a PMD supports this, then rxqinfo/txqinfo->burst_info[0] != '\0'.
> > > >
> > > > The rx/tx handlers are the same for all queues of a ethdev port.
> > > > What is the added value to put this in a per queue api ?
> > >
> > > With some symbol table lookup tools it is possible to do introspection
> > > to find the symbol from the function pointer. Without breaking API/ABI.
> >
> > Sounds cool, any link can be reached ?
> >
> > VPP uses as below, but will fail for static function.
> >
> > static const char *
> > ptr2sname (void *p)
> > {
> > Dl_info info = { 0 };
> >
> > if (dladdr (p, &info) == 0)
> > return 0;
> >
> > return info.dli_sname;
> > }
>
> You need to link with -g and not strip the binary.
You mean gdb debug mode, I guess they will not accept this also. ;-)
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [RFC v1 0/3] show the Rx/Tx burst description field
2019-08-12 15:42 0% ` Wang, Haiyue
@ 2019-08-12 15:54 0% ` Stephen Hemminger
2019-08-12 16:00 0% ` Wang, Haiyue
0 siblings, 1 reply; 200+ results
From: Stephen Hemminger @ 2019-08-12 15:54 UTC (permalink / raw)
To: Wang, Haiyue; +Cc: David Marchand, dev
On Mon, 12 Aug 2019 15:42:45 +0000
"Wang, Haiyue" <haiyue.wang@intel.com> wrote:
> > -----Original Message-----
> > From: Stephen Hemminger [mailto:stephen@networkplumber.org]
> > Sent: Monday, August 12, 2019 23:38
> > To: David Marchand <david.marchand@redhat.com>
> > Cc: Wang, Haiyue <haiyue.wang@intel.com>; dev <dev@dpdk.org>
> > Subject: Re: [dpdk-dev] [RFC v1 0/3] show the Rx/Tx burst description field
> >
> > On Mon, 12 Aug 2019 16:27:11 +0200
> > David Marchand <david.marchand@redhat.com> wrote:
> >
> > > On Mon, Aug 12, 2019 at 4:20 PM Haiyue Wang <haiyue.wang@intel.com> wrote:
> > > >
> > > > Since some PMDs have multi-path for Rx/Tx, FD.io VPP will tell you in
> > > > the Debug CLI what rx/tx function is being used:
> > > > #show hardware-interface
> > > >
> > > > tx burst function: ice_xmit_pkts
> > > > rx burst function: ice_recv_scattered_pkts
> > > >
> > > > But if the tx/rx is static, then 'dladdr' will return nil:
> > > >
> > > > tx burst function: (nil) │······················
> > > > rx burst function: (nil) │······················
> > > >
> > > > For making things consistent and gracefull, we introduce an new string
> > > > field to describe the Rx/Tx burst information. This is vendor-neutral,
> > > > it is used to identify the Rx/Tx burst selection if the PMD has more
> > > > than one.
> > > >
> > > > If a PMD supports this, then rxqinfo/txqinfo->burst_info[0] != '\0'.
> > >
> > > The rx/tx handlers are the same for all queues of a ethdev port.
> > > What is the added value to put this in a per queue api ?
> >
> > With some symbol table lookup tools it is possible to do introspection
> > to find the symbol from the function pointer. Without breaking API/ABI.
>
> Sounds cool, any link can be reached ?
>
> VPP uses as below, but will fail for static function.
>
> static const char *
> ptr2sname (void *p)
> {
> Dl_info info = { 0 };
>
> if (dladdr (p, &info) == 0)
> return 0;
>
> return info.dli_sname;
> }
You need to link with -g and not strip the binary.
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [RFC v1 0/3] show the Rx/Tx burst description field
2019-08-12 15:38 3% ` Stephen Hemminger
@ 2019-08-12 15:42 0% ` Wang, Haiyue
2019-08-12 15:54 0% ` Stephen Hemminger
0 siblings, 1 reply; 200+ results
From: Wang, Haiyue @ 2019-08-12 15:42 UTC (permalink / raw)
To: Stephen Hemminger, David Marchand; +Cc: dev
> -----Original Message-----
> From: Stephen Hemminger [mailto:stephen@networkplumber.org]
> Sent: Monday, August 12, 2019 23:38
> To: David Marchand <david.marchand@redhat.com>
> Cc: Wang, Haiyue <haiyue.wang@intel.com>; dev <dev@dpdk.org>
> Subject: Re: [dpdk-dev] [RFC v1 0/3] show the Rx/Tx burst description field
>
> On Mon, 12 Aug 2019 16:27:11 +0200
> David Marchand <david.marchand@redhat.com> wrote:
>
> > On Mon, Aug 12, 2019 at 4:20 PM Haiyue Wang <haiyue.wang@intel.com> wrote:
> > >
> > > Since some PMDs have multi-path for Rx/Tx, FD.io VPP will tell you in
> > > the Debug CLI what rx/tx function is being used:
> > > #show hardware-interface
> > >
> > > tx burst function: ice_xmit_pkts
> > > rx burst function: ice_recv_scattered_pkts
> > >
> > > But if the tx/rx is static, then 'dladdr' will return nil:
> > >
> > > tx burst function: (nil) │······················
> > > rx burst function: (nil) │······················
> > >
> > > For making things consistent and gracefull, we introduce an new string
> > > field to describe the Rx/Tx burst information. This is vendor-neutral,
> > > it is used to identify the Rx/Tx burst selection if the PMD has more
> > > than one.
> > >
> > > If a PMD supports this, then rxqinfo/txqinfo->burst_info[0] != '\0'.
> >
> > The rx/tx handlers are the same for all queues of a ethdev port.
> > What is the added value to put this in a per queue api ?
>
> With some symbol table lookup tools it is possible to do introspection
> to find the symbol from the function pointer. Without breaking API/ABI.
Sounds cool, any link can be reached ?
VPP uses as below, but will fail for static function.
static const char *
ptr2sname (void *p)
{
Dl_info info = { 0 };
if (dladdr (p, &info) == 0)
return 0;
return info.dli_sname;
}
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [RFC v1 0/3] show the Rx/Tx burst description field
@ 2019-08-12 15:38 3% ` Stephen Hemminger
2019-08-12 15:42 0% ` Wang, Haiyue
0 siblings, 1 reply; 200+ results
From: Stephen Hemminger @ 2019-08-12 15:38 UTC (permalink / raw)
To: David Marchand; +Cc: Haiyue Wang, dev
On Mon, 12 Aug 2019 16:27:11 +0200
David Marchand <david.marchand@redhat.com> wrote:
> On Mon, Aug 12, 2019 at 4:20 PM Haiyue Wang <haiyue.wang@intel.com> wrote:
> >
> > Since some PMDs have multi-path for Rx/Tx, FD.io VPP will tell you in
> > the Debug CLI what rx/tx function is being used:
> > #show hardware-interface
> >
> > tx burst function: ice_xmit_pkts
> > rx burst function: ice_recv_scattered_pkts
> >
> > But if the tx/rx is static, then 'dladdr' will return nil:
> >
> > tx burst function: (nil) │······················
> > rx burst function: (nil) │······················
> >
> > For making things consistent and gracefull, we introduce an new string
> > field to describe the Rx/Tx burst information. This is vendor-neutral,
> > it is used to identify the Rx/Tx burst selection if the PMD has more
> > than one.
> >
> > If a PMD supports this, then rxqinfo/txqinfo->burst_info[0] != '\0'.
>
> The rx/tx handlers are the same for all queues of a ethdev port.
> What is the added value to put this in a per queue api ?
With some symbol table lookup tools it is possible to do introspection
to find the symbol from the function pointer. Without breaking API/ABI.
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] [RFC v1 1/3] ethdev: add the Rx/Tx burst description field in queue information
@ 2019-08-12 15:37 3% ` Stephen Hemminger
0 siblings, 0 replies; 200+ results
From: Stephen Hemminger @ 2019-08-12 15:37 UTC (permalink / raw)
To: Haiyue Wang; +Cc: dev
On Mon, 12 Aug 2019 22:15:03 +0800
Haiyue Wang <haiyue.wang@intel.com> wrote:
> Since each PMD has different Rx/Tx burst capabilities such as Vector,
> Scattered, Bulk etc, adding the Rx/Tx burst description field in queue
> information to provide apps current configuration of PMD Rx/Tx burst.
>
> This is a self-description for each PMD in its own format.
>
> Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
I can see why it might help with diagnosing issues on VPP,
but it would have bigger impacts for other users of DPDK.
Think of a better way that doesn't break ABI.
This is not enough value to make the DPDK more unstable.
^ permalink raw reply [relevance 3%]
* [dpdk-dev] [PATCH] version: 19.11-rc0
@ 2019-08-12 11:43 6% David Marchand
2019-08-13 12:18 6% ` [dpdk-dev] [PATCH v2] " David Marchand
0 siblings, 1 reply; 200+ results
From: David Marchand @ 2019-08-12 11:43 UTC (permalink / raw)
To: dev; +Cc: thomas
Start a new release cycle with empty release notes.
Signed-off-by: David Marchand <david.marchand@redhat.com>
---
VERSION | 2 +-
doc/guides/rel_notes/release_19_11.rst | 214 +++++++++++++++++++++++++++++++++
2 files changed, 215 insertions(+), 1 deletion(-)
create mode 100644 doc/guides/rel_notes/release_19_11.rst
diff --git a/VERSION b/VERSION
index 909cfd6..fff18fc 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-19.08.0
+19.11.0-rc0
diff --git a/doc/guides/rel_notes/release_19_11.rst b/doc/guides/rel_notes/release_19_11.rst
new file mode 100644
index 0000000..0bfbf19
--- /dev/null
+++ b/doc/guides/rel_notes/release_19_11.rst
@@ -0,0 +1,214 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+ Copyright 2019 The DPDK contributors
+
+DPDK Release 19.11
+==================
+
+.. **Read this first.**
+
+ The text in the sections below explains how to update the release notes.
+
+ Use proper spelling, capitalization and punctuation in all sections.
+
+ Variable and config names should be quoted as fixed width text:
+ ``LIKE_THIS``.
+
+ Build the docs and view the output file to ensure the changes are correct::
+
+ make doc-guides-html
+
+ xdg-open build/doc/html/guides/rel_notes/release_19_11.html
+
+
+New Features
+------------
+
+.. This section should contain new features added in this release.
+ Sample format:
+
+ * **Add a title in the past tense with a full stop.**
+
+ Add a short 1-2 sentence description in the past tense.
+ The description should be enough to allow someone scanning
+ the release notes to understand the new feature.
+
+ If the feature adds a lot of sub-features you can use a bullet list
+ like this:
+
+ * Added feature foo to do something.
+ * Enhanced feature bar to do something else.
+
+ Refer to the previous release notes for examples.
+
+ Suggested order in release notes items:
+ * Core libs (EAL, mempool, ring, mbuf, buses)
+ * Device abstraction libs and PMDs
+ - ethdev (lib, PMDs)
+ - cryptodev (lib, PMDs)
+ - eventdev (lib, PMDs)
+ - etc
+ * Other libs
+ * Apps, Examples, Tools (if significant)
+
+ This section is a comment. Do not overwrite or remove it.
+ Also, make sure to start the actual text at the margin.
+ =========================================================
+
+
+Removed Items
+-------------
+
+.. This section should contain removed items in this release. Sample format:
+
+ * Add a short 1-2 sentence description of the removed item
+ in the past tense.
+
+ This section is a comment. Do not overwrite or remove it.
+ Also, make sure to start the actual text at the margin.
+ =========================================================
+
+
+API Changes
+-----------
+
+.. This section should contain API changes. Sample format:
+
+ * sample: Add a short 1-2 sentence description of the API change
+ which was announced in the previous releases and made in this release.
+ Start with a scope label like "ethdev:".
+ Use fixed width quotes for ``function_names`` or ``struct_names``.
+ Use the past tense.
+
+ This section is a comment. Do not overwrite or remove it.
+ Also, make sure to start the actual text at the margin.
+ =========================================================
+
+
+ABI Changes
+-----------
+
+.. This section should contain ABI changes. Sample format:
+
+ * sample: Add a short 1-2 sentence description of the ABI change
+ which was announced in the previous releases and made in this release.
+ Start with a scope label like "ethdev:".
+ Use fixed width quotes for ``function_names`` or ``struct_names``.
+ Use the past tense.
+
+ This section is a comment. Do not overwrite or remove it.
+ Also, make sure to start the actual text at the margin.
+ =========================================================
+
+
+Shared Library Versions
+-----------------------
+
+.. Update any library version updated in this release
+ and prepend with a ``+`` sign, like this:
+
+ libfoo.so.1
+ + libupdated.so.2
+ libbar.so.1
+
+ This section is a comment. Do not overwrite or remove it.
+ =========================================================
+
+The libraries prepended with a plus sign were incremented in this version.
+
+.. code-block:: diff
+
+ librte_acl.so.2
+ librte_bbdev.so.1
+ librte_bitratestats.so.2
+ librte_bpf.so.1
+ librte_bus_dpaa.so.2
+ librte_bus_fslmc.so.2
+ librte_bus_ifpga.so.2
+ librte_bus_pci.so.2
+ librte_bus_vdev.so.2
+ librte_bus_vmbus.so.2
+ librte_cfgfile.so.2
+ librte_cmdline.so.2
+ librte_compressdev.so.1
+ librte_cryptodev.so.7
+ librte_distributor.so.1
+ librte_eal.so.10
+ librte_efd.so.1
+ librte_ethdev.so.12
+ librte_eventdev.so.6
+ librte_flow_classify.so.1
+ librte_gro.so.1
+ librte_gso.so.1
+ librte_hash.so.2
+ librte_ip_frag.so.1
+ librte_ipsec.so.1
+ librte_jobstats.so.1
+ librte_kni.so.2
+ librte_kvargs.so.1
+ librte_latencystats.so.1
+ librte_lpm.so.2
+ librte_mbuf.so.5
+ librte_member.so.1
+ librte_mempool.so.5
+ librte_meter.so.3
+ librte_metrics.so.1
+ librte_net.so.1
+ librte_pci.so.1
+ librte_pdump.so.3
+ librte_pipeline.so.3
+ librte_pmd_bnxt.so.2
+ librte_pmd_bond.so.2
+ librte_pmd_i40e.so.2
+ librte_pmd_ixgbe.so.2
+ librte_pmd_dpaa2_qdma.so.1
+ librte_pmd_ring.so.2
+ librte_pmd_softnic.so.1
+ librte_pmd_vhost.so.2
+ librte_port.so.3
+ librte_power.so.1
+ librte_rawdev.so.1
+ librte_rcu.so.1
+ librte_reorder.so.1
+ librte_ring.so.2
+ librte_sched.so.2
+ librte_security.so.2
+ librte_stack.so.1
+ librte_table.so.3
+ librte_timer.so.1
+ librte_vhost.so.4
+
+
+Known Issues
+------------
+
+.. This section should contain new known issues in this release. Sample format:
+
+ * **Add title in present tense with full stop.**
+
+ Add a short 1-2 sentence description of the known issue
+ in the present tense. Add information on any known workarounds.
+
+ This section is a comment. Do not overwrite or remove it.
+ Also, make sure to start the actual text at the margin.
+ =========================================================
+
+
+Tested Platforms
+----------------
+
+.. This section should contain a list of platforms that were tested
+ with this release.
+
+ The format is:
+
+ * <vendor> platform with <vendor> <type of devices> combinations
+
+ * List of CPU
+ * List of OS
+ * List of devices
+ * Other relevant details...
+
+ This section is a comment. Do not overwrite or remove it.
+ Also, make sure to start the actual text at the margin.
+ =========================================================
+
--
1.8.3.1
^ permalink raw reply [relevance 6%]
* Re: [dpdk-dev] [PATCH] eal: change max hugepage sizes to 4
2019-08-12 9:49 0% ` David Marchand
2019-08-12 10:01 0% ` Thomas Monjalon
@ 2019-08-12 10:38 0% ` Burakov, Anatoly
1 sibling, 0 replies; 200+ results
From: Burakov, Anatoly @ 2019-08-12 10:38 UTC (permalink / raw)
To: David Marchand
Cc: Thomas Monjalon, Hemant Agrawal, Gagandeep Singh, dev,
Olivier Matz, Andrew Rybchenko, Nipun Gupta,
Honnappa Nagarahalli, Steve Capper, Jerin Jacob Kollanukkaran,
Bruce Richardson, Gavin Hu, Ananyev, Konstantin,
David Christensen
On 12-Aug-19 10:49 AM, David Marchand wrote:
> On Mon, Aug 12, 2019 at 11:43 AM Burakov, Anatoly
> <anatoly.burakov@intel.com> wrote:
>> On 08-Aug-19 8:31 AM, Thomas Monjalon wrote:
>>> I would suggest to restrict the change to Arm only with an ifdef,
>>> in order to limit the risk for this release.
>>> We can think about a dynamic hugepage scan in the next release.
>>>
>>
>> I don't see how this is necessary. The 3 is an arbitrary number here,
>> and the ABI isn't broken as this is an internal structure. We could
>> increase it to 16 for all i care, and it wouldn't make any difference to
>> the rest of the code - we never populate more than we can find anyway.
>
> I agree on the principle.
> But at the time this popped up, we were really close to the release.
> It seemed a way to mitigate any unforeseen issue by limiting to the
> platform that was affected.
>
Fair enough. A follow up is needed so. Frankly, i don't see the need to
complicate things with "dynamic" stuff here - a static array of 8 or 16
page sizes should be enough for everyone (TM).
--
Thanks,
Anatoly
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [PATCH] eal: change max hugepage sizes to 4
2019-08-12 9:49 0% ` David Marchand
@ 2019-08-12 10:01 0% ` Thomas Monjalon
2019-08-12 10:38 0% ` Burakov, Anatoly
1 sibling, 0 replies; 200+ results
From: Thomas Monjalon @ 2019-08-12 10:01 UTC (permalink / raw)
To: Burakov, Anatoly
Cc: David Marchand, Hemant Agrawal, Gagandeep Singh, dev,
Olivier Matz, Andrew Rybchenko, Nipun Gupta,
Honnappa Nagarahalli, Steve Capper, Jerin Jacob Kollanukkaran,
Bruce Richardson, Gavin Hu, Ananyev, Konstantin,
David Christensen
12/08/2019 11:49, David Marchand:
> On Mon, Aug 12, 2019 at 11:43 AM Burakov, Anatoly
> <anatoly.burakov@intel.com> wrote:
> > On 08-Aug-19 8:31 AM, Thomas Monjalon wrote:
> > > I would suggest to restrict the change to Arm only with an ifdef,
> > > in order to limit the risk for this release.
> > > We can think about a dynamic hugepage scan in the next release.
> >
> > I don't see how this is necessary. The 3 is an arbitrary number here,
> > and the ABI isn't broken as this is an internal structure. We could
> > increase it to 16 for all i care, and it wouldn't make any difference to
> > the rest of the code - we never populate more than we can find anyway.
>
> I agree on the principle.
> But at the time this popped up, we were really close to the release.
> It seemed a way to mitigate any unforeseen issue by limiting to the
> platform that was affected.
Exactly, we were extra cautious.
Please increase the value for everybody, thanks.
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [PATCH] eal: change max hugepage sizes to 4
2019-08-12 9:43 3% ` Burakov, Anatoly
@ 2019-08-12 9:49 0% ` David Marchand
2019-08-12 10:01 0% ` Thomas Monjalon
2019-08-12 10:38 0% ` Burakov, Anatoly
0 siblings, 2 replies; 200+ results
From: David Marchand @ 2019-08-12 9:49 UTC (permalink / raw)
To: Burakov, Anatoly
Cc: Thomas Monjalon, Hemant Agrawal, Gagandeep Singh, dev,
Olivier Matz, Andrew Rybchenko, Nipun Gupta,
Honnappa Nagarahalli, Steve Capper, Jerin Jacob Kollanukkaran,
Bruce Richardson, Gavin Hu, Ananyev, Konstantin,
David Christensen
On Mon, Aug 12, 2019 at 11:43 AM Burakov, Anatoly
<anatoly.burakov@intel.com> wrote:
> On 08-Aug-19 8:31 AM, Thomas Monjalon wrote:
> > I would suggest to restrict the change to Arm only with an ifdef,
> > in order to limit the risk for this release.
> > We can think about a dynamic hugepage scan in the next release.
> >
>
> I don't see how this is necessary. The 3 is an arbitrary number here,
> and the ABI isn't broken as this is an internal structure. We could
> increase it to 16 for all i care, and it wouldn't make any difference to
> the rest of the code - we never populate more than we can find anyway.
I agree on the principle.
But at the time this popped up, we were really close to the release.
It seemed a way to mitigate any unforeseen issue by limiting to the
platform that was affected.
--
David Marchand
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [PATCH] eal: change max hugepage sizes to 4
@ 2019-08-12 9:43 3% ` Burakov, Anatoly
2019-08-12 9:49 0% ` David Marchand
0 siblings, 1 reply; 200+ results
From: Burakov, Anatoly @ 2019-08-12 9:43 UTC (permalink / raw)
To: Thomas Monjalon, Hemant Agrawal, Gagandeep Singh
Cc: dev, David Marchand, Olivier Matz, Andrew Rybchenko, Nipun Gupta,
honnappa.nagarahalli, Steve Capper, jerinj, bruce.richardson,
gavin.hu, konstantin.ananyev, drc
On 08-Aug-19 8:31 AM, Thomas Monjalon wrote:
> 07/08/2019 15:28, Hemant Agrawal:
>> HI Thomas,
>>
>>>>> DPDK currently is supporting maximum 3 hugepage, sizes whereas
>>>>> system can support more than this e.g.
>>>>> 64K, 2M, 32M and 1G.
>>>>
>>>> You can mention ARM platform here, and that this issue starts with
>>>> kernel 5.2 (and I would try to mention this in the title as well).
>>>> This is better than an annotation that will be lost.
>>>>
>>>>
>>>>> Having these four hugepage sizes available to use by DPDK, which is
>>>>> valid in case of '--in-memory' EAL option or using 4 separate mount
>>>>> points for each hugepage size;
>>>>> hugepage_info_init() API reports an error.
>>>>
>>>> Can you describe what is the impact from a user point of view rather
>>>> than mentioning this internal function?
>>>
>>> Yes please, we need to understand how much it is critical.
>>> Should we Cc stable@dpdk.org for backport?
>>> Should it be merged at the last minute in 19.08?
>>
>> VPP usages in-memory option. So, VPP on ARM with kernel 5.2 wont' work without this patch.
>
> Do you want to send a v2 with a better explanation?
>
> I would suggest to restrict the change to Arm only with an ifdef,
> in order to limit the risk for this release.
> We can think about a dynamic hugepage scan in the next release.
>
I don't see how this is necessary. The 3 is an arbitrary number here,
and the ABI isn't broken as this is an internal structure. We could
increase it to 16 for all i care, and it wouldn't make any difference to
the rest of the code - we never populate more than we can find anyway.
--
Thanks,
Anatoly
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] [PATCH 1/2] doc: announce ethdev ABI change for LRO fields
2019-08-06 15:27 4% ` Andrew Rybchenko
@ 2019-08-10 21:40 4% ` Thomas Monjalon
0 siblings, 0 replies; 200+ results
From: Thomas Monjalon @ 2019-08-10 21:40 UTC (permalink / raw)
To: Matan Azrad
Cc: dev, Andrew Rybchenko, Ferruh Yigit, Konstantin Ananyev, Olivier Matz
06/08/2019 17:27, Andrew Rybchenko:
> On 8/6/19 5:56 PM, Matan Azrad wrote:
> > It may be needed by the user to limit the LRO session packet size.
> > In order to allow the above limitation, a new Rx configuration may be
> > added for the maximum LRO session size.
> >
> > A new capability may be added too to expose the maximum LRO session size
> > supported by the port.
> >
> > Signed-off-by: Matan Azrad <matan@mellanox.com>
> > ---
> > +* ethdev: new 32-bit fields may be added for maximum LRO session size, in
> > + struct ``rte_eth_dev_info`` for the port capability and in struct
> > + ``rte_eth_rxmode`` for the port configuration.
>
> Acked-by: Andrew Rybchenko <arybchenko@solarflare.com>
>
> I don't know examples when the information is required, but can imagine.
Acked-by: Thomas Monjalon <thomas@monjalon.net>
We have only 2 acks but as they are simple new fields,
better to announce their addition in advance and allow for more
discussion during 19.11 release cycle.
Applied (only patch 1 of 2)
^ permalink raw reply [relevance 4%]
* Re: [dpdk-dev] [patch v5] doc: announce API change in ethdev offload flags
@ 2019-08-10 21:10 3% ` Thomas Monjalon
0 siblings, 0 replies; 200+ results
From: Thomas Monjalon @ 2019-08-10 21:10 UTC (permalink / raw)
To: pbhagavatula
Cc: dev, Ananyev, Konstantin, jerinj, stephen, arybchenko,
hemant.agrawal, Yigit, Ferruh, Richardson, Bruce, Neil Horman,
Mcnamara, John, Kovacevic, Marko
> > Add new offload flags ``DEV_RX_OFFLOAD_RSS_HASH`` and
> > ``DEV_RX_OFFLOAD_FLOW_MARK``.
> > Add new function ``rte_eth_dev_set_supported_ptypes`` to allow application
> > to set specific ptypes to be updated in ``rte_mbuf::packet_type``
> >
> > Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com>
> > Acked-by: Andrew Rybchenko <arybchenko@solarflare.com>
> > Acked-by: Jerin Jacob <jerinj@marvell.com>
> > Acked-by: Hemant Agrawal <hemant.agrawal@nxp.com>
> Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
We'll probably have to discuss more the details of these new APIs,
but the overall idea looks good.
I am not sure there is any API or ABI breakage here,
so the announce may not be required.
Anyway applied, thanks.
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] [PATCH] doc: announce ABI change for RSS hash funtion
2019-08-06 14:45 4% ` Ananyev, Konstantin
@ 2019-08-10 20:39 4% ` Thomas Monjalon
0 siblings, 0 replies; 200+ results
From: Thomas Monjalon @ 2019-08-10 20:39 UTC (permalink / raw)
To: Su, Simei
Cc: dev, Ananyev, Konstantin, Zhang, Qi Z, Wu, Jingjing, Xing,
Beilei, Yang, Qiming, orika, adrien.mazarguil, arybchenko, Yigit,
Ferruh
06/08/2019 16:45, Ananyev, Konstantin:
> From: Thomas Monjalon
> > 04/07/2019 06:43, simei:
> > > From: Simei Su <simei.su@intel.com>
> > >
> > > Add new field SYMMETRIC_TOEPLITZ in rte_eth_hash_function. This
> > > can support symmetric hash function by rte_flow RSS action.
> > >
> > > Signed-off-by: Simei Su <simei.su@intel.com>
> > > ---
> > > +* ethdev: New member in ``rte_eth_hash_funtion`` to support symmetric hash funtion.
> >
> > That's unfortunate there is a typo in the name of the enum you want to change.
> >
> > Do you have any reference to the algo you want to support? A paper maybe?
>
> If I am not mistaken that's just an intent to enable symmetric RSS hash-function via standard RSS rte_flow API -
> feature already available with i40e and newest Intel HW.
> (AFAIK on i40e right now it could be configured via RTE_ETH_HASH_FILTER_SYM_HASH_ENA_PER_PORT).
> If so, then I think the author may need to mention what PMDs will support that feature in 19.11.
Without any more comment, this patch cannot be accepted in 19.08.
^ permalink raw reply [relevance 4%]
* [dpdk-dev] [PATCH v2] doc: announce lcore_config symbol removal
2019-07-31 11:06 5% [dpdk-dev] [PATCH] doc: announce lcore_config symbol removal David Marchand
2019-07-31 13:48 0% ` Stephen Hemminger
@ 2019-08-08 9:31 5% ` David Marchand
1 sibling, 0 replies; 200+ results
From: David Marchand @ 2019-08-08 9:31 UTC (permalink / raw)
To: dev; +Cc: thomas, stephen, bruce.richardson, jerinj, arybchenko
New accessors have been introduced to provide the hidden information.
This symbol can now be kept internal.
Signed-off-by: David Marchand <david.marchand@redhat.com>
Acked-by: Stephen Hemminger <stephen@networkplumber.org>
Acked-by: Bruce Richardson <bruce.richardson@intel.com>
Acked-by: Jerin Jacob <jerinj@marvell.com>
Acked-by: Andrew Rybchenko <arybchenko@solarflare.com>
---
Changelog since v1:
- added acks,
- fixed little typo,
---
doc/guides/rel_notes/deprecation.rst | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/doc/guides/rel_notes/deprecation.rst b/doc/guides/rel_notes/deprecation.rst
index 70fef52..ee15ab6 100644
--- a/doc/guides/rel_notes/deprecation.rst
+++ b/doc/guides/rel_notes/deprecation.rst
@@ -23,6 +23,10 @@ Deprecation Notices
* eal: The function ``rte_eal_remote_launch`` will return new error codes
after read or write error on the pipe, instead of calling ``rte_panic``.
+* eal: The ``lcore_config`` struct and global symbol will be made private to
+ remove it from the externally visible ABI and allow it to be updated in the
+ future.
+
* eal: both declaring and identifying devices will be streamlined in v18.11.
New functions will appear to query a specific port from buses, classes of
device and device drivers. Device declaration will be made coherent with the
--
1.8.3.1
^ permalink raw reply [relevance 5%]
* [dpdk-dev] 19.11 Intel Roadmap
@ 2019-08-08 8:12 4% O'Driscoll, Tim
0 siblings, 0 replies; 200+ results
From: O'Driscoll, Tim @ 2019-08-08 8:12 UTC (permalink / raw)
To: dev
These are the features that we plan to submit for the 19.11 release:
Intel(r) Ethernet 810 Series Network Adapter Enhancements: Support for loading of device-specific Dynamic Device Personalization (DDP) profiles to support new protocols, rte_flow/flow director/switch filter updates to support DDP profiles, AVX2 versions of the ICE and IAVF PMDs to improve performance, multi-process support, RSS support via rte_flow, support for high/low priority flows via rte_flow, flexible descriptor support per Rx queue.
IPsec Enhancements: The rte_security API will be updated to support inline crypto statistics. A security association database will be added to the IPsec library. A new rte_security type (RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO) will be added to improve performance for IPsec with software crypto. The IPsec Security Gateway sample app will be updated to demonstrate the IPSec library's ability to support multiple IPsec sessions for the same SA (inline-crypto session plus lookaside-none).
Compression Enhancements: Intel(r) QuickAssist Technology support for stateful decompression, enhancements to the performance test tool including the ability to use external mbufs, and removal of the Experimental label from the compressdev API.
Crypto Enhancements: Increased Intel(r) QuickAssist Technology support for asymmetric (RSA) and symmetric (single-pass GCM) crypto operations, modifications to the cryptodev API to support sessionless asymmetric crypto.
Hierarchical QoS Scheduler Enhancements: The pipe (subscriber) configuration will be made more flexible by moving configuration data from the port to the sub-port level. Enhancements will also be made to provide better support for over-subscription, allowing unused bandwidth for a pipe to be redistributed to other pipes in the same sub-port.
ABI Stability: Agree and document new ABI stability policy. Implement code changes (hiding of internal structures etc.) to make ABI easier to maintain.
Virtio packed ring (introduced in Virtio 1.1 spec) performance optimisations.
Support graceful shutdown for the Intel(r) FPGA Programmable Acceleration Card N3000. The server will be able to gracefully shutdown and reload the card after certain errors (e.g. FPGA Die Temperature higher than Temperature Threshold or FPGA AUX Voltage lower than Voltage Threshold).
Extend Rawdev NTB PMD: The rawdev PMD for Non-Transparent Bridging will be extended to add a FIFO ring for Rx/Tx.
A sample application will be added for the Intel(r) QuickData Technology PMD (drivers/raw/ioat).
Intel PMDs will be updated to support RTE_ETH_DEV_CLOSE_REMOVE.
^ permalink raw reply [relevance 4%]
* Re: [dpdk-dev] [RFC 0/3] ethdev: add ptype as Rx offload
2019-08-07 15:22 0% ` Stephen Hemminger
@ 2019-08-07 15:44 0% ` Andrew Rybchenko
0 siblings, 0 replies; 200+ results
From: Andrew Rybchenko @ 2019-08-07 15:44 UTC (permalink / raw)
To: Stephen Hemminger
Cc: Jerin Jacob Kollanukkaran, Pavan Nikhilesh Bhagavatula,
Hemant Agrawal, dev
On 8/7/19 6:22 PM, Stephen Hemminger wrote:
> On Wed, 7 Aug 2019 11:32:35 +0300
> Andrew Rybchenko <arybchenko@solarflare.com> wrote:
>
>> On 8/7/19 5:04 AM, Jerin Jacob Kollanukkaran wrote:
>>>> -----Original Message-----
>>>> From: Stephen Hemminger <stephen@networkplumber.org>
>>>> Sent: Wednesday, August 7, 2019 4:45 AM
>>>> To: Andrew Rybchenko <arybchenko@solarflare.com>
>>>> Cc: Pavan Nikhilesh Bhagavatula <pbhagavatula@marvell.com>; Hemant
>>>> Agrawal <hemant.agrawal@nxp.com>; Jerin Jacob Kollanukkaran
>>>> <jerinj@marvell.com>; dev@dpdk.org
>>>> Subject: [EXT] Re: [dpdk-dev] [RFC 0/3] ethdev: add ptype as Rx offload
>>>>
>>>> On Tue, 6 Aug 2019 12:06:35 +0300
>>>> Andrew Rybchenko <arybchenko@solarflare.com> wrote:
>>>>
>>>>> On 8/6/19 11:47 AM, Pavan Nikhilesh Bhagavatula wrote:
>>>>>>> -----Original Message-----
>>>>>>> From: Hemant Agrawal <hemant.agrawal@nxp.com>
>>>>>>> Sent: Tuesday, August 6, 2019 1:49 PM
>>>>>>> To: Pavan Nikhilesh Bhagavatula <pbhagavatula@marvell.com>; Jerin
>>>>>>> Jacob Kollanukkaran <jerinj@marvell.com>
>>>>>>> Cc: dev@dpdk.org
>>>>>>> Subject: RE: [dpdk-dev] [RFC 0/3] ethdev: add ptype as Rx offload
>>>>>>>> Add PTYPE to DEV_RX_OFFLOAD_* flags.
>>>>>>>>
>>>>>>>> Currently, most of the NICs already support PTYPE parsing and
>>>>>>>> update
>>>>>>> the
>>>>>>>> mbuf->packet_type through an internal lookup table, but there is
>>>>>>>> mbuf->no
>>>>>>> way to
>>>>>>>> disable the lookup if the application is not intrested in ptypes
>>>>>>> returned by
>>>>>>>> `rte_eth_dev_get_supported_ptypes`.
>>>>>>>>
>>>>>>> [Hemant] it will also mean introducing another check in datapath,
>>>>>>> if the application has asked for PTYPE offload - copy the results
>>>>>>> to mbuf-
>>>>>>>> packet_type otherwise don't do it.
>>>>>> I think that having the check would give better performance than
>>>>>> loading ptype table to L1 doing a lookup and copying it to mbuf when the
>>>> application doesn't need it.
>>>>> Anyway, if PMD decides that it is better to always provide packet type
>>>>> information - there is no harm. Basically if the offload is not
>>>>> requested it makes packet_type undefined in mbuf.
>>>>>
>>>>>>> Your second patch is incomplete in the sense that it only adds the
>>>>>>> capability. But it does not disable the lookups?
>>>>>> It is upto the maintainer of the PMD to disable the lookup in data
>>>>>> path. If there is a scope of optimization then they could do it. There is no
>>>> harm in exposing PTYPE even RX_OFFLOAD_PTYPE is not enabled.
>>>>>> I was hesitant to touch data path as it would be impossible to verify
>>>> performance effect on all NICs.
>>>>> I think it is the right way to approach it especially taking
>>>>> transition into account.
>>>>>
>>>> With hardline API policy, this has to fail on compile for old applications.
>>
>> Stephen, could you explain a bit more why.
>
> Existing releases packets will be received with ptype for hardware that
> supports it. We should not require users to change their application to
> continue to get mbufs with ptype. If your change would break that, and
> require application to change; then your change should break the API in
> a hard way that causes compile rather than runtime failure.
Many thanks, I got it.
> The best solution would be to just keep old applications running and compiling
> without breaking anything. That means ptype should still be received.
>
> If (as an optimization) you want to allow application to turn of getting
> ptype; then that would be a useful. Probably best done at the port level
> as part of configuration.
I see, but it contradicts to the existing practice that offloads should
be disabled by default and a way to enable should be provided.
May be techboard should discuss it and make a decision (covering RSS
hash information and Rx mark mentioned in my review notes).
>>> Not specific to this API change. That's is the propriety any new symbol addition
>>> to the code base.
>>>
>>> Planning to make this API change available fromv19.11 LTS.
>>
>> The only way to to require applications to enable PTYPE offload to get
>> ptypes in mbuf since 19.11 LTS is to have deprecation notice in 19.08.
>>
>>>> You can't magically assume that applications using ptype will set new feature.
>>> When OFFLOAD flags got introduced, we decided to disable all offloads by default.
>>> So, need to add positive logic here to enable offload instead of enable something by
>>> Default and disable if required to get have synergy with other offloads.
>>>
>>> Will update the release note as usual to document the change.
>>> Since there is NO ABI change, IMO, we don't need deprecation notice.
>>
>> Sorry, but it is a behaviour change. Before an application does not need
>> to enable ptype offload, but now it is required. It means that application
>> will be broken and, therefore, it requires deprecation notice.
>
> The DPDK development community has to make not breaking applications
> a higher priority than adding marginal enhancements
Fair, but where is marginal enhancements boundary?
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [RFC 0/3] ethdev: add ptype as Rx offload
2019-08-07 8:32 0% ` Andrew Rybchenko
@ 2019-08-07 15:22 0% ` Stephen Hemminger
2019-08-07 15:44 0% ` Andrew Rybchenko
0 siblings, 1 reply; 200+ results
From: Stephen Hemminger @ 2019-08-07 15:22 UTC (permalink / raw)
To: Andrew Rybchenko
Cc: Jerin Jacob Kollanukkaran, Pavan Nikhilesh Bhagavatula,
Hemant Agrawal, dev
On Wed, 7 Aug 2019 11:32:35 +0300
Andrew Rybchenko <arybchenko@solarflare.com> wrote:
> On 8/7/19 5:04 AM, Jerin Jacob Kollanukkaran wrote:
> >> -----Original Message-----
> >> From: Stephen Hemminger <stephen@networkplumber.org>
> >> Sent: Wednesday, August 7, 2019 4:45 AM
> >> To: Andrew Rybchenko <arybchenko@solarflare.com>
> >> Cc: Pavan Nikhilesh Bhagavatula <pbhagavatula@marvell.com>; Hemant
> >> Agrawal <hemant.agrawal@nxp.com>; Jerin Jacob Kollanukkaran
> >> <jerinj@marvell.com>; dev@dpdk.org
> >> Subject: [EXT] Re: [dpdk-dev] [RFC 0/3] ethdev: add ptype as Rx offload
> >>
> >> On Tue, 6 Aug 2019 12:06:35 +0300
> >> Andrew Rybchenko <arybchenko@solarflare.com> wrote:
> >>
> >>> On 8/6/19 11:47 AM, Pavan Nikhilesh Bhagavatula wrote:
> >>>>> -----Original Message-----
> >>>>> From: Hemant Agrawal <hemant.agrawal@nxp.com>
> >>>>> Sent: Tuesday, August 6, 2019 1:49 PM
> >>>>> To: Pavan Nikhilesh Bhagavatula <pbhagavatula@marvell.com>; Jerin
> >>>>> Jacob Kollanukkaran <jerinj@marvell.com>
> >>>>> Cc: dev@dpdk.org
> >>>>> Subject: RE: [dpdk-dev] [RFC 0/3] ethdev: add ptype as Rx offload
> >>>>>> Add PTYPE to DEV_RX_OFFLOAD_* flags.
> >>>>>>
> >>>>>> Currently, most of the NICs already support PTYPE parsing and
> >>>>>> update
> >>>>> the
> >>>>>> mbuf->packet_type through an internal lookup table, but there is
> >>>>>> mbuf->no
> >>>>> way to
> >>>>>> disable the lookup if the application is not intrested in ptypes
> >>>>> returned by
> >>>>>> `rte_eth_dev_get_supported_ptypes`.
> >>>>>>
> >>>>> [Hemant] it will also mean introducing another check in datapath,
> >>>>> if the application has asked for PTYPE offload - copy the results
> >>>>> to mbuf-
> >>>>>> packet_type otherwise don't do it.
> >>>> I think that having the check would give better performance than
> >>>> loading ptype table to L1 doing a lookup and copying it to mbuf when the
> >> application doesn't need it.
> >>> Anyway, if PMD decides that it is better to always provide packet type
> >>> information - there is no harm. Basically if the offload is not
> >>> requested it makes packet_type undefined in mbuf.
> >>>
> >>>>> Your second patch is incomplete in the sense that it only adds the
> >>>>> capability. But it does not disable the lookups?
> >>>> It is upto the maintainer of the PMD to disable the lookup in data
> >>>> path. If there is a scope of optimization then they could do it. There is no
> >> harm in exposing PTYPE even RX_OFFLOAD_PTYPE is not enabled.
> >>>> I was hesitant to touch data path as it would be impossible to verify
> >> performance effect on all NICs.
> >>> I think it is the right way to approach it especially taking
> >>> transition into account.
> >>>
> >> With hardline API policy, this has to fail on compile for old applications.
>
> Stephen, could you explain a bit more why.
Existing releases packets will be received with ptype for hardware that
supports it. We should not require users to change their application to
continue to get mbufs with ptype. If your change would break that, and
require application to change; then your change should break the API in
a hard way that causes compile rather than runtime failure.
The best solution would be to just keep old applications running and compiling
without breaking anything. That means ptype should still be received.
If (as an optimization) you want to allow application to turn of getting
ptype; then that would be a useful. Probably best done at the port level
as part of configuration.
>
> > Not specific to this API change. That's is the propriety any new symbol addition
> > to the code base.
> >
> > Planning to make this API change available fromv19.11 LTS.
>
> The only way to to require applications to enable PTYPE offload to get
> ptypes in mbuf since 19.11 LTS is to have deprecation notice in 19.08.
>
> >> You can't magically assume that applications using ptype will set new feature.
> > When OFFLOAD flags got introduced, we decided to disable all offloads by default.
> > So, need to add positive logic here to enable offload instead of enable something by
> > Default and disable if required to get have synergy with other offloads.
> >
> > Will update the release note as usual to document the change.
> > Since there is NO ABI change, IMO, we don't need deprecation notice.
>
> Sorry, but it is a behaviour change. Before an application does not need
> to enable ptype offload, but now it is required. It means that application
> will be broken and, therefore, it requires deprecation notice.
The DPDK development community has to make not breaking applications
a higher priority than adding marginal enhancements
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [RFC 0/3] ethdev: add ptype as Rx offload
2019-08-07 2:04 3% [dpdk-dev] [RFC 0/3] ethdev: add ptype as Rx offload Jerin Jacob Kollanukkaran
@ 2019-08-07 8:32 0% ` Andrew Rybchenko
2019-08-07 15:22 0% ` Stephen Hemminger
0 siblings, 1 reply; 200+ results
From: Andrew Rybchenko @ 2019-08-07 8:32 UTC (permalink / raw)
To: Jerin Jacob Kollanukkaran, Stephen Hemminger
Cc: Pavan Nikhilesh Bhagavatula, Hemant Agrawal, dev
On 8/7/19 5:04 AM, Jerin Jacob Kollanukkaran wrote:
>> -----Original Message-----
>> From: Stephen Hemminger <stephen@networkplumber.org>
>> Sent: Wednesday, August 7, 2019 4:45 AM
>> To: Andrew Rybchenko <arybchenko@solarflare.com>
>> Cc: Pavan Nikhilesh Bhagavatula <pbhagavatula@marvell.com>; Hemant
>> Agrawal <hemant.agrawal@nxp.com>; Jerin Jacob Kollanukkaran
>> <jerinj@marvell.com>; dev@dpdk.org
>> Subject: [EXT] Re: [dpdk-dev] [RFC 0/3] ethdev: add ptype as Rx offload
>>
>> On Tue, 6 Aug 2019 12:06:35 +0300
>> Andrew Rybchenko <arybchenko@solarflare.com> wrote:
>>
>>> On 8/6/19 11:47 AM, Pavan Nikhilesh Bhagavatula wrote:
>>>>> -----Original Message-----
>>>>> From: Hemant Agrawal <hemant.agrawal@nxp.com>
>>>>> Sent: Tuesday, August 6, 2019 1:49 PM
>>>>> To: Pavan Nikhilesh Bhagavatula <pbhagavatula@marvell.com>; Jerin
>>>>> Jacob Kollanukkaran <jerinj@marvell.com>
>>>>> Cc: dev@dpdk.org
>>>>> Subject: RE: [dpdk-dev] [RFC 0/3] ethdev: add ptype as Rx offload
>>>>>> Add PTYPE to DEV_RX_OFFLOAD_* flags.
>>>>>>
>>>>>> Currently, most of the NICs already support PTYPE parsing and
>>>>>> update
>>>>> the
>>>>>> mbuf->packet_type through an internal lookup table, but there is
>>>>>> mbuf->no
>>>>> way to
>>>>>> disable the lookup if the application is not intrested in ptypes
>>>>> returned by
>>>>>> `rte_eth_dev_get_supported_ptypes`.
>>>>>>
>>>>> [Hemant] it will also mean introducing another check in datapath,
>>>>> if the application has asked for PTYPE offload - copy the results
>>>>> to mbuf-
>>>>>> packet_type otherwise don't do it.
>>>> I think that having the check would give better performance than
>>>> loading ptype table to L1 doing a lookup and copying it to mbuf when the
>> application doesn't need it.
>>> Anyway, if PMD decides that it is better to always provide packet type
>>> information - there is no harm. Basically if the offload is not
>>> requested it makes packet_type undefined in mbuf.
>>>
>>>>> Your second patch is incomplete in the sense that it only adds the
>>>>> capability. But it does not disable the lookups?
>>>> It is upto the maintainer of the PMD to disable the lookup in data
>>>> path. If there is a scope of optimization then they could do it. There is no
>> harm in exposing PTYPE even RX_OFFLOAD_PTYPE is not enabled.
>>>> I was hesitant to touch data path as it would be impossible to verify
>> performance effect on all NICs.
>>> I think it is the right way to approach it especially taking
>>> transition into account.
>>>
>> With hardline API policy, this has to fail on compile for old applications.
Stephen, could you explain a bit more why.
> Not specific to this API change. That's is the propriety any new symbol addition
> to the code base.
>
> Planning to make this API change available fromv19.11 LTS.
The only way to to require applications to enable PTYPE offload to get
ptypes in mbuf since 19.11 LTS is to have deprecation notice in 19.08.
>> You can't magically assume that applications using ptype will set new feature.
> When OFFLOAD flags got introduced, we decided to disable all offloads by default.
> So, need to add positive logic here to enable offload instead of enable something by
> Default and disable if required to get have synergy with other offloads.
>
> Will update the release note as usual to document the change.
> Since there is NO ABI change, IMO, we don't need deprecation notice.
Sorry, but it is a behaviour change. Before an application does not need
to enable ptype offload, but now it is required. It means that application
will be broken and, therefore, it requires deprecation notice.
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [RFC 0/3] ethdev: add ptype as Rx offload
@ 2019-08-07 2:04 3% Jerin Jacob Kollanukkaran
2019-08-07 8:32 0% ` Andrew Rybchenko
0 siblings, 1 reply; 200+ results
From: Jerin Jacob Kollanukkaran @ 2019-08-07 2:04 UTC (permalink / raw)
To: Stephen Hemminger, Andrew Rybchenko
Cc: Pavan Nikhilesh Bhagavatula, Hemant Agrawal, dev
> -----Original Message-----
> From: Stephen Hemminger <stephen@networkplumber.org>
> Sent: Wednesday, August 7, 2019 4:45 AM
> To: Andrew Rybchenko <arybchenko@solarflare.com>
> Cc: Pavan Nikhilesh Bhagavatula <pbhagavatula@marvell.com>; Hemant
> Agrawal <hemant.agrawal@nxp.com>; Jerin Jacob Kollanukkaran
> <jerinj@marvell.com>; dev@dpdk.org
> Subject: [EXT] Re: [dpdk-dev] [RFC 0/3] ethdev: add ptype as Rx offload
>
> On Tue, 6 Aug 2019 12:06:35 +0300
> Andrew Rybchenko <arybchenko@solarflare.com> wrote:
>
> > On 8/6/19 11:47 AM, Pavan Nikhilesh Bhagavatula wrote:
> > >
> > >> -----Original Message-----
> > >> From: Hemant Agrawal <hemant.agrawal@nxp.com>
> > >> Sent: Tuesday, August 6, 2019 1:49 PM
> > >> To: Pavan Nikhilesh Bhagavatula <pbhagavatula@marvell.com>; Jerin
> > >> Jacob Kollanukkaran <jerinj@marvell.com>
> > >> Cc: dev@dpdk.org
> > >> Subject: RE: [dpdk-dev] [RFC 0/3] ethdev: add ptype as Rx offload
> > >>> Add PTYPE to DEV_RX_OFFLOAD_* flags.
> > >>>
> > >>> Currently, most of the NICs already support PTYPE parsing and
> > >>> update
> > >> the
> > >>> mbuf->packet_type through an internal lookup table, but there is
> > >>> mbuf->no
> > >> way to
> > >>> disable the lookup if the application is not intrested in ptypes
> > >> returned by
> > >>> `rte_eth_dev_get_supported_ptypes`.
> > >>>
> > >> [Hemant] it will also mean introducing another check in datapath,
> > >> if the application has asked for PTYPE offload - copy the results
> > >> to mbuf-
> > >>> packet_type otherwise don't do it.
> > > I think that having the check would give better performance than
> > > loading ptype table to L1 doing a lookup and copying it to mbuf when the
> application doesn't need it.
> >
> > Anyway, if PMD decides that it is better to always provide packet type
> > information - there is no harm. Basically if the offload is not
> > requested it makes packet_type undefined in mbuf.
> >
> > >> Your second patch is incomplete in the sense that it only adds the
> > >> capability. But it does not disable the lookups?
> > > It is upto the maintainer of the PMD to disable the lookup in data
> > > path. If there is a scope of optimization then they could do it. There is no
> harm in exposing PTYPE even RX_OFFLOAD_PTYPE is not enabled.
> > > I was hesitant to touch data path as it would be impossible to verify
> performance effect on all NICs.
> >
> > I think it is the right way to approach it especially taking
> > transition into account.
> >
>
> With hardline API policy, this has to fail on compile for old applications.
Not specific to this API change. That's is the propriety any new symbol addition
to the code base.
Planning to make this API change available from v19.11 LTS.
> You can't magically assume that applications using ptype will set new feature.
When OFFLOAD flags got introduced, we decided to disable all offloads by default.
So, need to add positive logic here to enable offload instead of enable something by
Default and disable if required to get have synergy with other offloads.
Will update the release note as usual to document the change.
Since there is NO ABI change, IMO, we don't need deprecation notice.
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] [PATCH 1/2] doc: announce ethdev ABI change for LRO fields
2019-08-06 14:56 4% [dpdk-dev] [PATCH 1/2] doc: announce ethdev ABI change for LRO fields Matan Azrad
@ 2019-08-06 15:27 4% ` Andrew Rybchenko
2019-08-10 21:40 4% ` Thomas Monjalon
0 siblings, 1 reply; 200+ results
From: Andrew Rybchenko @ 2019-08-06 15:27 UTC (permalink / raw)
To: Matan Azrad, dev
Cc: Thomas Monjalon, Ferruh Yigit, Konstantin Ananyev, Olivier Matz
On 8/6/19 5:56 PM, Matan Azrad wrote:
> It may be needed by the user to limit the LRO session packet size.
> In order to allow the above limitation, a new Rx configuration may be
> added for the maximum LRO session size.
>
> A new capability may be added too to expose the maximum LRO session size
> supported by the port.
>
> Signed-off-by: Matan Azrad <matan@mellanox.com>
> ---
> doc/guides/rel_notes/deprecation.rst | 4 ++++
> 1 file changed, 4 insertions(+)
>
> diff --git a/doc/guides/rel_notes/deprecation.rst b/doc/guides/rel_notes/deprecation.rst
> index 37b8592..c0cd9bc 100644
> --- a/doc/guides/rel_notes/deprecation.rst
> +++ b/doc/guides/rel_notes/deprecation.rst
> @@ -59,6 +59,10 @@ Deprecation Notices
> Target release for removal of the legacy API will be defined once most
> PMDs have switched to rte_flow.
>
> +* ethdev: new 32-bit fields may be added for maximum LRO session size, in
> + struct ``rte_eth_dev_info`` for the port capability and in struct
> + ``rte_eth_rxmode`` for the port configuration.
> +
> * cryptodev: support for using IV with all sizes is added, J0 still can
> be used but only when IV length in following structs ``rte_crypto_auth_xform``,
> ``rte_crypto_aead_xform`` is set to zero. When IV length is greater or equal
>
Acked-by: Andrew Rybchenko <arybchenko@solarflare.com>
I don't know examples when the information is required, but can imagine.
^ permalink raw reply [relevance 4%]
* [dpdk-dev] [PATCH 1/2] doc: announce ethdev ABI change for LRO fields
@ 2019-08-06 14:56 4% Matan Azrad
2019-08-06 15:27 4% ` Andrew Rybchenko
0 siblings, 1 reply; 200+ results
From: Matan Azrad @ 2019-08-06 14:56 UTC (permalink / raw)
To: dev
Cc: Thomas Monjalon, Ferruh Yigit, Andrew Rybchenko,
Konstantin Ananyev, Olivier Matz
It may be needed by the user to limit the LRO session packet size.
In order to allow the above limitation, a new Rx configuration may be
added for the maximum LRO session size.
A new capability may be added too to expose the maximum LRO session size
supported by the port.
Signed-off-by: Matan Azrad <matan@mellanox.com>
---
doc/guides/rel_notes/deprecation.rst | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/doc/guides/rel_notes/deprecation.rst b/doc/guides/rel_notes/deprecation.rst
index 37b8592..c0cd9bc 100644
--- a/doc/guides/rel_notes/deprecation.rst
+++ b/doc/guides/rel_notes/deprecation.rst
@@ -59,6 +59,10 @@ Deprecation Notices
Target release for removal of the legacy API will be defined once most
PMDs have switched to rte_flow.
+* ethdev: new 32-bit fields may be added for maximum LRO session size, in
+ struct ``rte_eth_dev_info`` for the port capability and in struct
+ ``rte_eth_rxmode`` for the port configuration.
+
* cryptodev: support for using IV with all sizes is added, J0 still can
be used but only when IV length in following structs ``rte_crypto_auth_xform``,
``rte_crypto_aead_xform`` is set to zero. When IV length is greater or equal
--
1.8.3.1
^ permalink raw reply [relevance 4%]
* Re: [dpdk-dev] [PATCH] doc: announce ABI change for RSS hash funtion
2019-08-06 11:34 4% ` Thomas Monjalon
@ 2019-08-06 14:45 4% ` Ananyev, Konstantin
2019-08-10 20:39 4% ` Thomas Monjalon
0 siblings, 1 reply; 200+ results
From: Ananyev, Konstantin @ 2019-08-06 14:45 UTC (permalink / raw)
To: Thomas Monjalon, Su, Simei
Cc: dev, Zhang, Qi Z, Wu, Jingjing, Xing, Beilei, Yang, Qiming,
orika, adrien.mazarguil, arybchenko, Yigit, Ferruh
> -----Original Message-----
> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Thomas Monjalon
> Sent: Tuesday, August 6, 2019 12:35 PM
> To: Su, Simei <simei.su@intel.com>
> Cc: dev@dpdk.org; Zhang, Qi Z <qi.z.zhang@intel.com>; Wu, Jingjing <jingjing.wu@intel.com>; Xing, Beilei <beilei.xing@intel.com>; Yang,
> Qiming <qiming.yang@intel.com>; orika@mellanox.com; adrien.mazarguil@6wind.com; arybchenko@solarflare.com; Yigit, Ferruh
> <ferruh.yigit@intel.com>
> Subject: Re: [dpdk-dev] [PATCH] doc: announce ABI change for RSS hash funtion
>
> 04/07/2019 06:43, simei:
> > From: Simei Su <simei.su@intel.com>
> >
> > Add new field SYMMETRIC_TOEPLITZ in rte_eth_hash_function. This
> > can support symmetric hash function by rte_flow RSS action.
> >
> > Signed-off-by: Simei Su <simei.su@intel.com>
> > ---
> > +* ethdev: New member in ``rte_eth_hash_funtion`` to support symmetric hash funtion.
>
> That's unfortunate there is a typo in the name of the enum you want to change.
>
> Do you have any reference to the algo you want to support? A paper maybe?
If I am not mistaken that's just an intent to enable symmetric RSS hash-function via standard RSS rte_flow API -
feature already available with i40e and newest Intel HW.
(AFAIK on i40e right now it could be configured via RTE_ETH_HASH_FILTER_SYM_HASH_ENA_PER_PORT).
If so, then I think the author may need to mention what PMDs will support that feature in 19.11.
Konstantin
>
>
^ permalink raw reply [relevance 4%]
* Re: [dpdk-dev] [PATCH v2 0/10] dpdk: introduce __rte_internal tag
2019-08-06 10:03 0% ` Thomas Monjalon
@ 2019-08-06 12:21 0% ` Neil Horman
0 siblings, 0 replies; 200+ results
From: Neil Horman @ 2019-08-06 12:21 UTC (permalink / raw)
To: Thomas Monjalon; +Cc: dev, Jerin Jacob Kollanukkaran, Bruce Richardson
On Tue, Aug 06, 2019 at 12:03:38PM +0200, Thomas Monjalon wrote:
> I think it would be good to rebase and send at the beginning of the 19.11 cycle.
> Thank you
>
I'm on PTO for the next 10 days or so, but yes, I'll take care of that asap.
Thanks
Neil
> 13/06/2019 16:23, Neil Horman:
> > Hey-
> > Based on our recent conversations regarding the use of symbols only
> > meant for internal dpdk consumption (between dpdk libraries), this is an idea
> > that I've come up with that I'd like to get some feedback on
> >
> > Summary:
> > 1) We have symbols in the DPDK that are meant to be used between DPDK libraries,
> > but not by applications linking to them
> > 2) We would like to document those symbols in the code, so as to note them
> > clearly as for being meant for internal use only
> > 3) Linker symbol visibility is a very coarse grained tool, and so there is no
> > good way in a single library to mark items as being meant for use only by other
> > DPDK libraries, at least not without some extensive runtime checking
> >
> >
> > Proposal:
> > I'm proposing that we introduce the __rte_internal tag. From a coding
> > standpoint it works a great deal like the __rte_experimental tag in that it
> > expempts the tagged symbol from ABI constraints (as the only users should be
> > represented in the DPDK build environment). Additionally, the __rte_internal
> > macro resolves differently based on the definition of the BUILDING_RTE_SDK flag
> > (working under the assumption that said flag should only ever be set if we are
> > actually building DPDK libraries which will make use of internal calls). If the
> > BUILDING_RTE_SDK flag is set __rte_internal resolves to __attribute__((section
> > "text.internal)), placing it in a special text section which is then used to
> > validate that the the symbol appears in the INTERNAL section of the
> > corresponding library version map). If BUILDING_RTE_SDK is not set, then
> > __rte_internal resolves to __attribute__((error("..."))), which causes any
> > caller of the tagged function to throw an error at compile time, indicating that
> > the symbol is not available for external use.
> >
> > This isn't a perfect solution, as applications can still hack around it of
> > course, but I think it hits some of the high points, restricting symbol access
> > for any library that prototypes its public and private symbols in the same
> > header file, excluding the internal symbols from ABI constraints, and clearly
> > documenting those symbols which we wish to limit to internal usage.
>
>
>
>
>
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [PATCH] doc: announce ABI change for rte flow RSS action
2019-08-06 11:31 4% ` Thomas Monjalon
2019-08-06 11:38 4% ` Zhang, Qi Z
@ 2019-08-06 11:40 4% ` Zhang, Qi Z
1 sibling, 0 replies; 200+ results
From: Zhang, Qi Z @ 2019-08-06 11:40 UTC (permalink / raw)
To: Thomas Monjalon, Su, Simei
Cc: dev, Wu, Jingjing, Xing, Beilei, Yang, Qiming, adrien.mazarguil, orika
> -----Original Message-----
> From: Zhang, Qi Z
> Sent: Tuesday, August 6, 2019 7:39 PM
> To: 'Thomas Monjalon' <thomas@monjalon.net>; Su, Simei
> <simei.su@intel.com>
> Cc: dev@dpdk.org; Wu, Jingjing <jingjing.wu@intel.com>; Xing, Beilei
> <beilei.xing@intel.com>; Yang, Qiming <qiming.yang@intel.com>;
> adrien.mazarguil@6wind.com; orika@mellanox.com
> Subject: RE: [dpdk-dev] [PATCH] doc: announce ABI change for rte flow RSS
> action
>
>
>
> > -----Original Message-----
> > From: Thomas Monjalon [mailto:thomas@monjalon.net]
> > Sent: Tuesday, August 6, 2019 7:31 PM
> > To: Su, Simei <simei.su@intel.com>
> > Cc: dev@dpdk.org; Zhang, Qi Z <qi.z.zhang@intel.com>; Wu, Jingjing
> > <jingjing.wu@intel.com>; Xing, Beilei <beilei.xing@intel.com>; Yang,
> > Qiming <qiming.yang@intel.com>; adrien.mazarguil@6wind.com;
> > orika@mellanox.com
> > Subject: Re: [dpdk-dev] [PATCH] doc: announce ABI change for rte flow
> > RSS action
> >
> > 04/07/2019 06:46, simei:
> > > From: Simei Su <simei.su@intel.com>
> > >
> > > Add new structure inputset in rte_flow_action_rss. This can support
> > > input set configuration by rte_flow RSS action.
> > >
> > > Signed-off-by: Simei Su <simei.su@intel.com>
> > > ---
> > > +* ethdev: New member in ``rte_flow_action_rss`` to support input
> > > +set change
> > > + by rte_flow RSS action. It ignores spec and focuses on mask only.
> >
> > There is no comment on this proposal.
> > I think it is missing some explanations about its usage.
> > Would you like to explain an use case and the rte_flow command?
> >
>
> This has been replaced by
> http://patches.dpdk.org/patch/56064/
Sorry, the latest RFC should be http://patchwork.dpdk.org/patch/57346/
>
> We don't need it now, I have updated patch status on patchwork.
>
> Thanks
> Qi
>
^ permalink raw reply [relevance 4%]
* Re: [dpdk-dev] [PATCH] doc: announce ABI change for rte flow RSS action
2019-08-06 11:31 4% ` Thomas Monjalon
@ 2019-08-06 11:38 4% ` Zhang, Qi Z
2019-08-06 11:40 4% ` Zhang, Qi Z
1 sibling, 0 replies; 200+ results
From: Zhang, Qi Z @ 2019-08-06 11:38 UTC (permalink / raw)
To: Thomas Monjalon, Su, Simei
Cc: dev, Wu, Jingjing, Xing, Beilei, Yang, Qiming, adrien.mazarguil, orika
> -----Original Message-----
> From: Thomas Monjalon [mailto:thomas@monjalon.net]
> Sent: Tuesday, August 6, 2019 7:31 PM
> To: Su, Simei <simei.su@intel.com>
> Cc: dev@dpdk.org; Zhang, Qi Z <qi.z.zhang@intel.com>; Wu, Jingjing
> <jingjing.wu@intel.com>; Xing, Beilei <beilei.xing@intel.com>; Yang, Qiming
> <qiming.yang@intel.com>; adrien.mazarguil@6wind.com;
> orika@mellanox.com
> Subject: Re: [dpdk-dev] [PATCH] doc: announce ABI change for rte flow RSS
> action
>
> 04/07/2019 06:46, simei:
> > From: Simei Su <simei.su@intel.com>
> >
> > Add new structure inputset in rte_flow_action_rss. This can support
> > input set configuration by rte_flow RSS action.
> >
> > Signed-off-by: Simei Su <simei.su@intel.com>
> > ---
> > +* ethdev: New member in ``rte_flow_action_rss`` to support input set
> > +change
> > + by rte_flow RSS action. It ignores spec and focuses on mask only.
>
> There is no comment on this proposal.
> I think it is missing some explanations about its usage.
> Would you like to explain an use case and the rte_flow command?
>
This has been replaced by
http://patches.dpdk.org/patch/56064/
We don't need it now, I have updated patch status on patchwork.
Thanks
Qi
^ permalink raw reply [relevance 4%]
* Re: [dpdk-dev] [PATCH] doc: announce ABI change for RSS hash funtion
@ 2019-08-06 11:34 4% ` Thomas Monjalon
2019-08-06 14:45 4% ` Ananyev, Konstantin
0 siblings, 1 reply; 200+ results
From: Thomas Monjalon @ 2019-08-06 11:34 UTC (permalink / raw)
To: simei
Cc: dev, qi.z.zhang, jingjing.wu, beilei.xing, qiming.yang, orika,
adrien.mazarguil, arybchenko, ferruh.yigit
04/07/2019 06:43, simei:
> From: Simei Su <simei.su@intel.com>
>
> Add new field SYMMETRIC_TOEPLITZ in rte_eth_hash_function. This
> can support symmetric hash function by rte_flow RSS action.
>
> Signed-off-by: Simei Su <simei.su@intel.com>
> ---
> +* ethdev: New member in ``rte_eth_hash_funtion`` to support symmetric hash funtion.
That's unfortunate there is a typo in the name of the enum you want to change.
Do you have any reference to the algo you want to support? A paper maybe?
^ permalink raw reply [relevance 4%]
* Re: [dpdk-dev] [PATCH] doc: announce ABI change for rte flow RSS action
@ 2019-08-06 11:31 4% ` Thomas Monjalon
2019-08-06 11:38 4% ` Zhang, Qi Z
2019-08-06 11:40 4% ` Zhang, Qi Z
0 siblings, 2 replies; 200+ results
From: Thomas Monjalon @ 2019-08-06 11:31 UTC (permalink / raw)
To: simei
Cc: dev, qi.z.zhang, jingjing.wu, beilei.xing, qiming.yang,
adrien.mazarguil, orika
04/07/2019 06:46, simei:
> From: Simei Su <simei.su@intel.com>
>
> Add new structure inputset in rte_flow_action_rss. This
> can support input set configuration by rte_flow RSS action.
>
> Signed-off-by: Simei Su <simei.su@intel.com>
> ---
> +* ethdev: New member in ``rte_flow_action_rss`` to support input set change
> + by rte_flow RSS action. It ignores spec and focuses on mask only.
There is no comment on this proposal.
I think it is missing some explanations about its usage.
Would you like to explain an use case and the rte_flow command?
^ permalink raw reply [relevance 4%]
* Re: [dpdk-dev] [PATCH v2 0/10] dpdk: introduce __rte_internal tag
@ 2019-08-06 10:03 0% ` Thomas Monjalon
2019-08-06 12:21 0% ` Neil Horman
0 siblings, 1 reply; 200+ results
From: Thomas Monjalon @ 2019-08-06 10:03 UTC (permalink / raw)
To: Neil Horman; +Cc: dev, Jerin Jacob Kollanukkaran, Bruce Richardson
I think it would be good to rebase and send at the beginning of the 19.11 cycle.
Thank you
13/06/2019 16:23, Neil Horman:
> Hey-
> Based on our recent conversations regarding the use of symbols only
> meant for internal dpdk consumption (between dpdk libraries), this is an idea
> that I've come up with that I'd like to get some feedback on
>
> Summary:
> 1) We have symbols in the DPDK that are meant to be used between DPDK libraries,
> but not by applications linking to them
> 2) We would like to document those symbols in the code, so as to note them
> clearly as for being meant for internal use only
> 3) Linker symbol visibility is a very coarse grained tool, and so there is no
> good way in a single library to mark items as being meant for use only by other
> DPDK libraries, at least not without some extensive runtime checking
>
>
> Proposal:
> I'm proposing that we introduce the __rte_internal tag. From a coding
> standpoint it works a great deal like the __rte_experimental tag in that it
> expempts the tagged symbol from ABI constraints (as the only users should be
> represented in the DPDK build environment). Additionally, the __rte_internal
> macro resolves differently based on the definition of the BUILDING_RTE_SDK flag
> (working under the assumption that said flag should only ever be set if we are
> actually building DPDK libraries which will make use of internal calls). If the
> BUILDING_RTE_SDK flag is set __rte_internal resolves to __attribute__((section
> "text.internal)), placing it in a special text section which is then used to
> validate that the the symbol appears in the INTERNAL section of the
> corresponding library version map). If BUILDING_RTE_SDK is not set, then
> __rte_internal resolves to __attribute__((error("..."))), which causes any
> caller of the tagged function to throw an error at compile time, indicating that
> the symbol is not available for external use.
>
> This isn't a perfect solution, as applications can still hack around it of
> course, but I think it hits some of the high points, restricting symbol access
> for any library that prototypes its public and private symbols in the same
> header file, excluding the internal symbols from ABI constraints, and clearly
> documenting those symbols which we wish to limit to internal usage.
^ permalink raw reply [relevance 0%]
* [dpdk-dev] [PATCH] doc: remove major in designation of normal releases
@ 2019-08-05 12:30 11% Thomas Monjalon
0 siblings, 0 replies; 200+ results
From: Thomas Monjalon @ 2019-08-05 12:30 UTC (permalink / raw)
To: John McNamara, Marko Kovacevic; +Cc: dev, orika
The word "major" was used to differentiate with release candidates
or stable maintenance releases.
However the word "major" can be understood as "LTS",
so it is less confusing to avoid this word.
Reported-by: Ori Kam <orika@mellanox.com>
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
---
doc/guides/contributing/documentation.rst | 2 +-
doc/guides/contributing/stable.rst | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/doc/guides/contributing/documentation.rst b/doc/guides/contributing/documentation.rst
index 27e4b13be..550d8dec2 100644
--- a/doc/guides/contributing/documentation.rst
+++ b/doc/guides/contributing/documentation.rst
@@ -62,7 +62,7 @@ added to by the developer.
The Release Notes document which features have been added in the current and previous releases of DPDK and highlight
any known issues.
- The Releases Notes also contain notifications of features that will change ABI compatibility in the next major release.
+ The Releases Notes also contain notifications of features that will change ABI compatibility in the next release.
Developers should include updates to the Release Notes with patch sets that relate to any of the following sections:
diff --git a/doc/guides/contributing/stable.rst b/doc/guides/contributing/stable.rst
index 6a5eee9bd..24b9e8b7d 100644
--- a/doc/guides/contributing/stable.rst
+++ b/doc/guides/contributing/stable.rst
@@ -25,7 +25,7 @@ Release to indicate longer term support.
Stable Releases
---------------
-Any major release of DPDK can be designated as a Stable Release if a
+Any release of DPDK can be designated as a Stable Release if a
maintainer volunteers to maintain it and there is a commitment from major
contributors to validate it before releases. If a release is to be designated
as a Stable Release, it should be done by 1 month after the master release.
--
2.21.0
^ permalink raw reply [relevance 11%]
* [dpdk-dev] [PATCH v2 3/3] doc: updates to versioning guide for abi versions
2019-08-02 15:38 11% [dpdk-dev] [PATCH v2 0/3] doc: changes to abi policy introducing major abi versions Ray Kinsella
2019-08-02 15:38 13% ` [dpdk-dev] [PATCH v2 1/3] doc: separate versioning.rst into version and policy Ray Kinsella
2019-08-02 15:38 32% ` [dpdk-dev] [PATCH v2 2/3] doc: changes to abi policy introducing major abi versions Ray Kinsella
@ 2019-08-02 15:38 28% ` Ray Kinsella
2 siblings, 0 replies; 200+ results
From: Ray Kinsella @ 2019-08-02 15:38 UTC (permalink / raw)
To: dev
Cc: mdr, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, nhorman,
maxime.coquelin, hemant.agrawal
Updates to the ABI versioning guide, to account for the changes to the DPDK
ABI/API policy.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
doc/guides/contributing/abi_versioning.rst | 240 +++++++++++++++++++----------
1 file changed, 159 insertions(+), 81 deletions(-)
diff --git a/doc/guides/contributing/abi_versioning.rst b/doc/guides/contributing/abi_versioning.rst
index 53e6ac0..38f5ec3 100644
--- a/doc/guides/contributing/abi_versioning.rst
+++ b/doc/guides/contributing/abi_versioning.rst
@@ -1,44 +1,128 @@
.. SPDX-License-Identifier: BSD-3-Clause
Copyright 2018 The DPDK contributors
-.. library_versioning:
-
-Library versioning
+DPDK ABI Versioning
+===================
+
+This document details the mechanics of ABI version management in DPDK.
+
+What is a library's soname?
+---------------------------
+
+System libraries usually adopt the familiar major and minor version naming
+convention, where major versions (e.g. ``librte_eal 20.x, 21.x``) are presumed
+to be ABI incompatible with each other and minor versions (e.g. ``librte_eal
+20.1, 20.2``) are presumed to be ABI compatible. A library's `soname
+<https://en.wikipedia.org/wiki/Soname>`_. is typically used to provide backward
+compatibility information about a given library, describing the lowest common
+denominator ABI supported by the library. The soname or logical name for the
+library, is typically comprised of the library's name and major version e.g.
+``librte_eal.so.20``.
+
+During an application's build process, a library's soname is noted as a runtime
+dependency of the application. This information is then used by the `dynamic
+linker <https://en.wikipedia.org/wiki/Dynamic_linker>`_ when resolving the
+applications dependencies at runtime, to load a library supporting the correct
+ABI version. The library loaded at runtime therefore, may be a minor revision
+supporting the same major abi version (e.g. ``librte_eal.20.2``), as the library
+used to link the application (e.g ``librte_eal.20.0``).
+
+Major ABI versions
------------------
-Downstreams might want to provide different DPDK releases at the same time to
-support multiple consumers of DPDK linked against older and newer sonames.
+An ABI version change to a given library, especially in core libraries such as
+``librte_mbuf``, may cause an implicit ripple effect on the ABI of it's
+dependent libraries, causing ABI breakages. There may however be no explicit
+reason to bump a dependent library's ABI version, as there may have been no
+obvious change to the dependent library's API, even though the library's ABI
+compatibility will have been broken.
+
+This interdependence of DPDK libraries, means that ABI versioning of libraries
+is more manageable at a project level, with all project libraries sharing a
+**single ABI version**. In addition, the need to maintain a stable ABI for some
+number of releases as described in the section `<The DPDK ABI policy>`_, means
+that ABI version increments need to carefully planned and managed at a project
+level.
+
+Major ABI versions are therefore declared typically aligned with an LTS release
+and is then supported some number of subsequent releases, shared across all
+libraries. This means that a single project level ABI version, reflected in all
+individual library's soname, library filenames and associated version maps
+persists over multiple releases.
+
+.. code-block:: none
-Also due to the interdependencies that DPDK libraries can have applications
-might end up with an executable space in which multiple versions of a library
-are mapped by ld.so.
+ $ head ./lib/librte_acl/rte_acl_version.map
+ DPDK_20.0 {
+ global:
+ ...
-Think of LibA that got an ABI bump and LibB that did not get an ABI bump but is
-depending on LibA.
+ $ head ./lib/librte_eal/rte_eal_version.map
+ DPDK_20.0 {
+ global:
+ ...
-.. note::
+When an ABI change is made between major ABI versions to a given library, a new
+section is added to that library's version map describing the impending new ABI
+version, as described in the section `<Examples of ABI Macro use>`_. The
+library's soname and filename however do not change, e.g. ``libacl.so.20``, as
+ABI compatibility with the last major ABI version continues to be preserved for
+that library.
- Application
- \-> LibA.old
- \-> LibB.new -> LibA.new
+.. code-block:: none
-That is a conflict which can be avoided by setting ``CONFIG_RTE_MAJOR_ABI``.
-If set, the value of ``CONFIG_RTE_MAJOR_ABI`` overwrites all - otherwise per
-library - versions defined in the libraries ``LIBABIVER``.
-An example might be ``CONFIG_RTE_MAJOR_ABI=16.11`` which will make all libraries
-``librte<?>.so.16.11`` instead of ``librte<?>.so.<LIBABIVER>``.
+ $ head ./lib/librte_acl/rte_acl_version.map
+ DPDK_20.0 {
+ global:
+ ...
+ DPDK_21.0 {
+ global:
+
+ } DPDK_20.0;
+ ...
+
+ $ head ./lib/librte_eal/rte_eal_version.map
+ DPDK_20.0 {
+ global:
+ ...
+
+However when a new ABI version is declared, for example DPDK 21.0, old
+depreciated functions may be safely removed at this point and the entire old
+major ABI version removed, see section `<Deprecating an entire ABI version>`_ on
+how this may be done.
+
+.. code-block:: none
+
+ $ head ./lib/librte_acl/rte_acl_version.map
+ DPDK_21.0 {
+ global:
+ ...
+
+ $ head -n 3 ./lib/librte_eal/rte_eal_version.map
+ DPDK_21.0 {
+ global:
+ ...
+
+At the same time, the major ABI version is changed atomically across all
+libraries by incrementing the major version in individual library's soname, e.g.
+``libacl.so.21``. This is done by bumping the LIBABIVER number in the libraries
+Makefile to indicate to dynamic linking applications that this is a later, and
+possibly incompatible library version:
+
+.. code-block:: c
+
+ -LIBABIVER := 20
+ +LIBABIVER := 21
-ABI versioning
---------------
Versioning Macros
-~~~~~~~~~~~~~~~~~
+-----------------
When a symbol is exported from a library to provide an API, it also provides a
calling convention (ABI) that is embodied in its name, return type and
arguments. Occasionally that function may need to change to accommodate new
-functionality or behavior. When that occurs, it is desirable to allow for
+functionality or behavior. When that occurs, it is may be required to allow for
backward compatibility for a time with older binaries that are dynamically
linked to the DPDK.
@@ -62,7 +146,7 @@ The macros exported are:
can still be mapped back to the public symbol name.
Examples of ABI Macro use
-^^^^^^^^^^^^^^^^^^^^^^^^^
+~~~~~~~~~~~~~~~~~~~~~~~~~
Updating a public API
_____________________
@@ -106,16 +190,16 @@ maintain previous ABI versions that are accessible only to previously compiled
binaries
The addition of a parameter to the function is ABI breaking as the function is
-public, and existing application may use it in its current form. However, the
+public, and existing application may use it in its current form. However, the
compatibility macros in DPDK allow a developer to use symbol versioning so that
multiple functions can be mapped to the same public symbol based on when an
-application was linked to it. To see how this is done, we start with the
-requisite libraries version map file. Initially the version map file for the
-acl library looks like this
+application was linked to it. To see how this is done, we start with the
+requisite libraries version map file. Initially the version map file for the acl
+library looks like this
.. code-block:: none
- DPDK_2.0 {
+ DPDK_20.0 {
global:
rte_acl_add_rules;
@@ -141,7 +225,7 @@ This file needs to be modified as follows
.. code-block:: none
- DPDK_2.0 {
+ DPDK_20.0 {
global:
rte_acl_add_rules;
@@ -163,16 +247,16 @@ This file needs to be modified as follows
local: *;
};
- DPDK_2.1 {
+ DPDK_21.0 {
global:
rte_acl_create;
- } DPDK_2.0;
+ } DPDK_20.0;
The addition of the new block tells the linker that a new version node is
-available (DPDK_2.1), which contains the symbol rte_acl_create, and inherits the
-symbols from the DPDK_2.0 node. This list is directly translated into a list of
-exported symbols when DPDK is compiled as a shared library
+available (DPDK_21.0), which contains the symbol rte_acl_create, and inherits
+the symbols from the DPDK_20.0 node. This list is directly translated into a
+list of exported symbols when DPDK is compiled as a shared library
Next, we need to specify in the code which function map to the rte_acl_create
symbol at which versions. First, at the site of the initial symbol definition,
@@ -191,22 +275,22 @@ with the public symbol name
Note that the base name of the symbol was kept intact, as this is conducive to
the macros used for versioning symbols. That is our next step, mapping this new
-symbol name to the initial symbol name at version node 2.0. Immediately after
+symbol name to the initial symbol name at version node 20.0. Immediately after
the function, we add this line of code
.. code-block:: c
- VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
+ VERSION_SYMBOL(rte_acl_create, _v20, 20.0);
Remembering to also add the rte_compat.h header to the requisite c file where
-these changes are being made. The above macro instructs the linker to create a
-new symbol ``rte_acl_create@DPDK_2.0``, which matches the symbol created in older
-builds, but now points to the above newly named function. We have now mapped
-the original rte_acl_create symbol to the original function (but with a new
-name)
+these changes are being made. The above macro instructs the linker to create a
+new symbol ``rte_acl_create@DPDK_20.0``, which matches the symbol created in
+older builds, but now points to the above newly named function. We have now
+mapped the original rte_acl_create symbol to the original function (but with a
+new name)
-Next, we need to create the 2.1 version of the symbol. We create a new function
-name, with a different suffix, and implement it appropriately
+Next, we need to create the 21.0 version of the symbol. We create a new function
+name, with a different suffix, and implement it appropriately
.. code-block:: c
@@ -220,12 +304,12 @@ name, with a different suffix, and implement it appropriately
return ctx;
}
-This code serves as our new API call. Its the same as our old call, but adds
-the new parameter in place. Next we need to map this function to the symbol
-``rte_acl_create@DPDK_2.1``. To do this, we modify the public prototype of the call
-in the header file, adding the macro there to inform all including applications,
-that on re-link, the default rte_acl_create symbol should point to this
-function. Note that we could do this by simply naming the function above
+This code serves as our new API call. Its the same as our old call, but adds the
+new parameter in place. Next we need to map this function to the symbol
+``rte_acl_create@DPDK_21.0``. To do this, we modify the public prototype of the
+call in the header file, adding the macro there to inform all including
+applications, that on re-link, the default rte_acl_create symbol should point to
+this function. Note that we could do this by simply naming the function above
rte_acl_create, and the linker would chose the most recent version tag to apply
in the version script, but we can also do this in the header file
@@ -233,11 +317,11 @@ in the version script, but we can also do this in the header file
struct rte_acl_ctx *
-rte_acl_create(const struct rte_acl_param *param);
- +rte_acl_create(const struct rte_acl_param *param, int debug);
- +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
+ +rte_acl_create_v21(const struct rte_acl_param *param, int debug);
+ +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 21.0);
The BIND_DEFAULT_SYMBOL macro explicitly tells applications that include this
-header, to link to the rte_acl_create_v21 function and apply the DPDK_2.1
+header, to link to the rte_acl_create_v21 function and apply the DPDK_21.0
version node to it. This method is more explicit and flexible than just
re-implementing the exact symbol name, and allows for other features (such as
linking to the old symbol version by default, when the new ABI is to be opt-in
@@ -257,6 +341,7 @@ assumption is that the most recent version of the symbol is the one you want to
map. So, back in the C file where, immediately after ``rte_acl_create_v21`` is
defined, we add this
+
.. code-block:: c
struct rte_acl_ctx *
@@ -270,21 +355,22 @@ That tells the compiler that, when building a static library, any calls to the
symbol ``rte_acl_create`` should be linked to ``rte_acl_create_v21``
That's it, on the next shared library rebuild, there will be two versions of
-rte_acl_create, an old DPDK_2.0 version, used by previously built applications,
-and a new DPDK_2.1 version, used by future built applications.
+rte_acl_create, an old DPDK_20.0 version, used by previously built applications,
+and a new DPDK_21.0 version, used by future built applications.
Deprecating part of a public API
________________________________
-Lets assume that you've done the above update, and after a few releases have
-passed you decide you would like to retire the old version of the function.
-After having gone through the ABI deprecation announcement process, removal is
-easy. Start by removing the symbol from the requisite version map file:
+Lets assume that you've done the above update, and in preparation for the next
+major ABI version you decide you would like to retire the old version of the
+function. After having gone through the ABI deprecation announcement process,
+removal is easy. Start by removing the symbol from the requisite version map
+file:
.. code-block:: none
- DPDK_2.0 {
+ DPDK_20.0 {
global:
rte_acl_add_rules;
@@ -306,10 +392,10 @@ easy. Start by removing the symbol from the requisite version map file:
local: *;
};
- DPDK_2.1 {
+ DPDK_21.0 {
global:
rte_acl_create;
- } DPDK_2.0;
+ } DPDK_20.0;
Next remove the corresponding versioned export.
@@ -320,34 +406,26 @@ Next remove the corresponding versioned export.
Note that the internal function definition could also be removed, but its used
-in our example by the newer version _v21, so we leave it in place. This is a
-coding style choice.
-
-Lastly, we need to bump the LIBABIVER number for this library in the Makefile to
-indicate to applications doing dynamic linking that this is a later, and
-possibly incompatible library version:
-
-.. code-block:: c
-
- -LIBABIVER := 1
- +LIBABIVER := 2
+in our example by the newer version _v21, so we leave it in place and declare it
+as static. This is a coding style choice.
Deprecating an entire ABI version
_________________________________
-While removing a symbol from and ABI may be useful, it is often more practical
-to remove an entire version node at once. If a version node completely
-specifies an API, then removing part of it, typically makes it incomplete. In
-those cases it is better to remove the entire node
+While removing a symbol from an ABI may be useful, it is more practical to
+remove an entire version node at once, as is typically done at the declaration
+of a major ABI version. If a version node completely specifies an API, then
+removing part of it, typically makes it incomplete. In those cases it is better
+to remove the entire node.
To do this, start by modifying the version map file, such that all symbols from
-the node to be removed are merged into the next node in the map
+the node to be removed are merged into the next node in the map.
In the case of our map above, it would transform to look as follows
.. code-block:: none
- DPDK_2.1 {
+ DPDK_21.0 {
global:
rte_acl_add_rules;
@@ -375,8 +453,8 @@ symbols.
.. code-block:: c
- -BIND_DEFAULT_SYMBOL(rte_acl_create, _v20, 2.0);
- +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
+ -BIND_DEFAULT_SYMBOL(rte_acl_create, _v20, 20.0);
+ +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 21.0);
Lastly, any VERSION_SYMBOL macros that point to the old version node should be
removed, taking care to keep, where need old code in place to support newer
--
2.7.4
^ permalink raw reply [relevance 28%]
* [dpdk-dev] [PATCH v2 2/3] doc: changes to abi policy introducing major abi versions
2019-08-02 15:38 11% [dpdk-dev] [PATCH v2 0/3] doc: changes to abi policy introducing major abi versions Ray Kinsella
2019-08-02 15:38 13% ` [dpdk-dev] [PATCH v2 1/3] doc: separate versioning.rst into version and policy Ray Kinsella
@ 2019-08-02 15:38 32% ` Ray Kinsella
2019-08-02 15:38 28% ` [dpdk-dev] [PATCH v2 3/3] doc: updates to versioning guide for " Ray Kinsella
2 siblings, 0 replies; 200+ results
From: Ray Kinsella @ 2019-08-02 15:38 UTC (permalink / raw)
To: dev
Cc: mdr, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, nhorman,
maxime.coquelin, hemant.agrawal
This policy change introduces major ABI versions, these are
declared every year, typically aligned with the LTS release
and are supported by subsequent releases in the following year.
This change is intended to improve ABI stabilty for those projects
consuming DPDK.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
doc/guides/contributing/abi_policy.rst | 272 +++++++++++++++++++++++----------
doc/guides/contributing/stable.rst | 36 +++--
2 files changed, 213 insertions(+), 95 deletions(-)
diff --git a/doc/guides/contributing/abi_policy.rst b/doc/guides/contributing/abi_policy.rst
index 55bacb4..fa78622 100644
--- a/doc/guides/contributing/abi_policy.rst
+++ b/doc/guides/contributing/abi_policy.rst
@@ -1,5 +1,5 @@
.. SPDX-License-Identifier: BSD-3-Clause
- Copyright 2018 The DPDK contributors
+ Copyright 2019 The DPDK contributors
.. abi_api_policy:
@@ -9,25 +9,37 @@ DPDK ABI/API policy
Description
-----------
-This document details some methods for handling ABI management in the DPDK.
+This document details the DPDK ABI/API management policy, to ensure the
+long-term stability of the DPDK ABI/API.
General Guidelines
------------------
-#. Whenever possible, ABI should be preserved
-#. ABI/API may be changed with a deprecation process
-#. The modification of symbols can generally be managed with versioning
-#. Libraries or APIs marked in ``experimental`` state may change without constraint
-#. New APIs will be marked as ``experimental`` for at least one release to allow
- any issues found by users of the new API to be fixed quickly
-#. The addition of symbols is generally not problematic
-#. The removal of symbols generally is an ABI break and requires bumping of the
- LIBABIVER macro
-#. Updates to the minimum hardware requirements, which drop support for hardware which
- was previously supported, should be treated as an ABI change.
-
-What is an ABI
-~~~~~~~~~~~~~~
+#. Major ABI versions are declared every **year** and are then supported
+ for one year, typically aligned with the LTS release.
+#. ABI versioning is managed at a project level in DPDK, with the supported ABI
+ version reflected in all library's soname.
+#. The ABI should be preserved and not changed lightly. The ABI may still be
+ changed, by following the outlined deprecation process.
+#. The addition of symbols is generally not problematic. The modification of
+ symbols should be managed with ABI versioning.
+#. The removal of symbols is considered an ABI breakage, once approved these
+ will form part of the next ABI version.
+#. Libraries or APIs marked as ``experimental`` are not considered part of the
+ major ABI version and may change without constraint.
+#. Updates to the minimum hardware requirements, which drop support for hardware
+ which was previously supported, should be treated as an ABI change.
+
+.. note::
+
+ Note that in 2019 the DPDK community stated it's intention to move to ABI
+ stable releases over a number of releases. Beginning with maintaining ABI
+ stability through one year of DPDK releases starting from DPDK 19.11. This
+ policy will be reviewed in 2020 with intention of lengthening the stability
+ period.
+
+What is an ABI?
+~~~~~~~~~~~~~~~
An ABI (Application Binary Interface) is the set of runtime interfaces exposed
by a library. It is similar to an API (Application Programming Interface) but
@@ -39,30 +51,40 @@ Therefore, in the case of dynamic linking, it is critical that an ABI is
preserved, or (when modified), done in such a way that the application is unable
to behave improperly or in an unexpected fashion.
-
-ABI/API Deprecation
+The DPDK ABI policy
-------------------
-The DPDK ABI policy
-~~~~~~~~~~~~~~~~~~~
+A major ABI version is declared every years, aligned with that years LTS
+release, e.g. v19.11 . This ABI version is then supported for one year by all
+subsequent releases within that time period, until the next LTS release, e.g.
+v20.11.
+
+At the declaration of a major ABI version, major version numbers encoded in
+libraries soname's are bumped to indicate the new version, with minor version
+reset to 0. An example would be ``librte_eal.so.20.3`` would become
+``librte_eal.so.21.0``.
+
+The ABI may then change multiple times, without warning, between the last major
+ABI version increment and the HEAD label of the git tree, with the condition
+that ABI compatibility with the major ABI version is preserved and therefore
+soname's do not change.
-ABI versions are set at the time of major release labeling, and the ABI may
-change multiple times, without warning, between the last release label and the
-HEAD label of the git tree.
+Minor versions are incremented to indicate the release of a new ABI compatible
+DPDK release, typically the DPDK quarterly releases. An example of this, might
+be that ``librte_eal.so.20.1`` would indicate the first ABI compatible DPDK
+release, following the declaration of the new major ABI version 20.0.
-ABI versions, once released, are available until such time as their
-deprecation has been noted in the Release Notes for at least one major release
-cycle. For example consider the case where the ABI for DPDK 2.0 has been
-shipped and then a decision is made to modify it during the development of
-DPDK 2.1. The decision will be recorded in the Release Notes for the DPDK 2.1
-release and the modification will be made available in the DPDK 2.2 release.
+ABI versions, are supported by each release until such time as the next major
+ABI version is declared. At that time, the deprecation of the previous major ABI
+version will be noted in the Release Notes with guidance on individual symbol
+depreciation and upgrade notes provided.
-ABI versions may be deprecated in whole or in part as needed by a given
-update.
+ABI Changes
+~~~~~~~~~~~
-Some ABI changes may be too significant to reasonably maintain multiple
-versions. In those cases ABI's may be updated without backward compatibility
-being provided. The requirements for doing so are:
+The ABI may still change after the declaration of a major ABI version, that is
+new APIs may be still added or existing APIs may be modified. The requirements
+for doing so are:
#. At least 3 acknowledgments of the need to do so must be made on the
dpdk.org mailing list.
@@ -71,81 +93,165 @@ being provided. The requirements for doing so are:
no maintainer is available for the component, the tree/sub-tree maintainer
for that component must acknowledge the ABI change instead.
+ - The acknowledgment of a member of the technical board, as a delegate of the
+ `technical board <https://core.dpdk.org/techboard/>`_ acknowledging the
+ need for the ABI change, is also mandatory.
+
- It is also recommended that acknowledgments from different "areas of
interest" be sought for each deprecation, for example: from NIC vendors,
CPU vendors, end-users, etc.
-#. The changes (including an alternative map file) can be included with
- deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option,
- to provide more details about oncoming changes.
- ``RTE_NEXT_ABI`` wrapper will be removed when it become the default ABI.
- More preferred way to provide this information is sending the feature
- as a separate patch and reference it in deprecation notice.
+#. Backward compatibly with the major ABI version must be maintained through
+ `<ABI versioning>`_, with forward-only compatibility offered for any ABI
+ changes that are indicated to be part of the next ABI version.
+
+ - In situations were backward compatibility is not possible, read the
+ section `<ABI Breakages>`_.
-#. A full deprecation cycle, as explained above, must be made to offer
- downstream consumers sufficient warning of the change.
+ - No backward or forward compatibility is offered for API changes marked as
+ ``experimental``, as described in the section `<Experimental APIs>`_.
-Note that the above process for ABI deprecation should not be undertaken
-lightly. ABI stability is extremely important for downstream consumers of the
-DPDK, especially when distributed in shared object form. Every effort should
-be made to preserve the ABI whenever possible. The ABI should only be changed
-for significant reasons, such as performance enhancements. ABI breakage due to
-changes such as reorganizing public structure fields for aesthetic or
-readability purposes should be avoided.
+#. If a newly proposed API functionally replaces an existing one, when the new
+ API becomes non-experimental, then the old one is marked with
+ ``__rte_deprecated``.
+
+ - The depreciated API should follow the notification process to be removed,
+ see `<Examples of Deprecation Notices>`_;
+
+ - At the declaration of the next major ABI version, those ABI changes then
+ become a formal part of the new ABI and the requirement to preserve ABI
+ compatibility with the last major ABI version is then dropped.
+
+ - The responsibility for removing ABI compatibility preserving code rests
+ with the original contributor of the ABI changes, then with the
+ contributor's company and then finally with the maintainer.
+
+.. note::
+
+ Note that the above process for ABI deprecation should not be undertaken
+ lightly. ABI stability is extremely important for downstream consumers of the
+ DPDK, especially when distributed in shared object form. Every effort should
+ be made to preserve the ABI whenever possible. The ABI should only be changed
+ for significant reasons, such as performance enhancements. ABI breakage due
+ to changes such as reorganizing public structure fields for aesthetic or
+ readability purposes should be avoided.
+
+.. _forward-only:
+
+.. note::
+
+ Note that forward-only compatibility is offered for those changes made
+ between major ABI versions. As a library's soname however only describes
+ compatibility with the last major ABI version, until the next major ABI
+ version is declared, these changes therefore cannot be resolved as a runtime
+ dependency through the soname. Therefore any application wishing to make use
+ of these ABI changes can only ensure that it's runtime dependencies are met
+ through Operating System package versioning.
.. note::
Updates to the minimum hardware requirements, which drop support for hardware
which was previously supported, should be treated as an ABI change, and
- follow the relevant deprecation policy procedures as above: 3 acks and
- announcement at least one release in advance.
+ follow the relevant deprecation policy procedures as above: 3 acks, technical
+ board approval and announcement at least one release in advance.
+
+
+ABI Breakages
+^^^^^^^^^^^^^
+
+For those ABI changes that may be too significant to reasonably maintain
+multiple versions. In those cases, ABIs may be updated without backward
+compatibility being provided.
+
+The additional requirements to approve an ABI breakage, on top of those
+that described in the section `<ABI Changes>`_ are:
+
+#. ABI breaking changes (including an alternative map file) can be included with
+ deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option, to provide
+ more details about oncoming changes. ``RTE_NEXT_ABI`` wrapper will be removed
+ at the declaration of the next major ABI version.
+
+#. Once approved and after the depreciation notice has been observed these
+ changes will form part of the next declared major ABI version.
+
+Examples of ABI Changes
+^^^^^^^^^^^^^^^^^^^^^^^
+
+The following are examples of allowable ABI changes occurring between
+declarations of major ABI versions.
+
+* DPDK 19.11 release, defines the function ``rte_foo()``, and ``rte_foo()``
+ as part of the major ABI version DPDK 20.0.
+
+* DPDK 20.02 release defines a new function ``rte_foo(uint8_t bar)``, and
+ this is not a problem as long as the symbol ``rte_foo@DPDK20.0`` is
+ preserved through `<ABI versioning>`_.
+
+ - The new function may be marked with the ``__rte_experimental`` tag for a
+ number of releases, as described in the section `<Experimental APIs>`_;
+
+ - Once ``rte_foo(uint8_t bar)`` becomes non-experimental ``rte_foo()`` is then
+ declared as ``__rte_depreciated``, with an associated deprecation notice
+ provided.
+
+* DPDK 19.11 is not re-released to include ``rte_foo(uint8_t bar)``, the new
+ version of ``rte_foo`` only exists from DPDK 20.02 onwards as described in the
+ note on `forward-only`_ compatibility.
+
+* DPDK 20.02 release defines the experimental function ``__rte_experimental
+ rte_baz()``. This function may or may not exist in the DPDK 20.05 release.
+
+* An application ``dPacket`` wishes to use ``rte_foo(uint8_t bar)``, before the
+ declaration of the DPDK 21.0 major API version. The application can only
+ ensure it's runtime dependencies are met by specifying ``DPDK (>= 20.2)`` as
+ an explicit package dependency, as the soname only may only indicate the
+ supporting major ABI version.
+
+* At the release of DPDK 20.11, the function ``rte_foo(uint8_t bar)`` becomes
+ formally part of then new major ABI version DPDK 21.0 and ``rte_foo()`` may be
+ removed.
+
Examples of Deprecation Notices
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
The following are some examples of ABI deprecation notices which would be
added to the Release Notes:
-* The Macro ``#RTE_FOO`` is deprecated and will be removed with version 2.0,
- to be replaced with the inline function ``rte_foo()``.
+* The Macro ``#RTE_FOO`` is deprecated and will be removed with ABI version
+ 21.0, to be replaced with the inline function ``rte_foo()``.
* The function ``rte_mbuf_grok()`` has been updated to include a new parameter
- in version 2.0. Backwards compatibility will be maintained for this function
- until the release of version 2.1
+ in version 20.2. Backwards compatibility will be maintained for this function
+ until the release of the new DPDK major ABI version 21.0, in DPDK version
+ 20.11.
-* The members of ``struct rte_foo`` have been reorganized in release 2.0 for
+* The members of ``struct rte_foo`` have been reorganized in DPDK 20.02 for
performance reasons. Existing binary applications will have backwards
- compatibility in release 2.0, while newly built binaries will need to
- reference the new structure variant ``struct rte_foo2``. Compatibility will
- be removed in release 2.2, and all applications will require updating and
+ compatibility in release 20.02, while newly built binaries will need to
+ reference the new structure variant ``struct rte_foo2``. Compatibility will be
+ removed in release 20.11, and all applications will require updating and
rebuilding to the new structure at that time, which will be renamed to the
original ``struct rte_foo``.
* Significant ABI changes are planned for the ``librte_dostuff`` library. The
- upcoming release 2.0 will not contain these changes, but release 2.1 will,
+ upcoming release 20.02 will not contain these changes, but release 20.11 will,
and no backwards compatibility is planned due to the extensive nature of
- these changes. Binaries using this library built prior to version 2.1 will
+ these changes. Binaries using this library built prior to ABI version 21 will
require updating and recompilation.
-New API replacing previous one
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Experimental
+------------
-If a new API proposed functionally replaces an existing one, when the new API
-becomes non-experimental then the old one is marked with ``__rte_deprecated``.
-Deprecated APIs are removed completely just after the next LTS.
+APIs
+~~~~
-Reminder that old API should follow deprecation process to be removed.
-
-
-Experimental APIs
------------------
-
-APIs marked as ``experimental`` are not considered part of the ABI and may
-change without warning at any time. Since changes to APIs are most likely
-immediately after their introduction, as users begin to take advantage of
-those new APIs and start finding issues with them, new DPDK APIs will be
-automatically marked as ``experimental`` to allow for a period of stabilization
-before they become part of a tracked ABI.
+APIs marked as ``experimental`` are not considered part of an ABI version and
+may change without warning at any time. Since changes to APIs are most likely
+immediately after their introduction, as users begin to take advantage of those
+new APIs and start finding issues with them, new DPDK APIs will be automatically
+marked as ``experimental`` to allow for a period of stabilization before they
+become part of a tracked ABI version.
Note that marking an API as experimental is a multi step process.
To mark an API as experimental, the symbols which are desired to be exported
@@ -167,3 +273,11 @@ For removing the experimental tag associated with an API, deprecation notice
is not required. Though, an API should remain in experimental state for at least
one release. Thereafter, normal process of posting patch for review to mailing
list can be followed.
+
+Libraries
+~~~~~~~~~
+
+Libraries marked as ``experimental`` are entirely not considered part of an ABI
+version, and may change without warning at any time. Experimental libraries
+always have a major version of ``0`` to indicate they exist outside of ABI
+versioning, with minor version incremented with each ABI change to library.
diff --git a/doc/guides/contributing/stable.rst b/doc/guides/contributing/stable.rst
index 6a5eee9..db783c7 100644
--- a/doc/guides/contributing/stable.rst
+++ b/doc/guides/contributing/stable.rst
@@ -53,6 +53,9 @@ year's November (X.11) release will be maintained as an LTS for 2 years.
After the X.11 release, an LTS branch will be created for it at
http://git.dpdk.org/dpdk-stable where bugfixes will be backported to.
+A LTS release may align with the declaration of a new major ABI version,
+please read the `<DPDK ABI/API policy>`_ for more information.
+
It is anticipated that there will be at least 4 releases per year of the LTS
or approximately 1 every 3 months. However, the cadence can be shorter or
longer depending on the number and criticality of the backported
@@ -68,10 +71,13 @@ point the LTS branch will no longer be maintained with no further releases.
What changes should be backported
---------------------------------
-Backporting should be limited to bug fixes. All patches accepted on the master
-branch with a Fixes: tag should be backported to the relevant stable/LTS
-branches, unless the submitter indicates otherwise. If there are exceptions,
-they will be discussed on the mailing lists.
+Backporting is a naturally conservative activity, and therefore should only
+include bug fixes and support for new hardware, were adding support does not
+necessitate DPDK ABI/API changes.
+
+All patches accepted on the master branch with a Fixes: tag should be backported
+to the relevant stable/LTS branches, unless the submitter indicates otherwise.
+If there are exceptions, they will be discussed on the mailing lists.
Fixes suitable for backport should have a ``Cc: stable@dpdk.org`` tag in the
commit message body as follows::
@@ -86,13 +92,18 @@ commit message body as follows::
Signed-off-by: Alex Smith <alex.smith@example.com>
-Fixes not suitable for backport should not include the ``Cc: stable@dpdk.org`` tag.
+Fixes not suitable for backport should not include the ``Cc: stable@dpdk.org``
+tag.
-Features should not be backported to stable releases. It may be acceptable, in
-limited cases, to back port features for the LTS release where:
+New features, with the exception of new hardware support, should not be
+backported to stable releases. In the case of new hardware support or any other
+exceptional circumstances limited backporting maybe permitted to the LTS release
+where:
-* There is a justifiable use case (for example a new PMD).
-* The change is non-invasive.
+* There is a justifiable use case, for example the change is required to support
+ a new platform or device (for example a new PMD).
+* The change is ABI/API preserving, it does not present an obvious "new feature"
+ to end consumer.
* The work of preparing the backport is done by the proposer.
* There is support within the community.
@@ -119,10 +130,3 @@ A Stable Release will be released by:
list.
Stable releases are available on the `dpdk.org download page <http://core.dpdk.org/download/>`_.
-
-
-ABI
----
-
-The Stable Release should not be seen as a way of breaking or circumventing
-the DPDK ABI policy.
--
2.7.4
^ permalink raw reply [relevance 32%]
* [dpdk-dev] [PATCH v2 1/3] doc: separate versioning.rst into version and policy
2019-08-02 15:38 11% [dpdk-dev] [PATCH v2 0/3] doc: changes to abi policy introducing major abi versions Ray Kinsella
@ 2019-08-02 15:38 13% ` Ray Kinsella
2019-08-02 15:38 32% ` [dpdk-dev] [PATCH v2 2/3] doc: changes to abi policy introducing major abi versions Ray Kinsella
2019-08-02 15:38 28% ` [dpdk-dev] [PATCH v2 3/3] doc: updates to versioning guide for " Ray Kinsella
2 siblings, 0 replies; 200+ results
From: Ray Kinsella @ 2019-08-02 15:38 UTC (permalink / raw)
To: dev
Cc: mdr, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, nhorman,
maxime.coquelin, hemant.agrawal
Separate versioning.rst into abi versioning and abi policy guidance, in
preparation for adding more detail to the abi policy.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
doc/guides/contributing/abi_policy.rst | 169 +++++++++
doc/guides/contributing/abi_versioning.rst | 427 +++++++++++++++++++++
doc/guides/contributing/index.rst | 3 +-
doc/guides/contributing/versioning.rst | 591 -----------------------------
4 files changed, 598 insertions(+), 592 deletions(-)
create mode 100644 doc/guides/contributing/abi_policy.rst
create mode 100644 doc/guides/contributing/abi_versioning.rst
delete mode 100644 doc/guides/contributing/versioning.rst
diff --git a/doc/guides/contributing/abi_policy.rst b/doc/guides/contributing/abi_policy.rst
new file mode 100644
index 0000000..55bacb4
--- /dev/null
+++ b/doc/guides/contributing/abi_policy.rst
@@ -0,0 +1,169 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+ Copyright 2018 The DPDK contributors
+
+.. abi_api_policy:
+
+DPDK ABI/API policy
+===================
+
+Description
+-----------
+
+This document details some methods for handling ABI management in the DPDK.
+
+General Guidelines
+------------------
+
+#. Whenever possible, ABI should be preserved
+#. ABI/API may be changed with a deprecation process
+#. The modification of symbols can generally be managed with versioning
+#. Libraries or APIs marked in ``experimental`` state may change without constraint
+#. New APIs will be marked as ``experimental`` for at least one release to allow
+ any issues found by users of the new API to be fixed quickly
+#. The addition of symbols is generally not problematic
+#. The removal of symbols generally is an ABI break and requires bumping of the
+ LIBABIVER macro
+#. Updates to the minimum hardware requirements, which drop support for hardware which
+ was previously supported, should be treated as an ABI change.
+
+What is an ABI
+~~~~~~~~~~~~~~
+
+An ABI (Application Binary Interface) is the set of runtime interfaces exposed
+by a library. It is similar to an API (Application Programming Interface) but
+is the result of compilation. It is also effectively cloned when applications
+link to dynamic libraries. That is to say when an application is compiled to
+link against dynamic libraries, it is assumed that the ABI remains constant
+between the time the application is compiled/linked, and the time that it runs.
+Therefore, in the case of dynamic linking, it is critical that an ABI is
+preserved, or (when modified), done in such a way that the application is unable
+to behave improperly or in an unexpected fashion.
+
+
+ABI/API Deprecation
+-------------------
+
+The DPDK ABI policy
+~~~~~~~~~~~~~~~~~~~
+
+ABI versions are set at the time of major release labeling, and the ABI may
+change multiple times, without warning, between the last release label and the
+HEAD label of the git tree.
+
+ABI versions, once released, are available until such time as their
+deprecation has been noted in the Release Notes for at least one major release
+cycle. For example consider the case where the ABI for DPDK 2.0 has been
+shipped and then a decision is made to modify it during the development of
+DPDK 2.1. The decision will be recorded in the Release Notes for the DPDK 2.1
+release and the modification will be made available in the DPDK 2.2 release.
+
+ABI versions may be deprecated in whole or in part as needed by a given
+update.
+
+Some ABI changes may be too significant to reasonably maintain multiple
+versions. In those cases ABI's may be updated without backward compatibility
+being provided. The requirements for doing so are:
+
+#. At least 3 acknowledgments of the need to do so must be made on the
+ dpdk.org mailing list.
+
+ - The acknowledgment of the maintainer of the component is mandatory, or if
+ no maintainer is available for the component, the tree/sub-tree maintainer
+ for that component must acknowledge the ABI change instead.
+
+ - It is also recommended that acknowledgments from different "areas of
+ interest" be sought for each deprecation, for example: from NIC vendors,
+ CPU vendors, end-users, etc.
+
+#. The changes (including an alternative map file) can be included with
+ deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option,
+ to provide more details about oncoming changes.
+ ``RTE_NEXT_ABI`` wrapper will be removed when it become the default ABI.
+ More preferred way to provide this information is sending the feature
+ as a separate patch and reference it in deprecation notice.
+
+#. A full deprecation cycle, as explained above, must be made to offer
+ downstream consumers sufficient warning of the change.
+
+Note that the above process for ABI deprecation should not be undertaken
+lightly. ABI stability is extremely important for downstream consumers of the
+DPDK, especially when distributed in shared object form. Every effort should
+be made to preserve the ABI whenever possible. The ABI should only be changed
+for significant reasons, such as performance enhancements. ABI breakage due to
+changes such as reorganizing public structure fields for aesthetic or
+readability purposes should be avoided.
+
+.. note::
+
+ Updates to the minimum hardware requirements, which drop support for hardware
+ which was previously supported, should be treated as an ABI change, and
+ follow the relevant deprecation policy procedures as above: 3 acks and
+ announcement at least one release in advance.
+
+Examples of Deprecation Notices
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The following are some examples of ABI deprecation notices which would be
+added to the Release Notes:
+
+* The Macro ``#RTE_FOO`` is deprecated and will be removed with version 2.0,
+ to be replaced with the inline function ``rte_foo()``.
+
+* The function ``rte_mbuf_grok()`` has been updated to include a new parameter
+ in version 2.0. Backwards compatibility will be maintained for this function
+ until the release of version 2.1
+
+* The members of ``struct rte_foo`` have been reorganized in release 2.0 for
+ performance reasons. Existing binary applications will have backwards
+ compatibility in release 2.0, while newly built binaries will need to
+ reference the new structure variant ``struct rte_foo2``. Compatibility will
+ be removed in release 2.2, and all applications will require updating and
+ rebuilding to the new structure at that time, which will be renamed to the
+ original ``struct rte_foo``.
+
+* Significant ABI changes are planned for the ``librte_dostuff`` library. The
+ upcoming release 2.0 will not contain these changes, but release 2.1 will,
+ and no backwards compatibility is planned due to the extensive nature of
+ these changes. Binaries using this library built prior to version 2.1 will
+ require updating and recompilation.
+
+New API replacing previous one
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If a new API proposed functionally replaces an existing one, when the new API
+becomes non-experimental then the old one is marked with ``__rte_deprecated``.
+Deprecated APIs are removed completely just after the next LTS.
+
+Reminder that old API should follow deprecation process to be removed.
+
+
+Experimental APIs
+-----------------
+
+APIs marked as ``experimental`` are not considered part of the ABI and may
+change without warning at any time. Since changes to APIs are most likely
+immediately after their introduction, as users begin to take advantage of
+those new APIs and start finding issues with them, new DPDK APIs will be
+automatically marked as ``experimental`` to allow for a period of stabilization
+before they become part of a tracked ABI.
+
+Note that marking an API as experimental is a multi step process.
+To mark an API as experimental, the symbols which are desired to be exported
+must be placed in an EXPERIMENTAL version block in the corresponding libraries'
+version map script.
+Secondly, the corresponding prototypes of those exported functions (in the
+development header files), must be marked with the ``__rte_experimental`` tag
+(see ``rte_compat.h``).
+The DPDK build makefiles perform a check to ensure that the map file and the
+C code reflect the same list of symbols.
+This check can be circumvented by defining ``ALLOW_EXPERIMENTAL_API``
+during compilation in the corresponding library Makefile.
+
+In addition to tagging the code with ``__rte_experimental``,
+the doxygen markup must also contain the EXPERIMENTAL string,
+and the MAINTAINERS file should note the EXPERIMENTAL libraries.
+
+For removing the experimental tag associated with an API, deprecation notice
+is not required. Though, an API should remain in experimental state for at least
+one release. Thereafter, normal process of posting patch for review to mailing
+list can be followed.
diff --git a/doc/guides/contributing/abi_versioning.rst b/doc/guides/contributing/abi_versioning.rst
new file mode 100644
index 0000000..53e6ac0
--- /dev/null
+++ b/doc/guides/contributing/abi_versioning.rst
@@ -0,0 +1,427 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+ Copyright 2018 The DPDK contributors
+
+.. library_versioning:
+
+Library versioning
+------------------
+
+Downstreams might want to provide different DPDK releases at the same time to
+support multiple consumers of DPDK linked against older and newer sonames.
+
+Also due to the interdependencies that DPDK libraries can have applications
+might end up with an executable space in which multiple versions of a library
+are mapped by ld.so.
+
+Think of LibA that got an ABI bump and LibB that did not get an ABI bump but is
+depending on LibA.
+
+.. note::
+
+ Application
+ \-> LibA.old
+ \-> LibB.new -> LibA.new
+
+That is a conflict which can be avoided by setting ``CONFIG_RTE_MAJOR_ABI``.
+If set, the value of ``CONFIG_RTE_MAJOR_ABI`` overwrites all - otherwise per
+library - versions defined in the libraries ``LIBABIVER``.
+An example might be ``CONFIG_RTE_MAJOR_ABI=16.11`` which will make all libraries
+``librte<?>.so.16.11`` instead of ``librte<?>.so.<LIBABIVER>``.
+
+
+ABI versioning
+--------------
+
+Versioning Macros
+~~~~~~~~~~~~~~~~~
+
+When a symbol is exported from a library to provide an API, it also provides a
+calling convention (ABI) that is embodied in its name, return type and
+arguments. Occasionally that function may need to change to accommodate new
+functionality or behavior. When that occurs, it is desirable to allow for
+backward compatibility for a time with older binaries that are dynamically
+linked to the DPDK.
+
+To support backward compatibility the ``rte_compat.h``
+header file provides macros to use when updating exported functions. These
+macros are used in conjunction with the ``rte_<library>_version.map`` file for
+a given library to allow multiple versions of a symbol to exist in a shared
+library so that older binaries need not be immediately recompiled.
+
+The macros exported are:
+
+* ``VERSION_SYMBOL(b, e, n)``: Creates a symbol version table entry binding
+ versioned symbol ``b@DPDK_n`` to the internal function ``b_e``.
+
+* ``BIND_DEFAULT_SYMBOL(b, e, n)``: Creates a symbol version entry instructing
+ the linker to bind references to symbol ``b`` to the internal symbol
+ ``b_e``.
+
+* ``MAP_STATIC_SYMBOL(f, p)``: Declare the prototype ``f``, and map it to the
+ fully qualified function ``p``, so that if a symbol becomes versioned, it
+ can still be mapped back to the public symbol name.
+
+Examples of ABI Macro use
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Updating a public API
+_____________________
+
+Assume we have a function as follows
+
+.. code-block:: c
+
+ /*
+ * Create an acl context object for apps to
+ * manipulate
+ */
+ struct rte_acl_ctx *
+ rte_acl_create(const struct rte_acl_param *param)
+ {
+ ...
+ }
+
+
+Assume that struct rte_acl_ctx is a private structure, and that a developer
+wishes to enhance the acl api so that a debugging flag can be enabled on a
+per-context basis. This requires an addition to the structure (which, being
+private, is safe), but it also requires modifying the code as follows
+
+.. code-block:: c
+
+ /*
+ * Create an acl context object for apps to
+ * manipulate
+ */
+ struct rte_acl_ctx *
+ rte_acl_create(const struct rte_acl_param *param, int debug)
+ {
+ ...
+ }
+
+
+Note also that, being a public function, the header file prototype must also be
+changed, as must all the call sites, to reflect the new ABI footprint. We will
+maintain previous ABI versions that are accessible only to previously compiled
+binaries
+
+The addition of a parameter to the function is ABI breaking as the function is
+public, and existing application may use it in its current form. However, the
+compatibility macros in DPDK allow a developer to use symbol versioning so that
+multiple functions can be mapped to the same public symbol based on when an
+application was linked to it. To see how this is done, we start with the
+requisite libraries version map file. Initially the version map file for the
+acl library looks like this
+
+.. code-block:: none
+
+ DPDK_2.0 {
+ global:
+
+ rte_acl_add_rules;
+ rte_acl_build;
+ rte_acl_classify;
+ rte_acl_classify_alg;
+ rte_acl_classify_scalar;
+ rte_acl_create;
+ rte_acl_dump;
+ rte_acl_find_existing;
+ rte_acl_free;
+ rte_acl_ipv4vlan_add_rules;
+ rte_acl_ipv4vlan_build;
+ rte_acl_list_dump;
+ rte_acl_reset;
+ rte_acl_reset_rules;
+ rte_acl_set_ctx_classify;
+
+ local: *;
+ };
+
+This file needs to be modified as follows
+
+.. code-block:: none
+
+ DPDK_2.0 {
+ global:
+
+ rte_acl_add_rules;
+ rte_acl_build;
+ rte_acl_classify;
+ rte_acl_classify_alg;
+ rte_acl_classify_scalar;
+ rte_acl_create;
+ rte_acl_dump;
+ rte_acl_find_existing;
+ rte_acl_free;
+ rte_acl_ipv4vlan_add_rules;
+ rte_acl_ipv4vlan_build;
+ rte_acl_list_dump;
+ rte_acl_reset;
+ rte_acl_reset_rules;
+ rte_acl_set_ctx_classify;
+
+ local: *;
+ };
+
+ DPDK_2.1 {
+ global:
+ rte_acl_create;
+
+ } DPDK_2.0;
+
+The addition of the new block tells the linker that a new version node is
+available (DPDK_2.1), which contains the symbol rte_acl_create, and inherits the
+symbols from the DPDK_2.0 node. This list is directly translated into a list of
+exported symbols when DPDK is compiled as a shared library
+
+Next, we need to specify in the code which function map to the rte_acl_create
+symbol at which versions. First, at the site of the initial symbol definition,
+we need to update the function so that it is uniquely named, and not in conflict
+with the public symbol name
+
+.. code-block:: c
+
+ struct rte_acl_ctx *
+ -rte_acl_create(const struct rte_acl_param *param)
+ +rte_acl_create_v20(const struct rte_acl_param *param)
+ {
+ size_t sz;
+ struct rte_acl_ctx *ctx;
+ ...
+
+Note that the base name of the symbol was kept intact, as this is conducive to
+the macros used for versioning symbols. That is our next step, mapping this new
+symbol name to the initial symbol name at version node 2.0. Immediately after
+the function, we add this line of code
+
+.. code-block:: c
+
+ VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
+
+Remembering to also add the rte_compat.h header to the requisite c file where
+these changes are being made. The above macro instructs the linker to create a
+new symbol ``rte_acl_create@DPDK_2.0``, which matches the symbol created in older
+builds, but now points to the above newly named function. We have now mapped
+the original rte_acl_create symbol to the original function (but with a new
+name)
+
+Next, we need to create the 2.1 version of the symbol. We create a new function
+name, with a different suffix, and implement it appropriately
+
+.. code-block:: c
+
+ struct rte_acl_ctx *
+ rte_acl_create_v21(const struct rte_acl_param *param, int debug);
+ {
+ struct rte_acl_ctx *ctx = rte_acl_create_v20(param);
+
+ ctx->debug = debug;
+
+ return ctx;
+ }
+
+This code serves as our new API call. Its the same as our old call, but adds
+the new parameter in place. Next we need to map this function to the symbol
+``rte_acl_create@DPDK_2.1``. To do this, we modify the public prototype of the call
+in the header file, adding the macro there to inform all including applications,
+that on re-link, the default rte_acl_create symbol should point to this
+function. Note that we could do this by simply naming the function above
+rte_acl_create, and the linker would chose the most recent version tag to apply
+in the version script, but we can also do this in the header file
+
+.. code-block:: c
+
+ struct rte_acl_ctx *
+ -rte_acl_create(const struct rte_acl_param *param);
+ +rte_acl_create(const struct rte_acl_param *param, int debug);
+ +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
+
+The BIND_DEFAULT_SYMBOL macro explicitly tells applications that include this
+header, to link to the rte_acl_create_v21 function and apply the DPDK_2.1
+version node to it. This method is more explicit and flexible than just
+re-implementing the exact symbol name, and allows for other features (such as
+linking to the old symbol version by default, when the new ABI is to be opt-in
+for a period.
+
+One last thing we need to do. Note that we've taken what was a public symbol,
+and duplicated it into two uniquely and differently named symbols. We've then
+mapped each of those back to the public symbol ``rte_acl_create`` with different
+version tags. This only applies to dynamic linking, as static linking has no
+notion of versioning. That leaves this code in a position of no longer having a
+symbol simply named ``rte_acl_create`` and a static build will fail on that
+missing symbol.
+
+To correct this, we can simply map a function of our choosing back to the public
+symbol in the static build with the ``MAP_STATIC_SYMBOL`` macro. Generally the
+assumption is that the most recent version of the symbol is the one you want to
+map. So, back in the C file where, immediately after ``rte_acl_create_v21`` is
+defined, we add this
+
+.. code-block:: c
+
+ struct rte_acl_ctx *
+ rte_acl_create_v21(const struct rte_acl_param *param, int debug)
+ {
+ ...
+ }
+ MAP_STATIC_SYMBOL(struct rte_acl_ctx *rte_acl_create(const struct rte_acl_param *param, int debug), rte_acl_create_v21);
+
+That tells the compiler that, when building a static library, any calls to the
+symbol ``rte_acl_create`` should be linked to ``rte_acl_create_v21``
+
+That's it, on the next shared library rebuild, there will be two versions of
+rte_acl_create, an old DPDK_2.0 version, used by previously built applications,
+and a new DPDK_2.1 version, used by future built applications.
+
+
+Deprecating part of a public API
+________________________________
+
+Lets assume that you've done the above update, and after a few releases have
+passed you decide you would like to retire the old version of the function.
+After having gone through the ABI deprecation announcement process, removal is
+easy. Start by removing the symbol from the requisite version map file:
+
+.. code-block:: none
+
+ DPDK_2.0 {
+ global:
+
+ rte_acl_add_rules;
+ rte_acl_build;
+ rte_acl_classify;
+ rte_acl_classify_alg;
+ rte_acl_classify_scalar;
+ rte_acl_dump;
+ - rte_acl_create
+ rte_acl_find_existing;
+ rte_acl_free;
+ rte_acl_ipv4vlan_add_rules;
+ rte_acl_ipv4vlan_build;
+ rte_acl_list_dump;
+ rte_acl_reset;
+ rte_acl_reset_rules;
+ rte_acl_set_ctx_classify;
+
+ local: *;
+ };
+
+ DPDK_2.1 {
+ global:
+ rte_acl_create;
+ } DPDK_2.0;
+
+
+Next remove the corresponding versioned export.
+
+.. code-block:: c
+
+ -VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
+
+
+Note that the internal function definition could also be removed, but its used
+in our example by the newer version _v21, so we leave it in place. This is a
+coding style choice.
+
+Lastly, we need to bump the LIBABIVER number for this library in the Makefile to
+indicate to applications doing dynamic linking that this is a later, and
+possibly incompatible library version:
+
+.. code-block:: c
+
+ -LIBABIVER := 1
+ +LIBABIVER := 2
+
+Deprecating an entire ABI version
+_________________________________
+
+While removing a symbol from and ABI may be useful, it is often more practical
+to remove an entire version node at once. If a version node completely
+specifies an API, then removing part of it, typically makes it incomplete. In
+those cases it is better to remove the entire node
+
+To do this, start by modifying the version map file, such that all symbols from
+the node to be removed are merged into the next node in the map
+
+In the case of our map above, it would transform to look as follows
+
+.. code-block:: none
+
+ DPDK_2.1 {
+ global:
+
+ rte_acl_add_rules;
+ rte_acl_build;
+ rte_acl_classify;
+ rte_acl_classify_alg;
+ rte_acl_classify_scalar;
+ rte_acl_dump;
+ rte_acl_create
+ rte_acl_find_existing;
+ rte_acl_free;
+ rte_acl_ipv4vlan_add_rules;
+ rte_acl_ipv4vlan_build;
+ rte_acl_list_dump;
+ rte_acl_reset;
+ rte_acl_reset_rules;
+ rte_acl_set_ctx_classify;
+
+ local: *;
+ };
+
+Then any uses of BIND_DEFAULT_SYMBOL that pointed to the old node should be
+updated to point to the new version node in any header files for all affected
+symbols.
+
+.. code-block:: c
+
+ -BIND_DEFAULT_SYMBOL(rte_acl_create, _v20, 2.0);
+ +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
+
+Lastly, any VERSION_SYMBOL macros that point to the old version node should be
+removed, taking care to keep, where need old code in place to support newer
+versions of the symbol.
+
+
+Running the ABI Validator
+-------------------------
+
+The ``devtools`` directory in the DPDK source tree contains a utility program,
+``validate-abi.sh``, for validating the DPDK ABI based on the Linux `ABI
+Compliance Checker
+<http://ispras.linuxbase.org/index.php/ABI_compliance_checker>`_.
+
+This has a dependency on the ``abi-compliance-checker`` and ``and abi-dumper``
+utilities which can be installed via a package manager. For example::
+
+ sudo yum install abi-compliance-checker
+ sudo yum install abi-dumper
+
+The syntax of the ``validate-abi.sh`` utility is::
+
+ ./devtools/validate-abi.sh <REV1> <REV2>
+
+Where ``REV1`` and ``REV2`` are valid gitrevisions(7)
+https://www.kernel.org/pub/software/scm/git/docs/gitrevisions.html
+on the local repo.
+
+For example::
+
+ # Check between the previous and latest commit:
+ ./devtools/validate-abi.sh HEAD~1 HEAD
+
+ # Check on a specific compilation target:
+ ./devtools/validate-abi.sh -t x86_64-native-linux-gcc HEAD~1 HEAD
+
+ # Check between two tags:
+ ./devtools/validate-abi.sh v2.0.0 v2.1.0
+
+ # Check between git master and local topic-branch "vhost-hacking":
+ ./devtools/validate-abi.sh master vhost-hacking
+
+After the validation script completes (it can take a while since it need to
+compile both tags) it will create compatibility reports in the
+``./abi-check/compat_report`` directory. Listed incompatibilities can be found
+as follows::
+
+ grep -lr Incompatible abi-check/compat_reports/
diff --git a/doc/guides/contributing/index.rst b/doc/guides/contributing/index.rst
index e2608d3..2fefd91 100644
--- a/doc/guides/contributing/index.rst
+++ b/doc/guides/contributing/index.rst
@@ -10,7 +10,8 @@ Contributor's Guidelines
coding_style
design
- versioning
+ abi_policy
+ abi_versioning
documentation
patches
vulnerability
diff --git a/doc/guides/contributing/versioning.rst b/doc/guides/contributing/versioning.rst
deleted file mode 100644
index 3ab2c43..0000000
--- a/doc/guides/contributing/versioning.rst
+++ /dev/null
@@ -1,591 +0,0 @@
-.. SPDX-License-Identifier: BSD-3-Clause
- Copyright 2018 The DPDK contributors
-
-DPDK ABI/API policy
-===================
-
-Description
------------
-
-This document details some methods for handling ABI management in the DPDK.
-
-General Guidelines
-------------------
-
-#. Whenever possible, ABI should be preserved
-#. ABI/API may be changed with a deprecation process
-#. The modification of symbols can generally be managed with versioning
-#. Libraries or APIs marked in ``experimental`` state may change without constraint
-#. New APIs will be marked as ``experimental`` for at least one release to allow
- any issues found by users of the new API to be fixed quickly
-#. The addition of symbols is generally not problematic
-#. The removal of symbols generally is an ABI break and requires bumping of the
- LIBABIVER macro
-#. Updates to the minimum hardware requirements, which drop support for hardware which
- was previously supported, should be treated as an ABI change.
-
-What is an ABI
-~~~~~~~~~~~~~~
-
-An ABI (Application Binary Interface) is the set of runtime interfaces exposed
-by a library. It is similar to an API (Application Programming Interface) but
-is the result of compilation. It is also effectively cloned when applications
-link to dynamic libraries. That is to say when an application is compiled to
-link against dynamic libraries, it is assumed that the ABI remains constant
-between the time the application is compiled/linked, and the time that it runs.
-Therefore, in the case of dynamic linking, it is critical that an ABI is
-preserved, or (when modified), done in such a way that the application is unable
-to behave improperly or in an unexpected fashion.
-
-
-ABI/API Deprecation
--------------------
-
-The DPDK ABI policy
-~~~~~~~~~~~~~~~~~~~
-
-ABI versions are set at the time of major release labeling, and the ABI may
-change multiple times, without warning, between the last release label and the
-HEAD label of the git tree.
-
-ABI versions, once released, are available until such time as their
-deprecation has been noted in the Release Notes for at least one major release
-cycle. For example consider the case where the ABI for DPDK 2.0 has been
-shipped and then a decision is made to modify it during the development of
-DPDK 2.1. The decision will be recorded in the Release Notes for the DPDK 2.1
-release and the modification will be made available in the DPDK 2.2 release.
-
-ABI versions may be deprecated in whole or in part as needed by a given
-update.
-
-Some ABI changes may be too significant to reasonably maintain multiple
-versions. In those cases ABI's may be updated without backward compatibility
-being provided. The requirements for doing so are:
-
-#. At least 3 acknowledgments of the need to do so must be made on the
- dpdk.org mailing list.
-
- - The acknowledgment of the maintainer of the component is mandatory, or if
- no maintainer is available for the component, the tree/sub-tree maintainer
- for that component must acknowledge the ABI change instead.
-
- - It is also recommended that acknowledgments from different "areas of
- interest" be sought for each deprecation, for example: from NIC vendors,
- CPU vendors, end-users, etc.
-
-#. The changes (including an alternative map file) can be included with
- deprecation notice, in wrapped way by the ``RTE_NEXT_ABI`` option,
- to provide more details about oncoming changes.
- ``RTE_NEXT_ABI`` wrapper will be removed when it become the default ABI.
- More preferred way to provide this information is sending the feature
- as a separate patch and reference it in deprecation notice.
-
-#. A full deprecation cycle, as explained above, must be made to offer
- downstream consumers sufficient warning of the change.
-
-Note that the above process for ABI deprecation should not be undertaken
-lightly. ABI stability is extremely important for downstream consumers of the
-DPDK, especially when distributed in shared object form. Every effort should
-be made to preserve the ABI whenever possible. The ABI should only be changed
-for significant reasons, such as performance enhancements. ABI breakage due to
-changes such as reorganizing public structure fields for aesthetic or
-readability purposes should be avoided.
-
-.. note::
-
- Updates to the minimum hardware requirements, which drop support for hardware
- which was previously supported, should be treated as an ABI change, and
- follow the relevant deprecation policy procedures as above: 3 acks and
- announcement at least one release in advance.
-
-Examples of Deprecation Notices
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The following are some examples of ABI deprecation notices which would be
-added to the Release Notes:
-
-* The Macro ``#RTE_FOO`` is deprecated and will be removed with version 2.0,
- to be replaced with the inline function ``rte_foo()``.
-
-* The function ``rte_mbuf_grok()`` has been updated to include a new parameter
- in version 2.0. Backwards compatibility will be maintained for this function
- until the release of version 2.1
-
-* The members of ``struct rte_foo`` have been reorganized in release 2.0 for
- performance reasons. Existing binary applications will have backwards
- compatibility in release 2.0, while newly built binaries will need to
- reference the new structure variant ``struct rte_foo2``. Compatibility will
- be removed in release 2.2, and all applications will require updating and
- rebuilding to the new structure at that time, which will be renamed to the
- original ``struct rte_foo``.
-
-* Significant ABI changes are planned for the ``librte_dostuff`` library. The
- upcoming release 2.0 will not contain these changes, but release 2.1 will,
- and no backwards compatibility is planned due to the extensive nature of
- these changes. Binaries using this library built prior to version 2.1 will
- require updating and recompilation.
-
-New API replacing previous one
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-If a new API proposed functionally replaces an existing one, when the new API
-becomes non-experimental then the old one is marked with ``__rte_deprecated``.
-Deprecated APIs are removed completely just after the next LTS.
-
-Reminder that old API should follow deprecation process to be removed.
-
-
-Experimental APIs
------------------
-
-APIs marked as ``experimental`` are not considered part of the ABI and may
-change without warning at any time. Since changes to APIs are most likely
-immediately after their introduction, as users begin to take advantage of
-those new APIs and start finding issues with them, new DPDK APIs will be
-automatically marked as ``experimental`` to allow for a period of stabilization
-before they become part of a tracked ABI.
-
-Note that marking an API as experimental is a multi step process.
-To mark an API as experimental, the symbols which are desired to be exported
-must be placed in an EXPERIMENTAL version block in the corresponding libraries'
-version map script.
-Secondly, the corresponding prototypes of those exported functions (in the
-development header files), must be marked with the ``__rte_experimental`` tag
-(see ``rte_compat.h``).
-The DPDK build makefiles perform a check to ensure that the map file and the
-C code reflect the same list of symbols.
-This check can be circumvented by defining ``ALLOW_EXPERIMENTAL_API``
-during compilation in the corresponding library Makefile.
-
-In addition to tagging the code with ``__rte_experimental``,
-the doxygen markup must also contain the EXPERIMENTAL string,
-and the MAINTAINERS file should note the EXPERIMENTAL libraries.
-
-For removing the experimental tag associated with an API, deprecation notice
-is not required. Though, an API should remain in experimental state for at least
-one release. Thereafter, normal process of posting patch for review to mailing
-list can be followed.
-
-
-Library versioning
-------------------
-
-Downstreams might want to provide different DPDK releases at the same time to
-support multiple consumers of DPDK linked against older and newer sonames.
-
-Also due to the interdependencies that DPDK libraries can have applications
-might end up with an executable space in which multiple versions of a library
-are mapped by ld.so.
-
-Think of LibA that got an ABI bump and LibB that did not get an ABI bump but is
-depending on LibA.
-
-.. note::
-
- Application
- \-> LibA.old
- \-> LibB.new -> LibA.new
-
-That is a conflict which can be avoided by setting ``CONFIG_RTE_MAJOR_ABI``.
-If set, the value of ``CONFIG_RTE_MAJOR_ABI`` overwrites all - otherwise per
-library - versions defined in the libraries ``LIBABIVER``.
-An example might be ``CONFIG_RTE_MAJOR_ABI=16.11`` which will make all libraries
-``librte<?>.so.16.11`` instead of ``librte<?>.so.<LIBABIVER>``.
-
-
-ABI versioning
---------------
-
-Versioning Macros
-~~~~~~~~~~~~~~~~~
-
-When a symbol is exported from a library to provide an API, it also provides a
-calling convention (ABI) that is embodied in its name, return type and
-arguments. Occasionally that function may need to change to accommodate new
-functionality or behavior. When that occurs, it is desirable to allow for
-backward compatibility for a time with older binaries that are dynamically
-linked to the DPDK.
-
-To support backward compatibility the ``rte_compat.h``
-header file provides macros to use when updating exported functions. These
-macros are used in conjunction with the ``rte_<library>_version.map`` file for
-a given library to allow multiple versions of a symbol to exist in a shared
-library so that older binaries need not be immediately recompiled.
-
-The macros exported are:
-
-* ``VERSION_SYMBOL(b, e, n)``: Creates a symbol version table entry binding
- versioned symbol ``b@DPDK_n`` to the internal function ``b_e``.
-
-* ``BIND_DEFAULT_SYMBOL(b, e, n)``: Creates a symbol version entry instructing
- the linker to bind references to symbol ``b`` to the internal symbol
- ``b_e``.
-
-* ``MAP_STATIC_SYMBOL(f, p)``: Declare the prototype ``f``, and map it to the
- fully qualified function ``p``, so that if a symbol becomes versioned, it
- can still be mapped back to the public symbol name.
-
-Examples of ABI Macro use
-^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Updating a public API
-_____________________
-
-Assume we have a function as follows
-
-.. code-block:: c
-
- /*
- * Create an acl context object for apps to
- * manipulate
- */
- struct rte_acl_ctx *
- rte_acl_create(const struct rte_acl_param *param)
- {
- ...
- }
-
-
-Assume that struct rte_acl_ctx is a private structure, and that a developer
-wishes to enhance the acl api so that a debugging flag can be enabled on a
-per-context basis. This requires an addition to the structure (which, being
-private, is safe), but it also requires modifying the code as follows
-
-.. code-block:: c
-
- /*
- * Create an acl context object for apps to
- * manipulate
- */
- struct rte_acl_ctx *
- rte_acl_create(const struct rte_acl_param *param, int debug)
- {
- ...
- }
-
-
-Note also that, being a public function, the header file prototype must also be
-changed, as must all the call sites, to reflect the new ABI footprint. We will
-maintain previous ABI versions that are accessible only to previously compiled
-binaries
-
-The addition of a parameter to the function is ABI breaking as the function is
-public, and existing application may use it in its current form. However, the
-compatibility macros in DPDK allow a developer to use symbol versioning so that
-multiple functions can be mapped to the same public symbol based on when an
-application was linked to it. To see how this is done, we start with the
-requisite libraries version map file. Initially the version map file for the
-acl library looks like this
-
-.. code-block:: none
-
- DPDK_2.0 {
- global:
-
- rte_acl_add_rules;
- rte_acl_build;
- rte_acl_classify;
- rte_acl_classify_alg;
- rte_acl_classify_scalar;
- rte_acl_create;
- rte_acl_dump;
- rte_acl_find_existing;
- rte_acl_free;
- rte_acl_ipv4vlan_add_rules;
- rte_acl_ipv4vlan_build;
- rte_acl_list_dump;
- rte_acl_reset;
- rte_acl_reset_rules;
- rte_acl_set_ctx_classify;
-
- local: *;
- };
-
-This file needs to be modified as follows
-
-.. code-block:: none
-
- DPDK_2.0 {
- global:
-
- rte_acl_add_rules;
- rte_acl_build;
- rte_acl_classify;
- rte_acl_classify_alg;
- rte_acl_classify_scalar;
- rte_acl_create;
- rte_acl_dump;
- rte_acl_find_existing;
- rte_acl_free;
- rte_acl_ipv4vlan_add_rules;
- rte_acl_ipv4vlan_build;
- rte_acl_list_dump;
- rte_acl_reset;
- rte_acl_reset_rules;
- rte_acl_set_ctx_classify;
-
- local: *;
- };
-
- DPDK_2.1 {
- global:
- rte_acl_create;
-
- } DPDK_2.0;
-
-The addition of the new block tells the linker that a new version node is
-available (DPDK_2.1), which contains the symbol rte_acl_create, and inherits the
-symbols from the DPDK_2.0 node. This list is directly translated into a list of
-exported symbols when DPDK is compiled as a shared library
-
-Next, we need to specify in the code which function map to the rte_acl_create
-symbol at which versions. First, at the site of the initial symbol definition,
-we need to update the function so that it is uniquely named, and not in conflict
-with the public symbol name
-
-.. code-block:: c
-
- struct rte_acl_ctx *
- -rte_acl_create(const struct rte_acl_param *param)
- +rte_acl_create_v20(const struct rte_acl_param *param)
- {
- size_t sz;
- struct rte_acl_ctx *ctx;
- ...
-
-Note that the base name of the symbol was kept intact, as this is conducive to
-the macros used for versioning symbols. That is our next step, mapping this new
-symbol name to the initial symbol name at version node 2.0. Immediately after
-the function, we add this line of code
-
-.. code-block:: c
-
- VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
-
-Remembering to also add the rte_compat.h header to the requisite c file where
-these changes are being made. The above macro instructs the linker to create a
-new symbol ``rte_acl_create@DPDK_2.0``, which matches the symbol created in older
-builds, but now points to the above newly named function. We have now mapped
-the original rte_acl_create symbol to the original function (but with a new
-name)
-
-Next, we need to create the 2.1 version of the symbol. We create a new function
-name, with a different suffix, and implement it appropriately
-
-.. code-block:: c
-
- struct rte_acl_ctx *
- rte_acl_create_v21(const struct rte_acl_param *param, int debug);
- {
- struct rte_acl_ctx *ctx = rte_acl_create_v20(param);
-
- ctx->debug = debug;
-
- return ctx;
- }
-
-This code serves as our new API call. Its the same as our old call, but adds
-the new parameter in place. Next we need to map this function to the symbol
-``rte_acl_create@DPDK_2.1``. To do this, we modify the public prototype of the call
-in the header file, adding the macro there to inform all including applications,
-that on re-link, the default rte_acl_create symbol should point to this
-function. Note that we could do this by simply naming the function above
-rte_acl_create, and the linker would chose the most recent version tag to apply
-in the version script, but we can also do this in the header file
-
-.. code-block:: c
-
- struct rte_acl_ctx *
- -rte_acl_create(const struct rte_acl_param *param);
- +rte_acl_create(const struct rte_acl_param *param, int debug);
- +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
-
-The BIND_DEFAULT_SYMBOL macro explicitly tells applications that include this
-header, to link to the rte_acl_create_v21 function and apply the DPDK_2.1
-version node to it. This method is more explicit and flexible than just
-re-implementing the exact symbol name, and allows for other features (such as
-linking to the old symbol version by default, when the new ABI is to be opt-in
-for a period.
-
-One last thing we need to do. Note that we've taken what was a public symbol,
-and duplicated it into two uniquely and differently named symbols. We've then
-mapped each of those back to the public symbol ``rte_acl_create`` with different
-version tags. This only applies to dynamic linking, as static linking has no
-notion of versioning. That leaves this code in a position of no longer having a
-symbol simply named ``rte_acl_create`` and a static build will fail on that
-missing symbol.
-
-To correct this, we can simply map a function of our choosing back to the public
-symbol in the static build with the ``MAP_STATIC_SYMBOL`` macro. Generally the
-assumption is that the most recent version of the symbol is the one you want to
-map. So, back in the C file where, immediately after ``rte_acl_create_v21`` is
-defined, we add this
-
-.. code-block:: c
-
- struct rte_acl_ctx *
- rte_acl_create_v21(const struct rte_acl_param *param, int debug)
- {
- ...
- }
- MAP_STATIC_SYMBOL(struct rte_acl_ctx *rte_acl_create(const struct rte_acl_param *param, int debug), rte_acl_create_v21);
-
-That tells the compiler that, when building a static library, any calls to the
-symbol ``rte_acl_create`` should be linked to ``rte_acl_create_v21``
-
-That's it, on the next shared library rebuild, there will be two versions of
-rte_acl_create, an old DPDK_2.0 version, used by previously built applications,
-and a new DPDK_2.1 version, used by future built applications.
-
-
-Deprecating part of a public API
-________________________________
-
-Lets assume that you've done the above update, and after a few releases have
-passed you decide you would like to retire the old version of the function.
-After having gone through the ABI deprecation announcement process, removal is
-easy. Start by removing the symbol from the requisite version map file:
-
-.. code-block:: none
-
- DPDK_2.0 {
- global:
-
- rte_acl_add_rules;
- rte_acl_build;
- rte_acl_classify;
- rte_acl_classify_alg;
- rte_acl_classify_scalar;
- rte_acl_dump;
- - rte_acl_create
- rte_acl_find_existing;
- rte_acl_free;
- rte_acl_ipv4vlan_add_rules;
- rte_acl_ipv4vlan_build;
- rte_acl_list_dump;
- rte_acl_reset;
- rte_acl_reset_rules;
- rte_acl_set_ctx_classify;
-
- local: *;
- };
-
- DPDK_2.1 {
- global:
- rte_acl_create;
- } DPDK_2.0;
-
-
-Next remove the corresponding versioned export.
-
-.. code-block:: c
-
- -VERSION_SYMBOL(rte_acl_create, _v20, 2.0);
-
-
-Note that the internal function definition could also be removed, but its used
-in our example by the newer version _v21, so we leave it in place. This is a
-coding style choice.
-
-Lastly, we need to bump the LIBABIVER number for this library in the Makefile to
-indicate to applications doing dynamic linking that this is a later, and
-possibly incompatible library version:
-
-.. code-block:: c
-
- -LIBABIVER := 1
- +LIBABIVER := 2
-
-Deprecating an entire ABI version
-_________________________________
-
-While removing a symbol from and ABI may be useful, it is often more practical
-to remove an entire version node at once. If a version node completely
-specifies an API, then removing part of it, typically makes it incomplete. In
-those cases it is better to remove the entire node
-
-To do this, start by modifying the version map file, such that all symbols from
-the node to be removed are merged into the next node in the map
-
-In the case of our map above, it would transform to look as follows
-
-.. code-block:: none
-
- DPDK_2.1 {
- global:
-
- rte_acl_add_rules;
- rte_acl_build;
- rte_acl_classify;
- rte_acl_classify_alg;
- rte_acl_classify_scalar;
- rte_acl_dump;
- rte_acl_create
- rte_acl_find_existing;
- rte_acl_free;
- rte_acl_ipv4vlan_add_rules;
- rte_acl_ipv4vlan_build;
- rte_acl_list_dump;
- rte_acl_reset;
- rte_acl_reset_rules;
- rte_acl_set_ctx_classify;
-
- local: *;
- };
-
-Then any uses of BIND_DEFAULT_SYMBOL that pointed to the old node should be
-updated to point to the new version node in any header files for all affected
-symbols.
-
-.. code-block:: c
-
- -BIND_DEFAULT_SYMBOL(rte_acl_create, _v20, 2.0);
- +BIND_DEFAULT_SYMBOL(rte_acl_create, _v21, 2.1);
-
-Lastly, any VERSION_SYMBOL macros that point to the old version node should be
-removed, taking care to keep, where need old code in place to support newer
-versions of the symbol.
-
-
-Running the ABI Validator
--------------------------
-
-The ``devtools`` directory in the DPDK source tree contains a utility program,
-``validate-abi.sh``, for validating the DPDK ABI based on the Linux `ABI
-Compliance Checker
-<http://ispras.linuxbase.org/index.php/ABI_compliance_checker>`_.
-
-This has a dependency on the ``abi-compliance-checker`` and ``and abi-dumper``
-utilities which can be installed via a package manager. For example::
-
- sudo yum install abi-compliance-checker
- sudo yum install abi-dumper
-
-The syntax of the ``validate-abi.sh`` utility is::
-
- ./devtools/validate-abi.sh <REV1> <REV2>
-
-Where ``REV1`` and ``REV2`` are valid gitrevisions(7)
-https://www.kernel.org/pub/software/scm/git/docs/gitrevisions.html
-on the local repo.
-
-For example::
-
- # Check between the previous and latest commit:
- ./devtools/validate-abi.sh HEAD~1 HEAD
-
- # Check on a specific compilation target:
- ./devtools/validate-abi.sh -t x86_64-native-linux-gcc HEAD~1 HEAD
-
- # Check between two tags:
- ./devtools/validate-abi.sh v2.0.0 v2.1.0
-
- # Check between git master and local topic-branch "vhost-hacking":
- ./devtools/validate-abi.sh master vhost-hacking
-
-After the validation script completes (it can take a while since it need to
-compile both tags) it will create compatibility reports in the
-``./abi-check/compat_report`` directory. Listed incompatibilities can be found
-as follows::
-
- grep -lr Incompatible abi-check/compat_reports/
--
2.7.4
^ permalink raw reply [relevance 13%]
* [dpdk-dev] [PATCH v2 0/3] doc: changes to abi policy introducing major abi versions
@ 2019-08-02 15:38 11% Ray Kinsella
2019-08-02 15:38 13% ` [dpdk-dev] [PATCH v2 1/3] doc: separate versioning.rst into version and policy Ray Kinsella
` (2 more replies)
0 siblings, 3 replies; 200+ results
From: Ray Kinsella @ 2019-08-02 15:38 UTC (permalink / raw)
To: dev
Cc: mdr, thomas, stephen, bruce.richardson, ferruh.yigit,
konstantin.ananyev, jerinj, olivier.matz, nhorman,
maxime.coquelin, hemant.agrawal
TL;DR Abbreviation:
A major ABI version that all DPDK releases during a one year period
support. ABI versioning is managed at a project-level, in place of library-level
management. ABI changes to add new features are permitted, as long as ABI
compatibility with the major ABI version is maintained.
Detail:
This patch introduces major ABI versions, supported for one year and released
aligned with the LTS release. This ABI version is then supported by all
subsequent releases within that one year period. The intention is that the one
year support period, will then be reviewed after the initial year with the
intention of lengthing the support period for the next ABI version.
ABI changes that preserve ABI compatibility with the major ABI version are
permitted in subsequent releases. ABI changes, follow similar approval rules as
before with the additional gate of now requiring technical board approval. The
merging and release of ABI breaking changes would now be pushed to the
declaration of the next major ABI version.
This change encourages developers to maintain ABI compatibility with the major
ABI version, by promoting a permissive culture around those changes that
preserve ABI compatibility. This approach begins to align DPDK with those
projects that declare major ABI versions (e.g. version 2.x, 3.x) and support
those versions for some period, typically two years or more.
To provide an example of how this might work in practice:
* DPDK v20 is declared as the supported ABI version for one year, aligned with
the DPDK v19.11 (LTS) release. All library sonames are updated to reflect the
new ABI version, e.g. librte_eal.so.20, librte_acl.so.20...
* DPDK v20.02 .. v20.08 releases are ABI compatible with the DPDK v20 ABI. ABI
changes are permitted from DPDK v20.02 onwards, with the condition that ABI
compatibility with DPDK v20 is preserved.
* DPDK v21 is declared as the new supported ABI version for two years, aligned
with the DPDK v20.11 (LTS) release. The DPDK v20 ABI is now depreciated,
library sonames are updated to v21 and ABI compatibility breaking changes may
be introduced.
---
v2
* Restructured the patch into 3 patches:
1. Splits the original versioning document into an ABI policy document
and ABI versioning document.
2. Add changes to the policy document introducing major ABI versions.
3. Fixes up the versioning document in light of major ABI versioning.
* Reduces the initial ABI stability from two years to one year, with a review
after the first year.
* Adds detail around ABI version handling for experimental libraries.
* Adds detail around chain of responsility for removing deprecated symbols.
Ray Kinsella (3):
doc: separate versioning.rst into version and policy
doc: changes to abi policy introducing major abi versions
doc: updates to versioning guide for abi versions
doc/guides/contributing/abi_policy.rst | 283 ++++++++++++++
doc/guides/contributing/abi_versioning.rst | 505 ++++++++++++++++++++++++
doc/guides/contributing/index.rst | 3 +-
doc/guides/contributing/stable.rst | 36 +-
doc/guides/contributing/versioning.rst | 591 -----------------------------
5 files changed, 810 insertions(+), 608 deletions(-)
create mode 100644 doc/guides/contributing/abi_policy.rst
create mode 100644 doc/guides/contributing/abi_versioning.rst
delete mode 100644 doc/guides/contributing/versioning.rst
--
2.7.4
^ permalink raw reply [relevance 11%]
* Re: [dpdk-dev] [PATCH] doc: announce lcore_config symbol removal
2019-07-31 11:06 5% [dpdk-dev] [PATCH] doc: announce lcore_config symbol removal David Marchand
@ 2019-07-31 13:48 0% ` Stephen Hemminger
2019-08-08 9:31 5% ` [dpdk-dev] [PATCH v2] " David Marchand
1 sibling, 0 replies; 200+ results
From: Stephen Hemminger @ 2019-07-31 13:48 UTC (permalink / raw)
To: David Marchand; +Cc: dev, thomas, bruce.richardson
On Wed, 31 Jul 2019 13:06:17 +0200
David Marchand <david.marchand@redhat.com> wrote:
> New accessors have been introduced to provide the hidden information.
> This symbol can now be kept internal.
>
> Signed-off-by: David Marchand <david.marchand@redhat.com>
> ---
> doc/guides/rel_notes/deprecation.rst | 4 ++++
> 1 file changed, 4 insertions(+)
>
> diff --git a/doc/guides/rel_notes/deprecation.rst b/doc/guides/rel_notes/deprecation.rst
> index 37b8592..b18de70 100644
> --- a/doc/guides/rel_notes/deprecation.rst
> +++ b/doc/guides/rel_notes/deprecation.rst
> @@ -23,6 +23,10 @@ Deprecation Notices
> * eal: The function ``rte_eal_remote_launch`` will return new error codes
> after read or write error on the pipe, instead of calling ``rte_panic``.
>
> +* eal: the ``lcore_config`` struct and global symbol will be made private to
> + remove it from the externally visible ABI and allow it to be updated in the
> + future.
> +
> * eal: both declaring and identifying devices will be streamlined in v18.11.
> New functions will appear to query a specific port from buses, classes of
> device and device drivers. Device declaration will be made coherent with the
Acked-by: Stephen Hemminger <stephen@networkplumber.org>
^ permalink raw reply [relevance 0%]
* [dpdk-dev] [PATCH] doc: announce lcore_config symbol removal
@ 2019-07-31 11:06 5% David Marchand
2019-07-31 13:48 0% ` Stephen Hemminger
2019-08-08 9:31 5% ` [dpdk-dev] [PATCH v2] " David Marchand
0 siblings, 2 replies; 200+ results
From: David Marchand @ 2019-07-31 11:06 UTC (permalink / raw)
To: dev; +Cc: thomas, stephen, bruce.richardson
New accessors have been introduced to provide the hidden information.
This symbol can now be kept internal.
Signed-off-by: David Marchand <david.marchand@redhat.com>
---
doc/guides/rel_notes/deprecation.rst | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/doc/guides/rel_notes/deprecation.rst b/doc/guides/rel_notes/deprecation.rst
index 37b8592..b18de70 100644
--- a/doc/guides/rel_notes/deprecation.rst
+++ b/doc/guides/rel_notes/deprecation.rst
@@ -23,6 +23,10 @@ Deprecation Notices
* eal: The function ``rte_eal_remote_launch`` will return new error codes
after read or write error on the pipe, instead of calling ``rte_panic``.
+* eal: the ``lcore_config`` struct and global symbol will be made private to
+ remove it from the externally visible ABI and allow it to be updated in the
+ future.
+
* eal: both declaring and identifying devices will be streamlined in v18.11.
New functions will appear to query a specific port from buses, classes of
device and device drivers. Device declaration will be made coherent with the
--
1.8.3.1
^ permalink raw reply [relevance 5%]
* Re: [dpdk-dev] [PATCH v2 2/2] drivers/raw: standardize naming
@ 2019-07-31 7:29 0% ` Xu, Rosen
0 siblings, 0 replies; 200+ results
From: Xu, Rosen @ 2019-07-31 7:29 UTC (permalink / raw)
To: Thomas Monjalon, dev, Mcnamara, John, Kovacevic, Marko,
Nipun Gupta, Zhang, Tianfei, Richardson, Bruce, Li, Xiaoyun, Wu,
Jingjing, Satha Rao, Vamsi Attunuru, Shreyansh Jain,
Hemant Agrawal
Cc: Neil Horman
Hi,
I'm okay if it's applied in 19.11.
> -----Original Message-----
> From: Thomas Monjalon [mailto:thomas@monjalon.net]
> Sent: Tuesday, July 30, 2019 21:40
> To: dev@dpdk.org; Mcnamara, John <john.mcnamara@intel.com>;
> Kovacevic, Marko <marko.kovacevic@intel.com>; Nipun Gupta
> <nipun.gupta@nxp.com>; Xu, Rosen <rosen.xu@intel.com>; Zhang, Tianfei
> <tianfei.zhang@intel.com>; Richardson, Bruce
> <bruce.richardson@intel.com>; Li, Xiaoyun <xiaoyun.li@intel.com>; Wu,
> Jingjing <jingjing.wu@intel.com>; Satha Rao <skoteshwar@marvell.com>;
> Vamsi Attunuru <vattunuru@marvell.com>; Shreyansh Jain
> <shreyansh.jain@nxp.com>; Hemant Agrawal <hemant.agrawal@nxp.com>
> Cc: Neil Horman <nhorman@tuxdriver.com>
> Subject: [PATCH v2 2/2] drivers/raw: standardize naming
>
> From: Bruce Richardson <bruce.richardson@intel.com>
>
> The driver names for rawdevs were both different in make and meson builds
> and were non-standard in the make version in that some included "rawdev"
> in the name while others didn't.
>
> Therefore, for global consistency of naming, we can use "rte_rawdev" rather
> than "rte_pmd" for the prefix for the libraries. While most other driver
> categories use "rte_pmd" as a prefix, there is precedent for this in the
> mempool drivers use "rte_mempool" as a prefix.
>
> Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
> Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
> ---
> doc/guides/rel_notes/release_19_08.rst | 3 +++
> drivers/raw/dpaa2_cmdif/Makefile | 4 ++--
> ...sion.map => rte_rawdev_dpaa2_cmdif_version.map} | 0
> drivers/raw/dpaa2_qdma/Makefile | 4 ++--
> ...rsion.map => rte_rawdev_dpaa2_qdma_version.map} | 0
> drivers/raw/ifpga/Makefile | 4 ++--
> ...ga_version.map => rte_rawdev_ifpga_version.map} | 0
> drivers/raw/ioat/Makefile | 4 ++--
> ...oat_version.map => rte_rawdev_ioat_version.map} | 0
> drivers/raw/meson.build | 2 +-
> drivers/raw/ntb/Makefile | 4 ++--
> ..._ntb_version.map => rte_rawdev_ntb_version.map} | 0
> drivers/raw/octeontx2_dma/Makefile | 4 ++--
> ...on.map => rte_rawdev_octeontx2_dma_version.map} | 0
> drivers/raw/skeleton/Makefile | 4 ++--
> ...version.map => rte_rawdev_skeleton_version.map} | 0
> mk/rte.app.mk | 14 +++++++-------
> 17 files changed, 25 insertions(+), 22 deletions(-) rename
> drivers/raw/dpaa2_cmdif/{rte_pmd_dpaa2_cmdif_version.map =>
> rte_rawdev_dpaa2_cmdif_version.map} (100%) rename
> drivers/raw/dpaa2_qdma/{rte_pmd_dpaa2_qdma_version.map =>
> rte_rawdev_dpaa2_qdma_version.map} (100%) rename
> drivers/raw/ifpga/{rte_pmd_ifpga_version.map =>
> rte_rawdev_ifpga_version.map} (100%) rename
> drivers/raw/ioat/{rte_pmd_ioat_version.map =>
> rte_rawdev_ioat_version.map} (100%) rename
> drivers/raw/ntb/{rte_pmd_ntb_version.map =>
> rte_rawdev_ntb_version.map} (100%) rename
> drivers/raw/octeontx2_dma/{rte_pmd_octeontx2_dma_version.map =>
> rte_rawdev_octeontx2_dma_version.map} (100%) rename
> drivers/raw/skeleton/{rte_pmd_skeleton_version.map =>
> rte_rawdev_skeleton_version.map} (100%)
>
> diff --git a/doc/guides/rel_notes/release_19_08.rst
> b/doc/guides/rel_notes/release_19_08.rst
> index fcd1386a5..13a98f520 100644
> --- a/doc/guides/rel_notes/release_19_08.rst
> +++ b/doc/guides/rel_notes/release_19_08.rst
> @@ -351,6 +351,9 @@ ABI Changes
> * bbdev: New operations and parameters added to support new 5GNR
> operations.
> The bbdev ABI is still kept experimental.
>
> +* rawdev: The driver names have been changed to ``librte_rawdev_*``.
> + Now they all have the same prefix, and same name with make and meson
> builds.
> +
>
> Shared Library Versions
> -----------------------
> diff --git a/drivers/raw/dpaa2_cmdif/Makefile
> b/drivers/raw/dpaa2_cmdif/Makefile
> index 9bd5ff229..2b4150c2d 100644
> --- a/drivers/raw/dpaa2_cmdif/Makefile
> +++ b/drivers/raw/dpaa2_cmdif/Makefile
> @@ -6,7 +6,7 @@ include $(RTE_SDK)/mk/rte.vars.mk # # library name # -
> LIB = librte_pmd_dpaa2_cmdif.a
> +LIB = librte_rawdev_dpaa2_cmdif.a
>
> CFLAGS += -DALLOW_EXPERIMENTAL_API
> CFLAGS += -O3
> @@ -23,7 +23,7 @@ LDLIBS += -lrte_mempool_dpaa2 LDLIBS += -
> lrte_rawdev LDLIBS += -lrte_common_dpaax
>
> -EXPORT_MAP := rte_pmd_dpaa2_cmdif_version.map
> +EXPORT_MAP := rte_rawdev_dpaa2_cmdif_version.map
>
> LIBABIVER := 2
>
> diff --git a/drivers/raw/dpaa2_cmdif/rte_pmd_dpaa2_cmdif_version.map
> b/drivers/raw/dpaa2_cmdif/rte_rawdev_dpaa2_cmdif_version.map
> similarity index 100%
> rename from drivers/raw/dpaa2_cmdif/rte_pmd_dpaa2_cmdif_version.map
> rename to drivers/raw/dpaa2_cmdif/rte_rawdev_dpaa2_cmdif_version.map
> diff --git a/drivers/raw/dpaa2_qdma/Makefile
> b/drivers/raw/dpaa2_qdma/Makefile index f9a810cc6..0009fd4c6 100644
> --- a/drivers/raw/dpaa2_qdma/Makefile
> +++ b/drivers/raw/dpaa2_qdma/Makefile
> @@ -6,7 +6,7 @@ include $(RTE_SDK)/mk/rte.vars.mk # # library name # -
> LIB = librte_pmd_dpaa2_qdma.a
> +LIB = librte_rawdev_dpaa2_qdma.a
>
> CFLAGS += -DALLOW_EXPERIMENTAL_API
> CFLAGS += -O3
> @@ -24,7 +24,7 @@ LDLIBS += -lrte_kvargs LDLIBS += -lrte_ring LDLIBS += -
> lrte_common_dpaax
>
> -EXPORT_MAP := rte_pmd_dpaa2_qdma_version.map
> +EXPORT_MAP := rte_rawdev_dpaa2_qdma_version.map
>
> LIBABIVER := 3
>
> diff --git a/drivers/raw/dpaa2_qdma/rte_pmd_dpaa2_qdma_version.map
> b/drivers/raw/dpaa2_qdma/rte_rawdev_dpaa2_qdma_version.map
> similarity index 100%
> rename from
> drivers/raw/dpaa2_qdma/rte_pmd_dpaa2_qdma_version.map
> rename to
> drivers/raw/dpaa2_qdma/rte_rawdev_dpaa2_qdma_version.map
> diff --git a/drivers/raw/ifpga/Makefile b/drivers/raw/ifpga/Makefile index
> 5fa9303d5..655b29288 100644
> --- a/drivers/raw/ifpga/Makefile
> +++ b/drivers/raw/ifpga/Makefile
> @@ -6,7 +6,7 @@ include $(RTE_SDK)/mk/rte.vars.mk # # library name # -
> LIB = librte_pmd_ifpga_rawdev.a
> +LIB = librte_rawdev_ifpga.a
>
> CFLAGS += -DALLOW_EXPERIMENTAL_API
> CFLAGS += -O3
> @@ -21,7 +21,7 @@ LDLIBS += -lrte_kvargs LDLIBS += -lrte_bus_pci LDLIBS
> += -lrte_bus_ifpga
>
> -EXPORT_MAP := rte_pmd_ifpga_version.map
> +EXPORT_MAP := rte_rawdev_ifpga_version.map
>
> LIBABIVER := 1
>
> diff --git a/drivers/raw/ifpga/rte_pmd_ifpga_version.map
> b/drivers/raw/ifpga/rte_rawdev_ifpga_version.map
> similarity index 100%
> rename from drivers/raw/ifpga/rte_pmd_ifpga_version.map
> rename to drivers/raw/ifpga/rte_rawdev_ifpga_version.map
> diff --git a/drivers/raw/ioat/Makefile b/drivers/raw/ioat/Makefile index
> 32f079845..e852afb57 100644
> --- a/drivers/raw/ioat/Makefile
> +++ b/drivers/raw/ioat/Makefile
> @@ -4,7 +4,7 @@
> include $(RTE_SDK)/mk/rte.vars.mk
>
> # library name
> -LIB = librte_pmd_ioat_rawdev.a
> +LIB = librte_rawdev_ioat.a
>
> # build flags
> CFLAGS += -O3
> @@ -18,7 +18,7 @@ LDLIBS += -lrte_mbuf -lrte_mempool LIBABIVER := 1
>
> # versioning export map
> -EXPORT_MAP := rte_pmd_ioat_version.map
> +EXPORT_MAP := rte_rawdev_ioat_version.map
>
> # library source files
> SRCS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV) += ioat_rawdev.c diff --
> git a/drivers/raw/ioat/rte_pmd_ioat_version.map
> b/drivers/raw/ioat/rte_rawdev_ioat_version.map
> similarity index 100%
> rename from drivers/raw/ioat/rte_pmd_ioat_version.map
> rename to drivers/raw/ioat/rte_rawdev_ioat_version.map
> diff --git a/drivers/raw/meson.build b/drivers/raw/meson.build index
> bcd5553e1..d7037cd87 100644
> --- a/drivers/raw/meson.build
> +++ b/drivers/raw/meson.build
> @@ -7,4 +7,4 @@ drivers = ['dpaa2_cmdif', 'dpaa2_qdma',
> 'skeleton']
> std_deps = ['rawdev']
> config_flag_fmt = 'RTE_LIBRTE_PMD_@0@_RAWDEV'
> -driver_name_fmt = 'rte_pmd_@0@'
> +driver_name_fmt = 'rte_rawdev_@0@'
> diff --git a/drivers/raw/ntb/Makefile b/drivers/raw/ntb/Makefile index
> edd49fe75..6fe2aaf40 100644
> --- a/drivers/raw/ntb/Makefile
> +++ b/drivers/raw/ntb/Makefile
> @@ -6,7 +6,7 @@ include $(RTE_SDK)/mk/rte.vars.mk # # library name # -
> LIB = librte_pmd_ntb.a
> +LIB = librte_rawdev_ntb.a
>
> CFLAGS += -DALLOW_EXPERIMENTAL_API
> CFLAGS += -O3
> @@ -15,7 +15,7 @@ LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool LDLIBS
> += -lrte_pci -lrte_bus_pci LDLIBS += -lrte_rawdev
>
> -EXPORT_MAP := rte_pmd_ntb_version.map
> +EXPORT_MAP := rte_rawdev_ntb_version.map
>
> LIBABIVER := 1
>
> diff --git a/drivers/raw/ntb/rte_pmd_ntb_version.map
> b/drivers/raw/ntb/rte_rawdev_ntb_version.map
> similarity index 100%
> rename from drivers/raw/ntb/rte_pmd_ntb_version.map
> rename to drivers/raw/ntb/rte_rawdev_ntb_version.map
> diff --git a/drivers/raw/octeontx2_dma/Makefile
> b/drivers/raw/octeontx2_dma/Makefile
> index 96f782eb6..f101e4916 100644
> --- a/drivers/raw/octeontx2_dma/Makefile
> +++ b/drivers/raw/octeontx2_dma/Makefile
> @@ -5,7 +5,7 @@
> include $(RTE_SDK)/mk/rte.vars.mk
>
> # library name
> -LIB = librte_pmd_octeontx2_dma.a
> +LIB = librte_rawdev_octeontx2_dma.a
>
> CFLAGS += -O3 $(WERROR_FLAGS)
> CFLAGS += -I$(RTE_SDK)/drivers/common/octeontx2/
> @@ -22,7 +22,7 @@ CFLAGS += -diag-disable 2259 endif endif
>
> -EXPORT_MAP := rte_pmd_octeontx2_dma_version.map
> +EXPORT_MAP := rte_rawdev_octeontx2_dma_version.map
>
> LIBABIVER := 1
>
> diff --git
> a/drivers/raw/octeontx2_dma/rte_pmd_octeontx2_dma_version.map
> b/drivers/raw/octeontx2_dma/rte_rawdev_octeontx2_dma_version.map
> similarity index 100%
> rename from
> drivers/raw/octeontx2_dma/rte_pmd_octeontx2_dma_version.map
> rename to
> drivers/raw/octeontx2_dma/rte_rawdev_octeontx2_dma_version.map
> diff --git a/drivers/raw/skeleton/Makefile b/drivers/raw/skeleton/Makefile
> index 9641e6505..783b1e952 100644
> --- a/drivers/raw/skeleton/Makefile
> +++ b/drivers/raw/skeleton/Makefile
> @@ -6,7 +6,7 @@ include $(RTE_SDK)/mk/rte.vars.mk # # library name # -
> LIB = librte_pmd_skeleton_rawdev.a
> +LIB = librte_rawdev_skeleton.a
>
> CFLAGS += -O3
> CFLAGS += $(WERROR_FLAGS)
> @@ -15,7 +15,7 @@ LDLIBS += -lrte_rawdev LDLIBS += -lrte_bus_vdev
> LDLIBS += -lrte_kvargs
>
> -EXPORT_MAP := rte_pmd_skeleton_version.map
> +EXPORT_MAP := rte_rawdev_skeleton_version.map
>
> LIBABIVER := 1
>
> diff --git a/drivers/raw/skeleton/rte_pmd_skeleton_version.map
> b/drivers/raw/skeleton/rte_rawdev_skeleton_version.map
> similarity index 100%
> rename from drivers/raw/skeleton/rte_pmd_skeleton_version.map
> rename to drivers/raw/skeleton/rte_rawdev_skeleton_version.map
> diff --git a/mk/rte.app.mk b/mk/rte.app.mk index a277c808e..ba5c39e01
> 100644
> --- a/mk/rte.app.mk
> +++ b/mk/rte.app.mk
> @@ -312,19 +312,19 @@ _LDLIBS-
> $(CONFIG_RTE_LIBRTE_PMD_OPDL_EVENTDEV) += -lrte_pmd_opdl_event
> endif # CONFIG_RTE_LIBRTE_EVENTDEV
>
> ifeq ($(CONFIG_RTE_LIBRTE_RAWDEV),y)
> -_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_SKELETON_RAWDEV) += -
> lrte_pmd_skeleton_rawdev
> +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_SKELETON_RAWDEV) +=
> +-lrte_rawdev_skeleton
> ifeq ($(CONFIG_RTE_EAL_VFIO)$(CONFIG_RTE_LIBRTE_FSLMC_BUS),yy)
> -_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_CMDIF_RAWDEV) += -
> lrte_pmd_dpaa2_cmdif
> -_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) += -
> lrte_pmd_dpaa2_qdma
> +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_CMDIF_RAWDEV) +=
> +-lrte_rawdev_dpaa2_cmdif
> +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) +=
> +-lrte_rawdev_dpaa2_qdma
> endif # CONFIG_RTE_LIBRTE_FSLMC_BUS
> _LDLIBS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS) += -lrte_bus_ifpga
> ifeq ($(CONFIG_RTE_LIBRTE_IFPGA_BUS),y)
> -_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV) += -
> lrte_pmd_ifpga_rawdev
> +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV) += -
> lrte_rawdev_ifpga
> _LDLIBS-$(CONFIG_RTE_LIBRTE_IPN3KE_PMD) += -lrte_pmd_ipn3ke
> endif # CONFIG_RTE_LIBRTE_IFPGA_BUS
> -_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV) += -
> lrte_pmd_ioat_rawdev
> -_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV) += -lrte_pmd_ntb
> -_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX2_DMA_RAWDEV) += -
> lrte_pmd_octeontx2_dma
> +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV) += -
> lrte_rawdev_ioat
> +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV) += -lrte_rawdev_ntb
> +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX2_DMA_RAWDEV) +=
> +-lrte_rawdev_octeontx2_dma
> endif # CONFIG_RTE_LIBRTE_RAWDEV
>
> endif # !CONFIG_RTE_BUILD_SHARED_LIBS
> --
> 2.21.0
Acked-by: Rosen Xu <rosen.xu@intel.com>
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [dpdk-stable] [PATCH] librte_flow_classify: fix out-of-bounds access
2019-07-30 16:55 0% ` Ferruh Yigit
@ 2019-07-30 17:30 0% ` Aaron Conole
0 siblings, 0 replies; 200+ results
From: Aaron Conole @ 2019-07-30 17:30 UTC (permalink / raw)
To: Ferruh Yigit
Cc: David Marchand, Bernard Iremonger, dev, dpdk stable,
Thomas Monjalon, Singh, Jasvinder, Flavia Musatescu,
Adrien Mazarguil
Ferruh Yigit <ferruh.yigit@intel.com> writes:
> On 7/30/2019 4:43 PM, Aaron Conole wrote:
>> Ferruh Yigit <ferruh.yigit@intel.com> writes:
>>
>>> On 7/30/2019 3:42 PM, Aaron Conole wrote:
>>>> David Marchand <david.marchand@redhat.com> writes:
>>>>
>>>>> On Wed, Jul 10, 2019 at 11:49 PM Thomas Monjalon <thomas@monjalon.net> wrote:
>>>>>>
>>>>>> 09/07/2019 13:09, Bernard Iremonger:
>>>>>>> This patch fixes the out-of-bounds coverity issue by removing the
>>>>>>> offending line of code at line 107 in rte_flow_classify_parse.c
>>>>>>> which is never executed.
>>>>>>>
>>>>>>> Coverity issue: 343454
>>>>>>>
>>>>>>> Fixes: be41ac2a330f ("flow_classify: introduce flow classify library")
>>>>>>> Cc: stable@dpdk.org
>>>>>>> Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
>>>>>>
>>>>>> Applied, thanks
>>>>>
>>>>> We have a segfault in the unit tests since this patch.
>>>>
>>>> I think this patch is still correct. The issue is in the semantic of
>>>> the flow classify pattern. It *MUST* always have a valid end marker,
>>>> but the test passes an invalid end marker. This causes the bounds to
>>>> exceed.
>>>>
>>>> So, it would be best to fix it, either by having a "failure" on unknown
>>>> markers (f.e. -1), or by passing a length around. However, the crash
>>>> should be expected. The fact that the previous code was also incorrect
>>>> and resulted in no segfault is pure luck.
>>>>
>>>> See rte_flow_classify_parse.c:80 and test_flow_classify.c:387
>>>>
>>>> I would be in favor of passing the lengths of the two arrays to these
>>>> APIs. That would let us still make use of the markers (for valid
>>>> construction), but also let us reason about lengths in a sane way.
>>>>
>>>> WDYT?
>>>>
>>>
>>> +1, I also just replied with something very similar.
>>>
>>> With current API the testcase is wrong, and it will crash, also the invalid
>>> action one has exact same problem.
>>>
>>> The API can be updated as you suggested, with a length field and testcases can
>>> be added back.
>>>
>>> What worries me more is the rte_flow, which uses same arguments, and open to
>>> same errors, should we consider updating rte_flow APIs to have lengths values too?
>>
>> Probably.
>>
>> Here's a first crack at the change I think is appropriate. I have done
>> some limited testing. Let me know if you want me to submit it formally.
>>
>> ---------------------------- 8< ---------------------------------
>> Subject: [PATCH] rte_flow_classify: fix up the API and preserve ABI
>>
>> Introduces a new API for doing length validations, and preserves the old semantics
>> and API. The previous API couldn't handle corrupted end markers. A future
>> version of the API might be able to eschew the end marker and trust the length,
>> but that is a discussion for future.
>>
>> Signed-off-by: Aaron Conole <aconole@redhat.com>
>> ---
>> app/test/test_flow_classify.c | 30 +-------
>> lib/librte_flow_classify/rte_flow_classify.c | 72 +++++++++++++++++---
>> lib/librte_flow_classify/rte_flow_classify.h | 28 ++++++++
>> 3 files changed, 91 insertions(+), 39 deletions(-)
>>
>> diff --git a/app/test/test_flow_classify.c b/app/test/test_flow_classify.c
>> index 6bbaad364..ff5265c6a 100644
>> --- a/app/test/test_flow_classify.c
>> +++ b/app/test/test_flow_classify.c
>> @@ -125,7 +125,6 @@ static struct rte_flow_item udp_item_bad = { RTE_FLOW_ITEM_TYPE_UDP,
>>
>> static struct rte_flow_item end_item = { RTE_FLOW_ITEM_TYPE_END,
>> 0, 0, 0 };
>> -static struct rte_flow_item end_item_bad = { -1, 0, 0, 0 };
>>
>> /* test TCP pattern:
>> * "eth / ipv4 src spec 1.2.3.4 src mask 255.255.255.00 dst spec 5.6.7.8
>> @@ -181,7 +180,6 @@ static struct rte_flow_action count_action = { RTE_FLOW_ACTION_TYPE_COUNT,
>> static struct rte_flow_action count_action_bad = { -1, 0};
>>
>> static struct rte_flow_action end_action = { RTE_FLOW_ACTION_TYPE_END, 0};
>> -static struct rte_flow_action end_action_bad = { -1, 0};
>>
>> static struct rte_flow_action actions[2];
>>
>> @@ -384,7 +382,7 @@ test_invalid_patterns(void)
>>
>> pattern[1] = ipv4_udp_item_1;
>> pattern[2] = udp_item_bad;
>> - pattern[3] = end_item_bad;
>> + pattern[3] = end_item;
>>
>> ret = rte_flow_classify_validate(cls->cls, &attr, pattern,
>> actions, &error);
>> @@ -458,32 +456,6 @@ test_invalid_actions(void)
>> return -1;
>> }
>>
>> - actions[0] = count_action;
>> - actions[1] = end_action_bad;
>> -
>> - ret = rte_flow_classify_validate(cls->cls, &attr, pattern,
>> - actions, &error);
>> - if (!ret) {
>> - printf("Line %i: rte_flow_classify_validate", __LINE__);
>> - printf(" should have failed!\n");
>> - return -1;
>> - }
>> -
>> - rule = rte_flow_classify_table_entry_add(cls->cls, &attr, pattern,
>> - actions, &key_found, &error);
>> - if (rule) {
>> - printf("Line %i: flow_classify_table_entry_add", __LINE__);
>> - printf(" should have failed!\n");
>> - return -1;
>> - }
>> -
>> - ret = rte_flow_classify_table_entry_delete(cls->cls, rule);
>> - if (!ret) {
>> - printf("Line %i: rte_flow_classify_table_entry_delete",
>> - __LINE__);
>> - printf("should have failed!\n");
>> - return -1;
>> - }
>> return 0;
>> }
>
> +1 to unit test updates, lgtm.
>
> And I am for pushing the library updates to the next release, but please find a
> few comments for now.
Okay - I'll do that. But we probably will need to note that these APIs
should get deprecated. Not sure if that should happen in this release -
as the author of most of the code, maybe you would take care of that
part? :)
>
>>
>> diff --git a/lib/librte_flow_classify/rte_flow_classify.c b/lib/librte_flow_classify/rte_flow_classify.c
>> index 5ff585803..3ca1b1b44 100644
>> --- a/lib/librte_flow_classify/rte_flow_classify.c
>> +++ b/lib/librte_flow_classify/rte_flow_classify.c
>> @@ -89,18 +89,51 @@ struct rte_flow_classify_rule {
>> void *entry_ptr; /* handle to the table entry for rule meta data */
>> };
>>
>> +static size_t
>> +calc_flow_item_alen(const struct rte_flow_item pattern[])
>> +{
>> + size_t i = 0;
>> + while (pattern && (pattern + i)->type != RTE_FLOW_ITEM_TYPE_END)
>> + i++;
>> + return i + 1;
>
> I think better to send '0' if the pointer is NULL, (instead of 1)
Okay. Makes sense.
> <...>
>
>> @@ -186,6 +186,34 @@ int
>> rte_flow_classify_table_create(struct rte_flow_classifier *cls,
>> struct rte_flow_classify_table_params *params);
>>
>> +/**
>> + * Flow classify validate
>> + *
>> + * @param cls
>> + * Handle to flow classifier instance
>> + * @param[in] attr
>> + * Flow rule attributes
>> + * @param[in] pattern
>> + * Pattern specification (list terminated by the END pattern item).
>> + * @param[in] actions
>> + * Associated actions (list terminated by the END pattern item).
>> + * @param[out] error
>> + * Perform verbose error reporting if not NULL. Structure
>> + * initialised in case of error only.
>> + * @return
>> + * 0 on success, error code otherwise
>> + */
>> +__rte_experimental
>> +int
>> +rte_flow_classify_validate_l(struct rte_flow_classifier *cls,
>> + const struct rte_flow_attr *attr,
>> + const struct rte_flow_item pattern[],
>> + const size_t pattern_l,
>> + const struct rte_flow_action actions[],
>> + const size_t actions_l,
>> + struct rte_flow_error *error);
>
> The doxygen comment is missing for 'pattern_l' & 'actions_l' but from code it is
> number of items in the lists, this is duplication of the END marker information.
> Instead, if those lengths are the length of the arrays will it be easier for the
> user? So user won't need to calculate the item count but can pass the size of
> the array. This still prevents API access out of the array.
>
> Anyway, as suggested above lets not make these decisions just a few days before
> the release, but just get the unit test fix for the release, does it make sense?
Sure.
> And if so, can you send the unit test patch?
Will do.
> Thanks,
> ferruh
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [dpdk-stable] [PATCH] librte_flow_classify: fix out-of-bounds access
2019-07-30 15:43 2% ` Aaron Conole
@ 2019-07-30 16:55 0% ` Ferruh Yigit
2019-07-30 17:30 0% ` Aaron Conole
0 siblings, 1 reply; 200+ results
From: Ferruh Yigit @ 2019-07-30 16:55 UTC (permalink / raw)
To: Aaron Conole
Cc: David Marchand, Bernard Iremonger, dev, dpdk stable,
Thomas Monjalon, Singh, Jasvinder, Flavia Musatescu,
Adrien Mazarguil
On 7/30/2019 4:43 PM, Aaron Conole wrote:
> Ferruh Yigit <ferruh.yigit@intel.com> writes:
>
>> On 7/30/2019 3:42 PM, Aaron Conole wrote:
>>> David Marchand <david.marchand@redhat.com> writes:
>>>
>>>> On Wed, Jul 10, 2019 at 11:49 PM Thomas Monjalon <thomas@monjalon.net> wrote:
>>>>>
>>>>> 09/07/2019 13:09, Bernard Iremonger:
>>>>>> This patch fixes the out-of-bounds coverity issue by removing the
>>>>>> offending line of code at line 107 in rte_flow_classify_parse.c
>>>>>> which is never executed.
>>>>>>
>>>>>> Coverity issue: 343454
>>>>>>
>>>>>> Fixes: be41ac2a330f ("flow_classify: introduce flow classify library")
>>>>>> Cc: stable@dpdk.org
>>>>>> Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
>>>>>
>>>>> Applied, thanks
>>>>
>>>> We have a segfault in the unit tests since this patch.
>>>
>>> I think this patch is still correct. The issue is in the semantic of
>>> the flow classify pattern. It *MUST* always have a valid end marker,
>>> but the test passes an invalid end marker. This causes the bounds to
>>> exceed.
>>>
>>> So, it would be best to fix it, either by having a "failure" on unknown
>>> markers (f.e. -1), or by passing a length around. However, the crash
>>> should be expected. The fact that the previous code was also incorrect
>>> and resulted in no segfault is pure luck.
>>>
>>> See rte_flow_classify_parse.c:80 and test_flow_classify.c:387
>>>
>>> I would be in favor of passing the lengths of the two arrays to these
>>> APIs. That would let us still make use of the markers (for valid
>>> construction), but also let us reason about lengths in a sane way.
>>>
>>> WDYT?
>>>
>>
>> +1, I also just replied with something very similar.
>>
>> With current API the testcase is wrong, and it will crash, also the invalid
>> action one has exact same problem.
>>
>> The API can be updated as you suggested, with a length field and testcases can
>> be added back.
>>
>> What worries me more is the rte_flow, which uses same arguments, and open to
>> same errors, should we consider updating rte_flow APIs to have lengths values too?
>
> Probably.
>
> Here's a first crack at the change I think is appropriate. I have done
> some limited testing. Let me know if you want me to submit it formally.
>
> ---------------------------- 8< ---------------------------------
> Subject: [PATCH] rte_flow_classify: fix up the API and preserve ABI
>
> Introduces a new API for doing length validations, and preserves the old semantics
> and API. The previous API couldn't handle corrupted end markers. A future
> version of the API might be able to eschew the end marker and trust the length,
> but that is a discussion for future.
>
> Signed-off-by: Aaron Conole <aconole@redhat.com>
> ---
> app/test/test_flow_classify.c | 30 +-------
> lib/librte_flow_classify/rte_flow_classify.c | 72 +++++++++++++++++---
> lib/librte_flow_classify/rte_flow_classify.h | 28 ++++++++
> 3 files changed, 91 insertions(+), 39 deletions(-)
>
> diff --git a/app/test/test_flow_classify.c b/app/test/test_flow_classify.c
> index 6bbaad364..ff5265c6a 100644
> --- a/app/test/test_flow_classify.c
> +++ b/app/test/test_flow_classify.c
> @@ -125,7 +125,6 @@ static struct rte_flow_item udp_item_bad = { RTE_FLOW_ITEM_TYPE_UDP,
>
> static struct rte_flow_item end_item = { RTE_FLOW_ITEM_TYPE_END,
> 0, 0, 0 };
> -static struct rte_flow_item end_item_bad = { -1, 0, 0, 0 };
>
> /* test TCP pattern:
> * "eth / ipv4 src spec 1.2.3.4 src mask 255.255.255.00 dst spec 5.6.7.8
> @@ -181,7 +180,6 @@ static struct rte_flow_action count_action = { RTE_FLOW_ACTION_TYPE_COUNT,
> static struct rte_flow_action count_action_bad = { -1, 0};
>
> static struct rte_flow_action end_action = { RTE_FLOW_ACTION_TYPE_END, 0};
> -static struct rte_flow_action end_action_bad = { -1, 0};
>
> static struct rte_flow_action actions[2];
>
> @@ -384,7 +382,7 @@ test_invalid_patterns(void)
>
> pattern[1] = ipv4_udp_item_1;
> pattern[2] = udp_item_bad;
> - pattern[3] = end_item_bad;
> + pattern[3] = end_item;
>
> ret = rte_flow_classify_validate(cls->cls, &attr, pattern,
> actions, &error);
> @@ -458,32 +456,6 @@ test_invalid_actions(void)
> return -1;
> }
>
> - actions[0] = count_action;
> - actions[1] = end_action_bad;
> -
> - ret = rte_flow_classify_validate(cls->cls, &attr, pattern,
> - actions, &error);
> - if (!ret) {
> - printf("Line %i: rte_flow_classify_validate", __LINE__);
> - printf(" should have failed!\n");
> - return -1;
> - }
> -
> - rule = rte_flow_classify_table_entry_add(cls->cls, &attr, pattern,
> - actions, &key_found, &error);
> - if (rule) {
> - printf("Line %i: flow_classify_table_entry_add", __LINE__);
> - printf(" should have failed!\n");
> - return -1;
> - }
> -
> - ret = rte_flow_classify_table_entry_delete(cls->cls, rule);
> - if (!ret) {
> - printf("Line %i: rte_flow_classify_table_entry_delete",
> - __LINE__);
> - printf("should have failed!\n");
> - return -1;
> - }
> return 0;
> }
+1 to unit test updates, lgtm.
And I am for pushing the library updates to the next release, but please find a
few comments for now.
>
> diff --git a/lib/librte_flow_classify/rte_flow_classify.c b/lib/librte_flow_classify/rte_flow_classify.c
> index 5ff585803..3ca1b1b44 100644
> --- a/lib/librte_flow_classify/rte_flow_classify.c
> +++ b/lib/librte_flow_classify/rte_flow_classify.c
> @@ -89,18 +89,51 @@ struct rte_flow_classify_rule {
> void *entry_ptr; /* handle to the table entry for rule meta data */
> };
>
> +static size_t
> +calc_flow_item_alen(const struct rte_flow_item pattern[])
> +{
> + size_t i = 0;
> + while (pattern && (pattern + i)->type != RTE_FLOW_ITEM_TYPE_END)
> + i++;
> + return i + 1;
I think better to send '0' if the pointer is NULL, (instead of 1)
<...>
> @@ -186,6 +186,34 @@ int
> rte_flow_classify_table_create(struct rte_flow_classifier *cls,
> struct rte_flow_classify_table_params *params);
>
> +/**
> + * Flow classify validate
> + *
> + * @param cls
> + * Handle to flow classifier instance
> + * @param[in] attr
> + * Flow rule attributes
> + * @param[in] pattern
> + * Pattern specification (list terminated by the END pattern item).
> + * @param[in] actions
> + * Associated actions (list terminated by the END pattern item).
> + * @param[out] error
> + * Perform verbose error reporting if not NULL. Structure
> + * initialised in case of error only.
> + * @return
> + * 0 on success, error code otherwise
> + */
> +__rte_experimental
> +int
> +rte_flow_classify_validate_l(struct rte_flow_classifier *cls,
> + const struct rte_flow_attr *attr,
> + const struct rte_flow_item pattern[],
> + const size_t pattern_l,
> + const struct rte_flow_action actions[],
> + const size_t actions_l,
> + struct rte_flow_error *error);
The doxygen comment is missing for 'pattern_l' & 'actions_l' but from code it is
number of items in the lists, this is duplication of the END marker information.
Instead, if those lengths are the length of the arrays will it be easier for the
user? So user won't need to calculate the item count but can pass the size of
the array. This still prevents API access out of the array.
Anyway, as suggested above lets not make these decisions just a few days before
the release, but just get the unit test fix for the release, does it make sense?
And if so, can you send the unit test patch?
Thanks,
ferruh
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [EXT] Re: [RFC 19.11 1/2] ethdev: make DPDK core functions non-inline
2019-07-30 16:05 5% ` Bruce Richardson
@ 2019-07-30 16:24 3% ` Jerin Jacob Kollanukkaran
0 siblings, 0 replies; 200+ results
From: Jerin Jacob Kollanukkaran @ 2019-07-30 16:24 UTC (permalink / raw)
To: Bruce Richardson; +Cc: Marcin Zapolski, dev
> -----Original Message-----
> From: Bruce Richardson <bruce.richardson@intel.com>
> Sent: Tuesday, July 30, 2019 9:36 PM
> To: Jerin Jacob Kollanukkaran <jerinj@marvell.com>
> Cc: Marcin Zapolski <marcinx.a.zapolski@intel.com>; dev@dpdk.org
> Subject: [EXT] Re: [dpdk-dev] [RFC 19.11 1/2] ethdev: make DPDK core functions
> non-inline
>
> On Tue, Jul 30, 2019 at 03:45:38PM +0000, Jerin Jacob Kollanukkaran wrote:
> > > -----Original Message-----
> > > From: Bruce Richardson <bruce.richardson@intel.com>
> > > Sent: Tuesday, July 30, 2019 9:02 PM
> > > To: Jerin Jacob Kollanukkaran <jerinj@marvell.com>
> > > Cc: Marcin Zapolski <marcinx.a.zapolski@intel.com>; dev@dpdk.org
> > > Subject: [EXT] Re: [dpdk-dev] [RFC 19.11 1/2] ethdev: make DPDK core
> > > functions non-inline
> > >
> > > --------------------------------------------------------------------
> > > -- On Tue, Jul 30, 2019 at 03:01:00PM +0000, Jerin Jacob
> > > Kollanukkaran wrote:
> > > > > -----Original Message----- From: dev <dev-bounces@dpdk.org> On
> > > > > Behalf Of Marcin Zapolski Sent: Tuesday, July 30, 2019 6:20 PM To:
> > > > > dev@dpdk.org Cc: Marcin Zapolski <marcinx.a.zapolski@intel.com>
> > > > > Subject: [dpdk-dev] [RFC 19.11 1/2] ethdev: make DPDK core
> > > > > functions
> > > > > non- inline
> > > > >
> > > > > Make rte_eth_rx_burst, rte_eth_tx_burst and other static inline
> > > > > ethdev functions not inline. They are referencing DPDK internal
> > > > > structures and inlining forces those structures to be exposed to
> > > > > user
> > > applications.
> > > > >
> > > > > In internal testing with i40e NICs a performance drop of about
> > > > > 2% was observed with testpmd.
> > > >
> > > > I tested on two class of arm64 machines(Highend and lowend) one
> > > > has 1.4% drop And other one has 3.6% drop.
> > > >
> > > This is with testpmd only right? I'd just point out that we need to
> > > remember that these numbers need to be scaled down appropriately for
> > > a realworld app where IO is only a (hopefully small) proportion of
> > > the packet processing budget. For example, I would expect the ~2%
> > > drop we saw in testpmd to correspond to <0.5% drop in something like OVS.
> >
> > I see it as bit different view, Cycles saved infrastructure layer,
> > cycles gained in application. So IMO it vary between end user
> > application need what kind of machine it runs.
> >
> Sure. My thinking more is that to get ABI compatibility involves some tradeoffs
> and spending one more cycle per-packet when an app workload is typically
> hundreds of cycles, I believe, is a small cost worth paying.
I agree. But, We need take only the cost, If it is really costs.I don't think, we don't
need two level of indirection.
>
> > >
> > > > I second to not expose internal data structure to avoid ABI break.
> > > >
> > > > IMO, This patch has performance issue due to it is fixing it in
> > > > simple way.
> > > >
> > > > It is not worth two have function call overhead to call the driver
> > > > function. Some thoughts below to reduce the performance impact
> > > > without exposing internal structures.
> > > >
> > > The big concern I have with what you propose is that would involve
> > > changing each and every ethdev driver in DPDK! I'd prefer to make
> > > sure that the impact of this change is actually felt in real-world
> > > apps before we start looking to make such updates across the DPDK
> codebase.
> >
> > I see those changes are NO BRAINER from driver POV. Once we add in one
> > driver, individual PMD Maintainer can update easily. I think, we can do it once
> for all.
>
> Ok, if it's doable in one go then sure. The issue is that if even one driver is not
> updated we can't switch over, all have to effectively be done simultaneously. [It
I agree. But I think, we can give enough time for driver to migrate.
If it does not migrate in time. We can take necessary action to fix it.
If it is a no brainer changes then all drivers will do it. We cannot pay cost
Because one driver is not maintaining. I can commit to take care of ALL Marvell PMDs.
> would also make backporting fixes trickier, but I wouldn't be concerned about
> that particularly.]
>
> Have you tried out making the changes to a driver or two, to see how large the
> delta is? [And to verify it doesn't affect performance]
I hope, The RFC author will take first stab, I can help in reviewing it.
>
> > I am sure, you must aware of How hard is make 2% improvement in
> > driver. I can spend time in This NO brainer to get 2% improvement back. I
> prefer later.
> >
> The other alternative I see is to leave the inline functions there, just disabled by
> default, and put in a build-time option for reduced ABI compatibility. That way
> the standard-built packages are all ABI compatible, but for those who absolutely
> need max perf and are rolling-their-own-build to get it can disable that ABI
> compatibility.
But currently there no ABI break. Right? We should do it before we have one on those
structures. I think, we should cleanup the low hanging ones first and to fast
path structures in parallel.
>
> /Bruce
^ permalink raw reply [relevance 3%]
* Re: [dpdk-dev] [RFC 19.11 1/2] ethdev: make DPDK core functions non-inline
2019-07-30 15:45 0% [dpdk-dev] [RFC 19.11 1/2] ethdev: make DPDK core functions non-inline Jerin Jacob Kollanukkaran
@ 2019-07-30 16:05 5% ` Bruce Richardson
2019-07-30 16:24 3% ` [dpdk-dev] [EXT] " Jerin Jacob Kollanukkaran
0 siblings, 1 reply; 200+ results
From: Bruce Richardson @ 2019-07-30 16:05 UTC (permalink / raw)
To: Jerin Jacob Kollanukkaran; +Cc: Marcin Zapolski, dev
On Tue, Jul 30, 2019 at 03:45:38PM +0000, Jerin Jacob Kollanukkaran wrote:
> > -----Original Message-----
> > From: Bruce Richardson <bruce.richardson@intel.com>
> > Sent: Tuesday, July 30, 2019 9:02 PM
> > To: Jerin Jacob Kollanukkaran <jerinj@marvell.com>
> > Cc: Marcin Zapolski <marcinx.a.zapolski@intel.com>; dev@dpdk.org
> > Subject: [EXT] Re: [dpdk-dev] [RFC 19.11 1/2] ethdev: make DPDK core functions
> > non-inline
> >
> > ----------------------------------------------------------------------
> > On Tue, Jul 30, 2019 at 03:01:00PM +0000, Jerin Jacob Kollanukkaran wrote:
> > > > -----Original Message----- From: dev <dev-bounces@dpdk.org> On
> > > > Behalf Of Marcin Zapolski Sent: Tuesday, July 30, 2019 6:20 PM To:
> > > > dev@dpdk.org Cc: Marcin Zapolski <marcinx.a.zapolski@intel.com>
> > > > Subject: [dpdk-dev] [RFC 19.11 1/2] ethdev: make DPDK core functions
> > > > non- inline
> > > >
> > > > Make rte_eth_rx_burst, rte_eth_tx_burst and other static inline
> > > > ethdev functions not inline. They are referencing DPDK internal
> > > > structures and inlining forces those structures to be exposed to user
> > applications.
> > > >
> > > > In internal testing with i40e NICs a performance drop of about 2%
> > > > was observed with testpmd.
> > >
> > > I tested on two class of arm64 machines(Highend and lowend) one has
> > > 1.4% drop And other one has 3.6% drop.
> > >
> > This is with testpmd only right? I'd just point out that we need to remember that
> > these numbers need to be scaled down appropriately for a realworld app where
> > IO is only a (hopefully small) proportion of the packet processing budget. For
> > example, I would expect the ~2% drop we saw in testpmd to correspond to
> > <0.5% drop in something like OVS.
>
> I see it as bit different view, Cycles saved infrastructure layer, cycles gained in
> application. So IMO it vary between end user application need what kind of
> machine it runs.
>
Sure. My thinking more is that to get ABI compatibility involves some
tradeoffs and spending one more cycle per-packet when an app workload is
typically hundreds of cycles, I believe, is a small cost worth paying.
> >
> > > I second to not expose internal data structure to avoid ABI break.
> > >
> > > IMO, This patch has performance issue due to it is fixing it in simple
> > > way.
> > >
> > > It is not worth two have function call overhead to call the driver
> > > function. Some thoughts below to reduce the performance impact
> > > without exposing internal structures.
> > >
> > The big concern I have with what you propose is that would involve changing
> > each and every ethdev driver in DPDK! I'd prefer to make sure that the impact of
> > this change is actually felt in real-world apps before we start looking to make
> > such updates across the DPDK codebase.
>
> I see those changes are NO BRAINER from driver POV. Once we add in one driver, individual
> PMD Maintainer can update easily. I think, we can do it once for all.
Ok, if it's doable in one go then sure. The issue is that if even one driver
is not updated we can't switch over, all have to effectively be done
simultaneously. [It would also make backporting fixes trickier, but I
wouldn't be concerned about that particularly.]
Have you tried out making the changes to a driver or two, to see how large
the delta is? [And to verify it doesn't affect performance]
> I am sure, you must aware of How hard is make 2% improvement in driver. I can spend time in
> This NO brainer to get 2% improvement back. I prefer later.
>
The other alternative I see is to leave the inline functions there, just
disabled by default, and put in a build-time option for reduced ABI
compatibility. That way the standard-built packages are all ABI compatible,
but for those who absolutely need max perf and are rolling-their-own-build
to get it can disable that ABI compatibility.
/Bruce
^ permalink raw reply [relevance 5%]
* Re: [dpdk-dev] [RFC 19.11 1/2] ethdev: make DPDK core functions non-inline
@ 2019-07-30 15:45 0% Jerin Jacob Kollanukkaran
2019-07-30 16:05 5% ` Bruce Richardson
0 siblings, 1 reply; 200+ results
From: Jerin Jacob Kollanukkaran @ 2019-07-30 15:45 UTC (permalink / raw)
To: Bruce Richardson; +Cc: Marcin Zapolski, dev
> -----Original Message-----
> From: Bruce Richardson <bruce.richardson@intel.com>
> Sent: Tuesday, July 30, 2019 9:02 PM
> To: Jerin Jacob Kollanukkaran <jerinj@marvell.com>
> Cc: Marcin Zapolski <marcinx.a.zapolski@intel.com>; dev@dpdk.org
> Subject: [EXT] Re: [dpdk-dev] [RFC 19.11 1/2] ethdev: make DPDK core functions
> non-inline
>
> ----------------------------------------------------------------------
> On Tue, Jul 30, 2019 at 03:01:00PM +0000, Jerin Jacob Kollanukkaran wrote:
> > > -----Original Message----- From: dev <dev-bounces@dpdk.org> On
> > > Behalf Of Marcin Zapolski Sent: Tuesday, July 30, 2019 6:20 PM To:
> > > dev@dpdk.org Cc: Marcin Zapolski <marcinx.a.zapolski@intel.com>
> > > Subject: [dpdk-dev] [RFC 19.11 1/2] ethdev: make DPDK core functions
> > > non- inline
> > >
> > > Make rte_eth_rx_burst, rte_eth_tx_burst and other static inline
> > > ethdev functions not inline. They are referencing DPDK internal
> > > structures and inlining forces those structures to be exposed to user
> applications.
> > >
> > > In internal testing with i40e NICs a performance drop of about 2%
> > > was observed with testpmd.
> >
> > I tested on two class of arm64 machines(Highend and lowend) one has
> > 1.4% drop And other one has 3.6% drop.
> >
> This is with testpmd only right? I'd just point out that we need to remember that
> these numbers need to be scaled down appropriately for a realworld app where
> IO is only a (hopefully small) proportion of the packet processing budget. For
> example, I would expect the ~2% drop we saw in testpmd to correspond to
> <0.5% drop in something like OVS.
I see it as bit different view, Cycles saved infrastructure layer, cycles gained in
application. So IMO it vary between end user application need what kind of
machine it runs.
>
> > I second to not expose internal data structure to avoid ABI break.
> >
> > IMO, This patch has performance issue due to it is fixing it in simple
> > way.
> >
> > It is not worth two have function call overhead to call the driver
> > function. Some thoughts below to reduce the performance impact
> > without exposing internal structures.
> >
> The big concern I have with what you propose is that would involve changing
> each and every ethdev driver in DPDK! I'd prefer to make sure that the impact of
> this change is actually felt in real-world apps before we start looking to make
> such updates across the DPDK codebase.
I see those changes are NO BRAINER from driver POV. Once we add in one driver, individual
PMD Maintainer can update easily. I think, we can do it once for all.
I am sure, you must aware of How hard is make 2% improvement in driver. I can spend time in
This NO brainer to get 2% improvement back. I prefer later.
>
> > And I think, We need to follow the similar mechanism for cryptodev,
> > Eventdev, rawdev Etc so bring the common scheme to address this semantics
> will be use full.
> >
> Agreed.
>
> Regards,
> /Bruce
^ permalink raw reply [relevance 0%]
* Re: [dpdk-dev] [dpdk-stable] [PATCH] librte_flow_classify: fix out-of-bounds access
@ 2019-07-30 15:43 2% ` Aaron Conole
2019-07-30 16:55 0% ` Ferruh Yigit
0 siblings, 1 reply; 200+ results
From: Aaron Conole @ 2019-07-30 15:43 UTC (permalink / raw)
To: Ferruh Yigit
Cc: David Marchand, Bernard Iremonger, dev, dpdk stable,
Thomas Monjalon, Singh, Jasvinder, Flavia Musatescu,
Adrien Mazarguil
Ferruh Yigit <ferruh.yigit@intel.com> writes:
> On 7/30/2019 3:42 PM, Aaron Conole wrote:
>> David Marchand <david.marchand@redhat.com> writes:
>>
>>> On Wed, Jul 10, 2019 at 11:49 PM Thomas Monjalon <thomas@monjalon.net> wrote:
>>>>
>>>> 09/07/2019 13:09, Bernard Iremonger:
>>>>> This patch fixes the out-of-bounds coverity issue by removing the
>>>>> offending line of code at line 107 in rte_flow_classify_parse.c
>>>>> which is never executed.
>>>>>
>>>>> Coverity issue: 343454
>>>>>
>>>>> Fixes: be41ac2a330f ("flow_classify: introduce flow classify library")
>>>>> Cc: stable@dpdk.org
>>>>> Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
>>>>
>>>> Applied, thanks
>>>
>>> We have a segfault in the unit tests since this patch.
>>
>> I think this patch is still correct. The issue is in the semantic of
>> the flow classify pattern. It *MUST* always have a valid end marker,
>> but the test passes an invalid end marker. This causes the bounds to
>> exceed.
>>
>> So, it would be best to fix it, either by having a "failure" on unknown
>> markers (f.e. -1), or by passing a length around. However, the crash
>> should be expected. The fact that the previous code was also incorrect
>> and resulted in no segfault is pure luck.
>>
>> See rte_flow_classify_parse.c:80 and test_flow_classify.c:387
>>
>> I would be in favor of passing the lengths of the two arrays to these
>> APIs. That would let us still make use of the markers (for valid
>> construction), but also let us reason about lengths in a sane way.
>>
>> WDYT?
>>
>
> +1, I also just replied with something very similar.
>
> With current API the testcase is wrong, and it will crash, also the invalid
> action one has exact same problem.
>
> The API can be updated as you suggested, with a length field and testcases can
> be added back.
>
> What worries me more is the rte_flow, which uses same arguments, and open to
> same errors, should we consider updating rte_flow APIs to have lengths values too?
Probably.
Here's a first crack at the change I think is appropriate. I have done
some limited testing. Let me know if you want me to submit it formally.
---------------------------- 8< ---------------------------------
Subject: [PATCH] rte_flow_classify: fix up the API and preserve ABI
Introduces a new API for doing length validations, and preserves the old semantics
and API. The previous API couldn't handle corrupted end markers. A future
version of the API might be able to eschew the end marker and trust the length,
but that is a discussion for future.
Signed-off-by: Aaron Conole <aconole@redhat.com>
---
app/test/test_flow_classify.c | 30 +-------
lib/librte_flow_classify/rte_flow_classify.c | 72 +++++++++++++++++---
lib/librte_flow_classify/rte_flow_classify.h | 28 ++++++++
3 files changed, 91 insertions(+), 39 deletions(-)
diff --git a/app/test/test_flow_classify.c b/app/test/test_flow_classify.c
index 6bbaad364..ff5265c6a 100644
--- a/app/test/test_flow_classify.c
+++ b/app/test/test_flow_classify.c
@@ -125,7 +125,6 @@ static struct rte_flow_item udp_item_bad = { RTE_FLOW_ITEM_TYPE_UDP,
static struct rte_flow_item end_item = { RTE_FLOW_ITEM_TYPE_END,
0, 0, 0 };
-static struct rte_flow_item end_item_bad = { -1, 0, 0, 0 };
/* test TCP pattern:
* "eth / ipv4 src spec 1.2.3.4 src mask 255.255.255.00 dst spec 5.6.7.8
@@ -181,7 +180,6 @@ static struct rte_flow_action count_action = { RTE_FLOW_ACTION_TYPE_COUNT,
static struct rte_flow_action count_action_bad = { -1, 0};
static struct rte_flow_action end_action = { RTE_FLOW_ACTION_TYPE_END, 0};
-static struct rte_flow_action end_action_bad = { -1, 0};
static struct rte_flow_action actions[2];
@@ -384,7 +382,7 @@ test_invalid_patterns(void)
pattern[1] = ipv4_udp_item_1;
pattern[2] = udp_item_bad;
- pattern[3] = end_item_bad;
+ pattern[3] = end_item;
ret = rte_flow_classify_validate(cls->cls, &attr, pattern,
actions, &error);
@@ -458,32 +456,6 @@ test_invalid_actions(void)
return -1;
}
- actions[0] = count_action;
- actions[1] = end_action_bad;
-
- ret = rte_flow_classify_validate(cls->cls, &attr, pattern,
- actions, &error);
- if (!ret) {
- printf("Line %i: rte_flow_classify_validate", __LINE__);
- printf(" should have failed!\n");
- return -1;
- }
-
- rule = rte_flow_classify_table_entry_add(cls->cls, &attr, pattern,
- actions, &key_found, &error);
- if (rule) {
- printf("Line %i: flow_classify_table_entry_add", __LINE__);
- printf(" should have failed!\n");
- return -1;
- }
-
- ret = rte_flow_classify_table_entry_delete(cls->cls, rule);
- if (!ret) {
- printf("Line %i: rte_flow_classify_table_entry_delete",
- __LINE__);
- printf("should have failed!\n");
- return -1;
- }
return 0;
}
diff --git a/lib/librte_flow_classify/rte_flow_classify.c b/lib/librte_flow_classify/rte_flow_classify.c
index 5ff585803..3ca1b1b44 100644
--- a/lib/librte_flow_classify/rte_flow_classify.c
+++ b/lib/librte_flow_classify/rte_flow_classify.c
@@ -89,18 +89,51 @@ struct rte_flow_classify_rule {
void *entry_ptr; /* handle to the table entry for rule meta data */
};
+static size_t
+calc_flow_item_alen(const struct rte_flow_item pattern[])
+{
+ size_t i = 0;
+ while (pattern && (pattern + i)->type != RTE_FLOW_ITEM_TYPE_END)
+ i++;
+ return i + 1;
+}
+
+static size_t
+calc_flow_action_alen(const struct rte_flow_action actions[])
+{
+ size_t i = 0;
+ while (actions && (actions + i)->type != RTE_FLOW_ACTION_TYPE_END)
+ i++;
+ return i + 1;
+}
+
+int
+rte_flow_classify_validate(struct rte_flow_classifier *cls,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item pattern[],
+ const struct rte_flow_action actions[],
+ struct rte_flow_error *error)
+{
+ return rte_flow_classify_validate_l(cls, attr, pattern,
+ calc_flow_item_alen(pattern),
+ actions,
+ calc_flow_action_alen(actions),
+ error);
+}
+
int
-rte_flow_classify_validate(
- struct rte_flow_classifier *cls,
- const struct rte_flow_attr *attr,
- const struct rte_flow_item pattern[],
- const struct rte_flow_action actions[],
- struct rte_flow_error *error)
+rte_flow_classify_validate_l(struct rte_flow_classifier *cls,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item pattern[],
+ size_t pattern_l,
+ const struct rte_flow_action actions[],
+ size_t actions_l,
+ struct rte_flow_error *error)
{
struct rte_flow_item *items;
parse_filter_t parse_filter;
uint32_t item_num = 0;
- uint32_t i = 0;
+ size_t i = 0;
int ret;
if (error == NULL)
@@ -134,17 +167,37 @@ rte_flow_classify_validate(
return -EINVAL;
}
+ while (i < actions_l && (actions + i)->type != RTE_FLOW_ACTION_TYPE_END)
+ i++;
+
+ if (i == actions_l) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION_NUM,
+ NULL, "Actions without end marker.");
+ return -EINVAL;
+ }
+
+ i = 0;
+
memset(&cls->ntuple_filter, 0, sizeof(cls->ntuple_filter));
/* Get the non-void item number of pattern */
- while ((pattern + i)->type != RTE_FLOW_ITEM_TYPE_END) {
+ while (i < pattern_l && (pattern + i)->type != RTE_FLOW_ITEM_TYPE_END) {
if ((pattern + i)->type != RTE_FLOW_ITEM_TYPE_VOID)
item_num++;
i++;
}
+
item_num++;
- items = malloc(item_num * sizeof(struct rte_flow_item));
+ if (i == pattern_l) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ NULL, "Pattern without end marker.");
+ return -EINVAL;
+ }
+
+ items = calloc(item_num, sizeof(struct rte_flow_item));
if (!items) {
rte_flow_error_set(error, ENOMEM,
RTE_FLOW_ERROR_TYPE_ITEM_NUM,
@@ -152,7 +205,6 @@ rte_flow_classify_validate(
return -ENOMEM;
}
- memset(items, 0, item_num * sizeof(struct rte_flow_item));
classify_pattern_skip_void_item(items, pattern);
parse_filter = classify_find_parse_filter_func(items);
diff --git a/lib/librte_flow_classify/rte_flow_classify.h b/lib/librte_flow_classify/rte_flow_classify.h
index 74d1ecaf5..0308f6fd2 100644
--- a/lib/librte_flow_classify/rte_flow_classify.h
+++ b/lib/librte_flow_classify/rte_flow_classify.h
@@ -186,6 +186,34 @@ int
rte_flow_classify_table_create(struct rte_flow_classifier *cls,
struct rte_flow_classify_table_params *params);
+/**
+ * Flow classify validate
+ *
+ * @param cls
+ * Handle to flow classifier instance
+ * @param[in] attr
+ * Flow rule attributes
+ * @param[in] pattern
+ * Pattern specification (list terminated by the END pattern item).
+ * @param[in] actions
+ * Associated actions (list terminated by the END pattern item).
+ * @param[out] error
+ * Perform verbose error reporting if not NULL. Structure
+ * initialised in case of error only.
+ * @return
+ * 0 on success, error code otherwise
+ */
+__rte_experimental
+int
+rte_flow_classify_validate_l(struct rte_flow_classifier *cls,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item pattern[],
+ const size_t pattern_l,
+ const struct rte_flow_action actions[],
+ const size_t actions_l,
+ struct rte_flow_error *error);
+
+
/**
* Flow classify validate
*
--
2.21.0
^ permalink raw reply [relevance 2%]
* Re: [dpdk-dev] [RFC 19.11 1/2] ethdev: make DPDK core functions non-inline
@ 2019-07-30 15:32 0% ` Bruce Richardson
0 siblings, 0 replies; 200+ results
From: Bruce Richardson @ 2019-07-30 15:32 UTC (permalink / raw)
To: Jerin Jacob Kollanukkaran; +Cc: Marcin Zapolski, dev
On Tue, Jul 30, 2019 at 03:01:00PM +0000, Jerin Jacob Kollanukkaran wrote:
> > -----Original Message----- From: dev <dev-bounces@dpdk.org> On Behalf
> > Of Marcin Zapolski Sent: Tuesday, July 30, 2019 6:20 PM To:
> > dev@dpdk.org Cc: Marcin Zapolski <marcinx.a.zapolski@intel.com>
> > Subject: [dpdk-dev] [RFC 19.11 1/2] ethdev: make DPDK core functions
> > non- inline
> >
> > Make rte_eth_rx_burst, rte_eth_tx_burst and other static inline ethdev
> > functions not inline. They are referencing DPDK internal structures and
> > inlining forces those structures to be exposed to user applications.
> >
> > In internal testing with i40e NICs a performance drop of about 2% was
> > observed with testpmd.
>
> I tested on two class of arm64 machines(Highend and lowend) one has 1.4%
> drop And other one has 3.6% drop.
>
This is with testpmd only right? I'd just point out that we need to
remember that these numbers need to be scaled down appropriately for a
realworld app where IO is only a (hopefully small) proportion of the packet
processing budget. For example, I would expect the ~2% drop we saw in
testpmd to correspond to <0.5% drop in something like OVS.
> I second to not expose internal data structure to avoid ABI break.
>
> IMO, This patch has performance issue due to it is fixing it in simple
> way.
>
> It is not worth two have function call overhead to call the driver
> function. Some thoughts below to reduce the performance impact without
> exposing internal structures.
>
The big concern I have with what you propose is that would involve changing
each and every ethdev driver in DPDK! I'd prefer to make sure that the
impact of this change is actually felt in real-world apps before we start
looking to make such updates across the DPDK codebase.
> And I think, We need to follow the similar mechanism for cryptodev, Eventdev, rawdev
> Etc so bring the common scheme to address this semantics will be use full.
>
Agreed.
Regards,
/Bruce
^ permalink raw reply [relevance 0%]
Results 7401-7600 of ~18000 next (older) | prev (newer) | reverse | sort options + mbox downloads above
-- links below jump to the message on this page --
2018-04-26 22:03 [dpdk-dev] [PATCH v4 0/4] lib/rib: Add Routing Information Base library Medvedkin Vladimir
2019-09-11 17:09 2% ` [dpdk-dev] [PATCH v5 00/12] lib: add RIB and FIB liraries Vladimir Medvedkin
2019-09-12 7:37 0% ` Morten Brørup
2019-09-12 9:47 0% ` Medvedkin, Vladimir
2019-05-25 18:43 [dpdk-dev] [RFC PATCH 0/2] introduce __rte_internal tag Neil Horman
2019-06-13 14:23 ` [dpdk-dev] [PATCH v2 0/10] dpdk: " Neil Horman
2019-08-06 10:03 0% ` Thomas Monjalon
2019-08-06 12:21 0% ` Neil Horman
2019-07-04 4:43 [dpdk-dev] [PATCH] doc: announce ABI change for RSS hash funtion simei
2019-08-06 11:34 4% ` Thomas Monjalon
2019-08-06 14:45 4% ` Ananyev, Konstantin
2019-08-10 20:39 4% ` Thomas Monjalon
2019-07-04 4:46 [dpdk-dev] [PATCH] doc: announce ABI change for rte flow RSS action simei
2019-08-06 11:31 4% ` Thomas Monjalon
2019-08-06 11:38 4% ` Zhang, Qi Z
2019-08-06 11:40 4% ` Zhang, Qi Z
2019-07-05 9:33 [dpdk-dev] [RFC PATCH 0/2] standardize rawdev names Bruce Richardson
2019-07-30 13:39 ` [dpdk-dev] [PATCH v2 " Thomas Monjalon
2019-07-30 13:39 ` [dpdk-dev] [PATCH v2 2/2] drivers/raw: standardize naming Thomas Monjalon
2019-07-31 7:29 0% ` Xu, Rosen
2019-07-09 11:09 [dpdk-dev] [PATCH] librte_flow_classify: fix out-of-bounds access Bernard Iremonger
2019-07-10 21:48 ` Thomas Monjalon
2019-07-29 13:09 ` David Marchand
2019-07-30 14:42 ` Aaron Conole
2019-07-30 14:48 ` [dpdk-dev] [dpdk-stable] " Ferruh Yigit
2019-07-30 15:43 2% ` Aaron Conole
2019-07-30 16:55 0% ` Ferruh Yigit
2019-07-30 17:30 0% ` Aaron Conole
2019-07-10 9:29 [dpdk-dev] [RFC] mbuf: support dynamic fields and flags Olivier Matz
2019-09-18 16:54 4% ` [dpdk-dev] [PATCH] " Olivier Matz
2019-09-21 4:54 0% ` Wang, Haiyue
2019-09-23 8:31 0% ` Olivier Matz
2019-09-23 11:01 0% ` Wang, Haiyue
2019-09-21 8:28 0% ` Wiles, Keith
2019-09-23 8:56 0% ` Morten Brørup
2019-09-23 9:41 0% ` Olivier Matz
2019-09-23 9:13 0% ` Olivier Matz
2019-09-23 15:14 2% ` Wiles, Keith
2019-09-23 16:16 3% ` Olivier Matz
2019-09-23 17:14 0% ` Wiles, Keith
2019-09-23 16:09 0% ` Wiles, Keith
2019-07-23 7:05 [dpdk-dev] [PATCH v8 1/3] eal/arm64: add 128-bit atomic compare exchange jerinj
2019-08-14 8:27 2% ` [dpdk-dev] [PATCH v9 " Phil Yang
2019-07-29 12:13 [dpdk-dev] [PATCH v9 1/5] mempool: populate mempool with the page sized chunks of memory vattunuru
2019-08-16 6:12 ` [dpdk-dev] [PATCH v10 0/5] kni: add IOVA=VA support vattunuru
2019-08-16 6:12 3% ` [dpdk-dev] [PATCH v10 3/5] kni: add app specific mempool create and free routines vattunuru
2019-07-30 12:49 [dpdk-dev] [RFC 19.11 0/2] Hide DPDK internal struct from public API Marcin Zapolski
2019-07-30 12:49 ` [dpdk-dev] [RFC 19.11 1/2] ethdev: make DPDK core functions non-inline Marcin Zapolski
2019-07-30 15:01 ` Jerin Jacob Kollanukkaran
2019-07-30 15:32 0% ` Bruce Richardson
2019-09-06 13:18 ` [dpdk-dev] [RFC 19.11 v2 0/3] Hide DPDK internal struct from public API Marcin Zapolski
2019-09-06 14:00 3% ` Bruce Richardson
2019-07-30 15:45 0% [dpdk-dev] [RFC 19.11 1/2] ethdev: make DPDK core functions non-inline Jerin Jacob Kollanukkaran
2019-07-30 16:05 5% ` Bruce Richardson
2019-07-30 16:24 3% ` [dpdk-dev] [EXT] " Jerin Jacob Kollanukkaran
2019-07-31 11:06 5% [dpdk-dev] [PATCH] doc: announce lcore_config symbol removal David Marchand
2019-07-31 13:48 0% ` Stephen Hemminger
2019-08-08 9:31 5% ` [dpdk-dev] [PATCH v2] " David Marchand
2019-08-02 15:38 11% [dpdk-dev] [PATCH v2 0/3] doc: changes to abi policy introducing major abi versions Ray Kinsella
2019-08-02 15:38 13% ` [dpdk-dev] [PATCH v2 1/3] doc: separate versioning.rst into version and policy Ray Kinsella
2019-08-02 15:38 32% ` [dpdk-dev] [PATCH v2 2/3] doc: changes to abi policy introducing major abi versions Ray Kinsella
2019-08-02 15:38 28% ` [dpdk-dev] [PATCH v2 3/3] doc: updates to versioning guide for " Ray Kinsella
2019-08-05 12:30 11% [dpdk-dev] [PATCH] doc: remove major in designation of normal releases Thomas Monjalon
2019-08-06 14:56 4% [dpdk-dev] [PATCH 1/2] doc: announce ethdev ABI change for LRO fields Matan Azrad
2019-08-06 15:27 4% ` Andrew Rybchenko
2019-08-10 21:40 4% ` Thomas Monjalon
2019-08-07 2:04 3% [dpdk-dev] [RFC 0/3] ethdev: add ptype as Rx offload Jerin Jacob Kollanukkaran
2019-08-07 8:32 0% ` Andrew Rybchenko
2019-08-07 15:22 0% ` Stephen Hemminger
2019-08-07 15:44 0% ` Andrew Rybchenko
2019-08-07 10:12 [dpdk-dev] [PATCH] eal: change max hugepage sizes to 4 Gagandeep Singh
2019-08-07 12:07 ` Thomas Monjalon
2019-08-07 13:28 ` Hemant Agrawal
2019-08-08 7:31 ` Thomas Monjalon
2019-08-12 9:43 3% ` Burakov, Anatoly
2019-08-12 9:49 0% ` David Marchand
2019-08-12 10:01 0% ` Thomas Monjalon
2019-08-12 10:38 0% ` Burakov, Anatoly
2019-08-08 8:12 4% [dpdk-dev] 19.11 Intel Roadmap O'Driscoll, Tim
2019-08-09 8:17 [dpdk-dev] [patch v4] doc: announce API change in ethdev offload flags pbhagavatula
2019-08-09 9:55 ` [dpdk-dev] [patch v5] " pbhagavatula
2019-08-09 10:13 ` Ananyev, Konstantin
2019-08-10 21:10 3% ` Thomas Monjalon
2019-08-11 16:06 [dpdk-dev] [PATCH v3 0/2] failsafe: add xstats Stephen Hemminger
2019-09-19 13:17 ` [dpdk-dev] [PATCH v5 " Stephen Hemminger
2019-09-19 13:17 ` [dpdk-dev] [PATCH v5 1/2] ethdev: expose basic xstats for driver use Stephen Hemminger
2019-09-26 12:46 3% ` Andrew Rybchenko
2019-09-26 16:09 0% ` Stephen Hemminger
2019-08-12 11:43 6% [dpdk-dev] [PATCH] version: 19.11-rc0 David Marchand
2019-08-13 12:18 6% ` [dpdk-dev] [PATCH v2] " David Marchand
2019-08-12 14:15 [dpdk-dev] [RFC v1 0/3] show the Rx/Tx burst description field Haiyue Wang
2019-08-12 14:15 ` [dpdk-dev] [RFC v1 1/3] ethdev: add the Rx/Tx burst description field in queue information Haiyue Wang
2019-08-12 15:37 3% ` Stephen Hemminger
2019-08-12 14:27 ` [dpdk-dev] [RFC v1 0/3] show the Rx/Tx burst description field David Marchand
2019-08-12 15:38 3% ` Stephen Hemminger
2019-08-12 15:42 0% ` Wang, Haiyue
2019-08-12 15:54 0% ` Stephen Hemminger
2019-08-12 16:00 0% ` Wang, Haiyue
2019-08-12 17:28 0% ` Stephen Hemminger
2019-08-12 17:36 0% ` Wang, Haiyue
2019-08-13 3:06 3% [dpdk-dev] [RFC v2 " Haiyue Wang
2019-08-13 10:02 [dpdk-dev] [PATCH 0/2] IXGBE vPMD changes for aarch64 Ruifeng Wang
2019-08-25 1:33 ` Ye Xiaolong
2019-08-26 2:52 ` Ruifeng Wang (Arm Technology China)
2019-08-26 10:39 3% ` Ferruh Yigit
2019-08-26 10:53 0% ` Ruifeng Wang (Arm Technology China)
2019-08-13 13:37 [dpdk-dev] [RFC] ethdev: support hairpin queue Ori Kam
2019-09-05 4:00 3% ` Wu, Jingjing
2019-09-05 5:44 0% ` Ori Kam
2019-09-06 3:08 0% ` Wu, Jingjing
2019-09-08 6:44 0% ` Ori Kam
2019-08-14 3:00 [dpdk-dev] [PATCH] ethdev: add more protocol support in flow API Wang Ying A
2019-08-14 3:24 ` [dpdk-dev] [PATCH v2] " Wang Ying A
2019-08-14 9:08 4% ` Adrien Mazarguil
2019-08-19 11:53 0% ` Zhang, Qi Z
2019-08-15 10:23 11% [dpdk-dev] [PATCH v3 0/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
2019-08-15 10:23 13% ` [dpdk-dev] [PATCH v3 1/4] doc: separate versioning.rst into version and policy Ray Kinsella
2019-08-15 10:23 31% ` [dpdk-dev] [PATCH v3 2/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
2019-08-30 16:20 10% ` Kevin Traynor
2019-09-24 11:32 10% ` Ray Kinsella
2019-09-25 12:29 5% ` Kevin Traynor
2019-08-15 10:23 30% ` [dpdk-dev] [PATCH v3 3/4] doc: updates to versioning guide for " Ray Kinsella
2019-08-15 10:23 13% ` [dpdk-dev] [PATCH v3 4/4] doc: add maintainer for abi policy Ray Kinsella
2019-08-15 15:06 [dpdk-dev] [RFC] ethdev: configure SR-IOV VF from host Thomas Monjalon
2019-08-15 15:34 3% ` Jerin Jacob Kollanukkaran
2019-08-15 17:59 0% ` Thomas Monjalon
2019-08-16 5:16 0% Jerin Jacob Kollanukkaran
2019-08-19 11:41 [dpdk-dev] [PATCH 00/11] Fixing log levels for dynamically loaded drivers David Marchand
2019-08-19 11:41 ` [dpdk-dev] [PATCH 02/11] log: define logtype register wrapper for drivers David Marchand
2019-09-02 14:29 ` Ferruh Yigit
2019-09-03 8:06 4% ` David Marchand
2019-09-03 8:47 3% ` Ferruh Yigit
2019-09-04 17:45 0% ` Thomas Monjalon
2019-08-20 9:37 [dpdk-dev] [PATCH] vhost: add __rte_experimental to rte_vhost_va_from_guest_pa Jim Harris
2019-09-18 13:12 ` Maxime Coquelin
2019-09-23 15:41 3% ` Ferruh Yigit
2019-08-22 8:42 [dpdk-dev] [PATCH v3] timer: use rte_mp_msg to get freq from primary process Jim Harris
2019-08-27 16:16 3% ` [dpdk-dev] [PATCH v6] eal: add tsc_hz to rte_mem_config Jim Harris
2019-08-22 16:07 9% [dpdk-dev] [PATCH v2 0/2] add abi version testing to app/test Ray Kinsella
2019-08-22 16:07 14% ` [dpdk-dev] [PATCH v2 1/2] app/test: add abi version testing functionality Ray Kinsella
2019-08-22 16:07 4% ` [dpdk-dev] [PATCH v2 2/2] app/test: lpm abi version testing Ray Kinsella
2019-08-23 15:49 4% ` [dpdk-dev] [PATCH v2 0/2] add abi version testing to app/test Aaron Conole
2019-08-26 16:45 9% ` Ray Kinsella
2019-08-27 8:17 7% ` Bruce Richardson
2019-08-27 8:28 8% ` Ray Kinsella
2019-08-27 14:19 7% ` Ray Kinsella
2019-08-23 14:45 [dpdk-dev] [PATCH 00/15] sched: subport level configuration of pipe nodes Jasvinder Singh
2019-08-23 14:46 4% ` [dpdk-dev] [PATCH 15/15] sched: remove redundant code Jasvinder Singh
2019-09-09 10:05 ` [dpdk-dev] [PATCH v2 00/15] sched: subport level configuration of pipe nodes Jasvinder Singh
2019-09-09 10:05 4% ` [dpdk-dev] [PATCH v2 15/15] sched: remove redundant code Jasvinder Singh
2019-09-26 8:52 ` [dpdk-dev] [PATCH v3 00/15] sched: subport level configuration of pipe nodes Jasvinder Singh
2019-09-26 8:52 4% ` [dpdk-dev] [PATCH v3 15/15] sched: remove redundant code Jasvinder Singh
2019-08-27 14:25 [dpdk-dev] [PATCH 00/51] ethdev: change rte_eth_dev_info_get() return value to int Andrew Rybchenko
2019-08-27 14:25 3% ` [dpdk-dev] [PATCH 01/51] " Andrew Rybchenko
2019-09-03 13:56 ` [dpdk-dev] [PATCH v2 00/54] " Andrew Rybchenko
2019-09-03 13:56 3% ` [dpdk-dev] [PATCH v2 02/54] " Andrew Rybchenko
2019-09-06 7:30 ` [dpdk-dev] [PATCH v3 00/54] " Andrew Rybchenko
2019-09-06 7:30 3% ` [dpdk-dev] [PATCH v3 02/54] " Andrew Rybchenko
2019-09-12 16:42 ` [dpdk-dev] [PATCH v4 00/54] " Andrew Rybchenko
2019-09-12 16:42 3% ` [dpdk-dev] [PATCH v4 02/54] " Andrew Rybchenko
2019-09-13 10:18 0% ` Iremonger, Bernard
2019-08-27 17:36 [dpdk-dev] [RFC] Proposal to remove EXPERIMENTAL label from compressdev API Trahe, Fiona
2019-09-23 10:51 3% ` Trahe, Fiona
2019-08-28 6:51 [dpdk-dev] [RFC] hash: introduce resizable hash list Bing Zhao
2019-08-28 6:51 ` [dpdk-dev] [RFC] rte_hash: introduce hash list into hash lib Bing Zhao
2019-08-28 11:53 3% ` Stephen Hemminger
2019-08-29 7:47 [dpdk-dev] [RFC] ethdev: add new fields for max LRO session size Matan Azrad
2019-09-13 17:50 ` Ferruh Yigit
2019-09-15 7:48 ` Matan Azrad
2019-09-16 15:37 ` Ferruh Yigit
2019-09-24 12:03 3% ` Matan Azrad
2019-08-29 7:59 [dpdk-dev] [PATCH 00/15] Introduce Virtio vDPA driver Maxime Coquelin
2019-08-29 7:59 4% ` [dpdk-dev] [PATCH 04/15] net/virtio: add virtio PCI subsystem device ID declaration Maxime Coquelin
2019-09-02 6:14 0% ` Tiwei Bie
2019-09-03 7:25 0% ` Maxime Coquelin
2019-09-03 15:40 [dpdk-dev] [RFC PATCH 0/9] security: add software synchronous crypto process Fan Zhang
2019-09-03 15:40 ` [dpdk-dev] [RFC PATCH 1/9] security: introduce CPU Crypto action type and API Fan Zhang
2019-09-04 10:32 ` Akhil Goyal
2019-09-04 13:06 ` Zhang, Roy Fan
2019-09-06 9:01 ` Akhil Goyal
2019-09-06 13:27 4% ` Ananyev, Konstantin
2019-09-10 10:44 4% ` Akhil Goyal
2019-09-11 12:29 4% ` Ananyev, Konstantin
2019-09-12 14:12 5% ` Akhil Goyal
2019-09-16 14:53 3% ` Ananyev, Konstantin
2019-09-16 15:08 0% ` Ananyev, Konstantin
2019-09-17 6:02 3% ` Akhil Goyal
2019-09-18 7:44 4% ` Ananyev, Konstantin
2019-09-25 18:24 4% ` Ananyev, Konstantin
2019-09-27 9:26 0% ` Akhil Goyal
2019-09-05 15:47 6% [dpdk-dev] [PATCH 1/2] version: 19.11-rc0 agupta3
2019-09-05 16:10 [dpdk-dev] [PATCH 00/13] ethdev: change promiscuous mode functions to return status Andrew Rybchenko
2019-09-05 16:10 3% ` [dpdk-dev] [PATCH 01/13] ethdev: change promiscuous mode controllers to return errors Andrew Rybchenko
2019-09-09 11:58 ` [dpdk-dev] [PATCH v2 00/13] ethdev: change promiscuous mode functions to return status Andrew Rybchenko
2019-09-09 11:58 3% ` [dpdk-dev] [PATCH v2 01/13] ethdev: change promiscuous mode controllers to return errors Andrew Rybchenko
2019-09-14 11:37 ` [dpdk-dev] [PATCH v3 00/13] ethdev: change promiscuous mode functions to return status Andrew Rybchenko
2019-09-14 11:37 3% ` [dpdk-dev] [PATCH v3 01/13] ethdev: change promiscuous mode controllers to return errors Andrew Rybchenko
2019-09-06 14:34 [dpdk-dev] [PATCH 0/2] ethdev: change xstats reset function return value to int Andrew Rybchenko
2019-09-06 14:34 4% ` [dpdk-dev] [PATCH 1/2] " Andrew Rybchenko
[not found] <20190806182500.22320>
2019-08-29 14:12 ` [dpdk-dev] [PATCH v6 00/10] vhost: support inflight share memory protocol feature JinYu
2019-08-29 14:12 ` [dpdk-dev] [PATCH v6 02/10] vhost: add packed ring JinYu
2019-09-06 16:42 3% ` Maxime Coquelin
2019-09-09 12:13 [dpdk-dev] [PATCH 0/7] ethdev: change allmulticast controls to return status Andrew Rybchenko
2019-09-09 12:13 4% ` [dpdk-dev] [PATCH 1/7] ethdev: change allmulticast mode controllers to return errors Andrew Rybchenko
2019-09-10 8:25 [dpdk-dev] [PATCH 00/18] ethdev: change link status get functions return value to int Andrew Rybchenko
2019-09-10 8:25 3% ` [dpdk-dev] [PATCH 02/18] " Andrew Rybchenko
2019-09-10 8:52 [dpdk-dev] [PATCH 0/7] ethdev: change MAC addr get function " Andrew Rybchenko
2019-09-10 8:52 4% ` [dpdk-dev] [PATCH 1/7] " Andrew Rybchenko
2019-09-10 9:02 [dpdk-dev] [PATCH 0/1] ethdev: change owner delete " Andrew Rybchenko
2019-09-10 9:02 4% ` [dpdk-dev] [PATCH 1/1] " Andrew Rybchenko
2019-09-10 16:33 3% [dpdk-dev] [RFC v3 0/4] get Rx/Tx packet burst mode information Haiyue Wang
2019-09-11 9:19 [dpdk-dev] [PATCH] mbuf: add bulk free function Morten Brørup
2019-09-11 11:18 ` Stephen Hemminger
2019-09-11 11:33 ` Olivier Matz
2019-09-11 11:39 3% ` Stephen Hemminger
2019-09-11 15:04 5% [dpdk-dev] DPDK techboard minutes of July 31 Maxime Coquelin
2019-09-11 15:05 3% [dpdk-dev] DPDK techboard minutes (2019-07-17) Olivier Matz
2019-09-12 16:46 5% [dpdk-dev] DPDK techboard minutes of September 11 Thomas Monjalon
2019-09-13 13:32 [dpdk-dev] The type string in the malloc library is unused Morten Brørup
[not found] ` <CAOaVG17_bhBOfJdvHZOzoVO3t7CXCZCa7f=_Q=_Pi5jgnY3GzA@mail.gmail.com>
2019-09-14 8:29 3% ` Morten Brørup
2019-09-17 9:09 [dpdk-dev] [PATCH v2 1/3] net/ifcvf: add multiqueue configuration Andy Pei
2019-09-17 9:09 ` [dpdk-dev] [PATCH v2 2/3] vhost: call vDPA callback at the end of vring enable handler Andy Pei
2019-09-23 8:12 3% ` Tiwei Bie
2019-09-19 16:36 [dpdk-dev] [PATCH v2 01/16] vhost: add single packet enqueue function Marvin Liu
2019-09-25 17:13 ` [dpdk-dev] [PATCH v3 00/15] vhost packed ring performance optimization Marvin Liu
2019-09-25 17:13 ` [dpdk-dev] [PATCH v3 13/15] vhost: cache address translation result Marvin Liu
2019-09-26 5:32 3% ` Tiwei Bie
2019-09-23 16:19 5% [dpdk-dev] RFC: hiding struct rte_eth_dev Ray Kinsella
2019-09-23 16:35 0% ` Bruce Richardson
2019-09-24 9:07 0% ` Morten Brørup
2019-09-24 16:42 0% ` Jerin Jacob
2019-09-24 16:50 3% ` Ananyev, Konstantin
2019-09-26 11:13 4% ` Andrew Rybchenko
2019-09-26 11:50 0% ` David Marchand
2019-09-26 11:52 0% ` David Marchand
2019-09-23 17:51 9% [dpdk-dev] [RFC] Proposals and notes from ABI stability panel @ DPDK Userspace Ray Kinsella
2019-09-25 13:31 8% ` Kevin Traynor
2019-09-25 14:29 9% ` Ray Kinsella
2019-09-25 14:40 7% ` [dpdk-dev] [dpdk-techboard] " Bruce Richardson
2019-09-25 14:49 4% ` Kevin Traynor
2019-09-25 15:06 4% ` Ray Kinsella
2019-09-24 17:39 3% [dpdk-dev] [PATCH v6] eal: make lcore_config private Stephen Hemminger
2019-09-25 10:23 10% [dpdk-dev] [PATCH v4 0/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
2019-09-25 10:23 13% ` [dpdk-dev] [PATCH v4 1/4] doc: separate versioning.rst into version and policy Ray Kinsella
2019-09-25 12:24 5% ` Neil Horman
2019-09-25 13:01 3% ` Ray Kinsella
2019-09-25 14:34 0% ` Neil Horman
2019-09-27 15:22 4% ` Ray Kinsella
2019-09-27 15:43 4% ` Aaron Conole
2019-09-27 16:43 4% ` Ray Kinsella
2019-09-25 10:23 31% ` [dpdk-dev] [PATCH v4 2/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
2019-09-25 10:23 30% ` [dpdk-dev] [PATCH v4 3/4] doc: updates to versioning guide for " Ray Kinsella
2019-09-25 10:23 13% ` [dpdk-dev] [PATCH v4 4/4] doc: add maintainer for abi policy Ray Kinsella
2019-09-25 14:26 [dpdk-dev] [PATCH v2 00/17] Add advanced features for Huawei hinic pmd Xiaoyun wang
2019-09-25 14:30 4% ` [dpdk-dev] [PATCH v2 15/17] net/hinic: add hinic PMD doc files Xiaoyun wang
2019-09-26 18:51 0% ` Ferruh Yigit
[not found] <20190917145234.16951>
2019-09-20 12:00 ` [dpdk-dev] [PATCH v7 00/10] vhost: support inflight share memory protocol feature Jin Yu
2019-09-20 12:01 ` [dpdk-dev] [PATCH v7 10/10] vhost: add vhost-user-blk example which support inflight Jin Yu
2019-09-25 14:45 3% ` Tiwei Bie
2019-09-26 14:29 0% ` Yu, Jin
2019-09-26 14:40 0% ` Tiwei Bie
2019-09-25 16:10 3% [dpdk-dev] [PATCH v7] eal: make lcore_config private Stephen Hemminger
2019-09-26 11:48 [dpdk-dev] [PATCH v1 0/4] get Rx/Tx packet burst mode information Haiyue Wang
2019-09-26 15:57 ` Stephen Hemminger
2019-09-26 16:36 3% ` Wang, Haiyue
2019-09-26 17:15 3% ` Stephen Hemminger
2019-09-26 17:36 0% ` Ferruh Yigit
2019-09-27 1:17 0% ` Wang, Haiyue
2019-09-27 14:02 5% [dpdk-dev] Minutes of Technical Board Meeting, 2019-09-25 Bruce Richardson
2019-09-27 16:30 10% [dpdk-dev] [PATCH v5 0/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
2019-09-27 16:30 13% ` [dpdk-dev] [PATCH v5 1/4] doc: separate versioning.rst into version and policy Ray Kinsella
2019-09-27 16:30 19% ` [dpdk-dev] [PATCH v5 2/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
2019-09-27 16:30 30% ` [dpdk-dev] [PATCH v5 3/4] doc: updates to versioning guide for " Ray Kinsella
2019-09-27 16:30 13% ` [dpdk-dev] [PATCH v5 4/4] doc: add maintainer for abi policy Ray Kinsella
2019-09-27 16:54 10% [dpdk-dev] [PATCH v6 0/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
2019-09-27 16:54 13% ` [dpdk-dev] [PATCH v6 1/4] doc: separate versioning.rst into version and policy Ray Kinsella
2019-09-27 16:54 12% ` [dpdk-dev] [PATCH v6 2/4] doc: changes to abi policy introducing major abi versions Ray Kinsella
2019-09-27 16:54 30% ` [dpdk-dev] [PATCH v6 3/4] doc: updates to versioning guide for " Ray Kinsella
2019-09-27 16:54 13% ` [dpdk-dev] [PATCH v6 4/4] doc: add maintainer for abi policy Ray Kinsella
2019-09-27 19:49 3% [dpdk-dev] [PATCH 0/2] Improve function versioning meson support Bruce Richardson
2019-09-27 19:49 11% ` [dpdk-dev] [PATCH 1/2] eal: split compat header file Bruce Richardson
2019-09-27 19:49 15% ` [dpdk-dev] [PATCH 2/2] build: support building ABI versioned files twice Bruce Richardson
2019-09-27 20:59 3% ` [dpdk-dev] [PATCH v2 0/2] Improve function versioning meson support Bruce Richardson
2019-09-27 20:59 10% ` [dpdk-dev] [PATCH v2 1/2] eal: split compat header file Bruce Richardson
2019-09-27 20:59 15% ` [dpdk-dev] [PATCH v2 2/2] build: support building ABI versioned files twice Bruce Richardson
2019-09-28 0:37 3% [dpdk-dev] [PATCH 0/5] mbuf related patches Stephen Hemminger
2019-09-30 9:21 8% [dpdk-dev] [PATCH 1/8] config: change ABI versioning for global Marcin Baran
2019-09-30 9:21 10% ` [dpdk-dev] [PATCH 2/8] buildtools: scripts for updating symbols abi version Marcin Baran
2019-09-30 9:21 23% ` [dpdk-dev] [PATCH 3/8] buildtools: add ABI versioning check script Marcin Baran
2019-09-30 9:21 1% ` [dpdk-dev] [PATCH 4/8] build: change ABI version to 20.0 Marcin Baran
2019-09-30 9:21 2% ` [dpdk-dev] [PATCH 5/8] lib: remove dead code from timer Marcin Baran
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).